root
/
xz
mirror of https://git.tukaani.org/xz.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Added support for flush marker, which will be in files 

that use LZMA_SYNC_FLUSH with encoder (not implemented
yet). This is a new feature in the raw LZMA format,
which isn't supported by old decoders. This shouldn't
be a problem in practice, since lzma_alone_encoder()
will not allow LZMA_SYNC_FLUSH, and thus not allow
creating files on decodable with old decoders.

Made lzma_decoder.c to require tab width of 4 characters
if one wants to fit the code in 80 columns. This makes
the code easier to read.
This commit is contained in:
Lasse Collin 2008-01-04 21:30:33 +02:00
parent bbfd1f6ab0
commit 0029cbbabe
2 changed files with 105 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/liblzma/lzma/lzma_common.h
View File

@ -81,6 +81,10 @@
// Price table size of Len Encoder // Price table size of Len Encoder
#define LEN_PRICES (LEN_SYMBOLS << POS_STATES_BITS_MAX) #define LEN_PRICES (LEN_SYMBOLS << POS_STATES_BITS_MAX)
// Special lengths used together with distance == UINT32_MAX
#define LEN_SPECIAL_EOPM MATCH_MIN_LEN
#define LEN_SPECIAL_FLUSH (LEN_SPECIAL_EOPM + 1)
// Optimal - Number of entries in the optimum array. // Optimal - Number of entries in the optimum array.
#define OPTS (1 << 12) #define OPTS (1 << 12)

203
src/liblzma/lzma/lzma_decoder.c
View File

@ -18,6 +18,9 @@
// //
/////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////
// NOTE: If you want to keep the line length in 80 characters, set
// tab width to 4 or less in your editor when editing this file.
#include "lzma_common.h" #include "lzma_common.h"
#include "lzma_decoder.h" #include "lzma_decoder.h"
#include "lz_decoder.h" #include "lz_decoder.h"
@ -44,21 +47,17 @@ do { \
if_bit_0(len_decoder.choice) { \ if_bit_0(len_decoder.choice) { \
update_bit_0(len_decoder.choice); \ update_bit_0(len_decoder.choice); \
target = MATCH_MIN_LEN; \ target = MATCH_MIN_LEN; \
bittree_decode(target, \ bittree_decode(target, len_decoder.low[pos_state], LEN_LOW_BITS); \
len_decoder.low[pos_state], LEN_LOW_BITS); \
} else { \ } else { \
update_bit_1(len_decoder.choice); \ update_bit_1(len_decoder.choice); \
if_bit_0(len_decoder.choice2) { \ if_bit_0(len_decoder.choice2) { \
update_bit_0(len_decoder.choice2); \ update_bit_0(len_decoder.choice2); \
target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \
bittree_decode(target, len_decoder.mid[pos_state], \ bittree_decode(target, len_decoder.mid[pos_state], LEN_MID_BITS); \
LEN_MID_BITS); \
} else { \ } else { \
update_bit_1(len_decoder.choice2); \ update_bit_1(len_decoder.choice2); \
target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS \ target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \
+ LEN_MID_SYMBOLS; \ bittree_decode(target, len_decoder.high, LEN_HIGH_BITS); \
bittree_decode(target, len_decoder.high, \
LEN_HIGH_BITS); \
} \ } \
} \ } \
} while (0) } while (0)
@ -76,15 +75,12 @@ do { \
if_bit_0(len_decoder.choice2) { \ if_bit_0(len_decoder.choice2) { \
update_bit_0_dummy(); \ update_bit_0_dummy(); \
target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \ target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS; \
bittree_decode_dummy(target, \ bittree_decode_dummy(target, len_decoder.mid[pos_state], \
len_decoder.mid[pos_state], \
LEN_MID_BITS); \ LEN_MID_BITS); \
} else { \ } else { \
update_bit_1_dummy(); \ update_bit_1_dummy(); \
target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS \ target = MATCH_MIN_LEN + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \
+ LEN_MID_SYMBOLS; \ bittree_decode_dummy(target, len_decoder.high, LEN_HIGH_BITS); \
bittree_decode_dummy(target, len_decoder.high, \
LEN_HIGH_BITS); \
} \ } \
} \ } \
} while (0) } while (0)
@ -151,6 +147,10 @@ struct lzma_coder_s {
/// Length of a repeated match. /// Length of a repeated match.
lzma_length_decoder rep_match_len_decoder; lzma_length_decoder rep_match_len_decoder;
/// True when we have produced at least one byte of output since the
/// beginning of the stream or the latest flush marker.
bool has_produced_output;
}; };
@ -176,23 +176,19 @@ decode_dummy(const lzma_coder *restrict coder,
update_bit_0_dummy(); update_bit_0_dummy();
const probability *subcoder = literal_get_subcoder( const probability *subcoder = literal_get_subcoder(
coder->literal_coder, coder->literal_coder, now_pos, lz_get_byte(coder->lz, 0));
now_pos, lz_get_byte(coder->lz, 0));
uint32_t symbol = 1; uint32_t symbol = 1;
if (!is_char_state(state)) { if (!is_char_state(state)) {
// Decode literal with match byte. // Decode literal with match byte.
assert(rep0 != UINT32_MAX); assert(rep0 != UINT32_MAX);
uint32_t match_byte uint32_t match_byte = lz_get_byte(coder->lz, rep0);
= lz_get_byte(coder->lz, rep0);
do { do {
match_byte <<= 1; match_byte <<= 1;
const uint32_t match_bit const uint32_t match_bit = match_byte & 0x100;
= match_byte & 0x100; const uint32_t subcoder_index = 0x100 + match_bit + symbol;
const uint32_t subcoder_index = 0x100
+ match_bit + symbol;
if_bit_0(subcoder[subcoder_index]) { if_bit_0(subcoder[subcoder_index]) {
update_bit_0_dummy(); update_bit_0_dummy();
@ -231,11 +227,10 @@ decode_dummy(const lzma_coder *restrict coder,
length_decode_dummy(len, coder->len_decoder, pos_state); length_decode_dummy(len, coder->len_decoder, pos_state);
update_match(state); update_match(state);
const uint32_t len_to_pos_state const uint32_t len_to_pos_state = get_len_to_pos_state(len);
= get_len_to_pos_state(len);
uint32_t pos_slot = 0; uint32_t pos_slot = 0;
bittree_decode_dummy(pos_slot, coder->pos_slot_decoder[ bittree_decode_dummy(pos_slot,
len_to_pos_state], POS_SLOT_BITS); coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS);
assert(pos_slot <= 63); assert(pos_slot <= 63);
if (pos_slot >= START_POS_MODEL_INDEX) { if (pos_slot >= START_POS_MODEL_INDEX) {
@ -247,21 +242,15 @@ decode_dummy(const lzma_coder *restrict coder,
assert(direct_bits <= 5); assert(direct_bits <= 5);
rep0 <<= direct_bits; rep0 <<= direct_bits;
assert(rep0 <= 96); assert(rep0 <= 96);
// -1 is fine, because // -1 is fine, because bittree_reverse_decode()
// bittree_reverse_decode() // starts from table index [1] (not [0]).
// starts from table index [1] assert((int32_t)(rep0 - pos_slot - 1) >= -1);
// (not [0]). assert((int32_t)(rep0 - pos_slot - 1) <= 82);
assert((int32_t)(rep0 - pos_slot - 1)
>= -1);
assert((int32_t)(rep0 - pos_slot - 1)
<= 82);
// We add the result to rep0, so rep0 // We add the result to rep0, so rep0
// must not be part of second argument // must not be part of second argument
// of the macro. // of the macro.
const int32_t offset const int32_t offset = rep0 - pos_slot - 1;
= rep0 - pos_slot - 1; bittree_reverse_decode_dummy(coder->pos_decoders + offset,
bittree_reverse_decode_dummy(
coder->pos_decoders + offset,
direct_bits); direct_bits);
} else { } else {
assert(pos_slot >= 14); assert(pos_slot >= 14);
@ -270,8 +259,7 @@ decode_dummy(const lzma_coder *restrict coder,
assert(direct_bits >= 2); assert(direct_bits >= 2);
rc_decode_direct_dummy(direct_bits); rc_decode_direct_dummy(direct_bits);
bittree_reverse_decode_dummy( bittree_reverse_decode_dummy(coder->pos_align_decoder,
coder->pos_align_decoder,
ALIGN_BITS); ALIGN_BITS);
} }
} }
@ -282,8 +270,7 @@ decode_dummy(const lzma_coder *restrict coder,
if_bit_0(coder->is_rep0[state]) { if_bit_0(coder->is_rep0[state]) {
update_bit_0_dummy(); update_bit_0_dummy();
if_bit_0(coder->is_rep0_long[state][ if_bit_0(coder->is_rep0_long[state][pos_state]) {
pos_state]) {
update_bit_0_dummy(); update_bit_0_dummy();
break; break;
} else { } else {
@ -306,18 +293,13 @@ decode_dummy(const lzma_coder *restrict coder,
} }
} }
length_decode_dummy(len, coder->rep_match_len_decoder, length_decode_dummy(len, coder->rep_match_len_decoder, pos_state);
pos_state);
} }
} while (0); } while (0);
rc_normalize(); rc_normalize();
// Validate the buffer position. return in_pos_local <= in_size;
if (in_pos_local > in_size)
return false;
return true;
} }
@ -351,6 +333,7 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// Misc // Misc
uint32_t now_pos = coder->now_pos; uint32_t now_pos = coder->now_pos;
bool has_produced_output = coder->has_produced_output;
// Variables derived from decoder settings // Variables derived from decoder settings
const uint32_t pos_mask = coder->pos_mask; const uint32_t pos_mask = coder->pos_mask;
@ -363,9 +346,9 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
in_limit = in_size - REQUIRED_IN_BUFFER_SIZE; in_limit = in_size - REQUIRED_IN_BUFFER_SIZE;
while (coder->lz.pos < coder->lz.limit && (in_pos_local < in_limit while (coder->lz.pos < coder->lz.limit
|| (has_safe_buffer && decode_dummy( && (in_pos_local < in_limit || (has_safe_buffer
coder, in, in_pos_local, in_size, && decode_dummy(coder, in, in_pos_local, in_size,
rc, state, rep0, now_pos)))) { rc, state, rep0, now_pos)))) {
///////////////////// /////////////////////
@ -379,8 +362,7 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// It's a literal i.e. a single 8-bit byte. // It's a literal i.e. a single 8-bit byte.
probability *subcoder = literal_get_subcoder( probability *subcoder = literal_get_subcoder(coder->literal_coder,
coder->literal_coder,
now_pos, lz_get_byte(coder->lz, 0)); now_pos, lz_get_byte(coder->lz, 0));
uint32_t symbol = 1; uint32_t symbol = 1;
@ -388,25 +370,20 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// Decode literal with match byte. // Decode literal with match byte.
assert(rep0 != UINT32_MAX); assert(rep0 != UINT32_MAX);
uint32_t match_byte uint32_t match_byte = lz_get_byte(coder->lz, rep0);
= lz_get_byte(coder->lz, rep0);
do { do {
match_byte <<= 1; match_byte <<= 1;
const uint32_t match_bit const uint32_t match_bit = match_byte & 0x100;
= match_byte & 0x100; const uint32_t subcoder_index = 0x100 + match_bit + symbol;
const uint32_t subcoder_index = 0x100
+ match_bit + symbol;
if_bit_0(subcoder[subcoder_index]) { if_bit_0(subcoder[subcoder_index]) {
update_bit_0(subcoder[ update_bit_0(subcoder[subcoder_index]);
subcoder_index]);
symbol <<= 1; symbol <<= 1;
if (match_bit != 0) if (match_bit != 0)
break; break;
} else { } else {
update_bit_1(subcoder[ update_bit_1(subcoder[subcoder_index]);
subcoder_index]);
symbol = (symbol << 1) | 1; symbol = (symbol << 1) | 1;
if (match_bit == 0) if (match_bit == 0)
break; break;
@ -431,6 +408,7 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol); coder->lz.dict[coder->lz.pos++] = (uint8_t)(symbol);
++now_pos; ++now_pos;
update_char(state); update_char(state);
has_produced_output = true;
continue; continue;
} }
@ -460,11 +438,10 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
update_match(state); update_match(state);
const uint32_t len_to_pos_state const uint32_t len_to_pos_state = get_len_to_pos_state(len);
= get_len_to_pos_state(len);
uint32_t pos_slot = 0; uint32_t pos_slot = 0;
bittree_decode(pos_slot, coder->pos_slot_decoder[ bittree_decode(pos_slot,
len_to_pos_state], POS_SLOT_BITS); coder->pos_slot_decoder[len_to_pos_state], POS_SLOT_BITS);
assert(pos_slot <= 63); assert(pos_slot <= 63);
if (pos_slot >= START_POS_MODEL_INDEX) { if (pos_slot >= START_POS_MODEL_INDEX) {
@ -480,17 +457,13 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// bittree_reverse_decode() // bittree_reverse_decode()
// starts from table index [1] // starts from table index [1]
// (not [0]). // (not [0]).
assert((int32_t)(rep0 - pos_slot - 1) assert((int32_t)(rep0 - pos_slot - 1) >= -1);
>= -1); assert((int32_t)(rep0 - pos_slot - 1) <= 82);
assert((int32_t)(rep0 - pos_slot - 1)
<= 82);
// We add the result to rep0, so rep0 // We add the result to rep0, so rep0
// must not be part of second argument // must not be part of second argument
// of the macro. // of the macro.
const int32_t offset const int32_t offset = rep0 - pos_slot - 1;
= rep0 - pos_slot - 1; bittree_reverse_decode(rep0, coder->pos_decoders + offset,
bittree_reverse_decode(rep0,
coder->pos_decoders + offset,
direct_bits); direct_bits);
} else { } else {
assert(pos_slot >= 14); assert(pos_slot >= 14);
@ -500,14 +473,33 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
rc_decode_direct(rep0, direct_bits); rc_decode_direct(rep0, direct_bits);
rep0 <<= ALIGN_BITS; rep0 <<= ALIGN_BITS;
bittree_reverse_decode(rep0, bittree_reverse_decode(rep0, coder->pos_align_decoder,
coder->pos_align_decoder,
ALIGN_BITS); ALIGN_BITS);
if (rep0 == UINT32_MAX) { if (rep0 == UINT32_MAX) {
if (len == LEN_SPECIAL_EOPM) {
// End of Payload Marker found. // End of Payload Marker found.
coder->lz.eopm_detected = true; coder->lz.eopm_detected = true;
break; break;
} else if (len == LEN_SPECIAL_FLUSH) {
// Flush marker detected. We must have produced
// at least one byte of output since the previous
// flush marker or the beginning of the stream.
// This is to prevent hanging the decoder with
// malicious input files.
if (!coder->has_produced_output)
return true;
coder->has_produced_output = false;
rc_reset(rc);
if (!rc_read_init(&rc, in, &in_pos_local, in_size))
break;
} else {
return true;
}
} }
} }
} else { } else {
@ -529,10 +521,8 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// The distance is rep0. // The distance is rep0.
if_bit_0(coder->is_rep0_long[state][ if_bit_0(coder->is_rep0_long[state][pos_state]) {
pos_state]) { update_bit_0(coder->is_rep0_long[state][pos_state]);
update_bit_0(coder->is_rep0_long[
state][pos_state]);
// Repeating exactly one byte. For // Repeating exactly one byte. For
// simplicity, it is done here inline // simplicity, it is done here inline
@ -544,24 +534,21 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// Security/sanity checks. See the end // Security/sanity checks. See the end
// of the main loop for explanation // of the main loop for explanation
// of these. // of these.
if ((rep0 >= coder->lz.pos if ((rep0 >= coder->lz.pos && !coder->lz.is_full)
&& !coder->lz.is_full) || in_pos_local > in_size)
|| in_pos_local return true;
> in_size)
goto error;
// Repeat one byte and start a new // Repeat one byte and start a new
// decoding loop. // decoding loop.
coder->lz.dict[coder->lz.pos] coder->lz.dict[coder->lz.pos]
= lz_get_byte( = lz_get_byte(coder->lz, rep0);
coder->lz, rep0);
++coder->lz.pos; ++coder->lz.pos;
++now_pos; ++now_pos;
has_produced_output = true;
continue; continue;
} else { } else {
update_bit_1(coder->is_rep0_long[ update_bit_1(coder->is_rep0_long[state][pos_state]);
state][pos_state]);
// Repeating more than one byte at // Repeating more than one byte at
// distance of rep0. // distance of rep0.
@ -584,12 +571,10 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
update_bit_1(coder->is_rep1[state]); update_bit_1(coder->is_rep1[state]);
if_bit_0(coder->is_rep2[state]) { if_bit_0(coder->is_rep2[state]) {
update_bit_0(coder->is_rep2[ update_bit_0(coder->is_rep2[state]);
state]);
distance = rep2; distance = rep2;
} else { } else {
update_bit_1(coder->is_rep2[ update_bit_1(coder->is_rep2[state]);
state]);
distance = rep3; distance = rep3;
rep3 = rep2; rep3 = rep2;
} }
@ -602,8 +587,7 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
} }
// Decode the length of the repeated match. // Decode the length of the repeated match.
length_decode(len, coder->rep_match_len_decoder, length_decode(len, coder->rep_match_len_decoder, pos_state);
pos_state);
update_rep(state); update_rep(state);
} }
@ -619,15 +603,16 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
assert(len <= MATCH_MAX_LEN); assert(len <= MATCH_MAX_LEN);
now_pos += len; now_pos += len;
has_produced_output = true;
// Validate the buffer position to avoid buffer overflows // Validate the buffer position to avoid buffer overflows
// on corrupted input data. // on corrupted input data.
if (in_pos_local > in_size) if (in_pos_local > in_size)
goto error; return true;
// Repeat len bytes from distance of rep0. // Repeat len bytes from distance of rep0.
if (!lzma_lz_out_repeat(&coder->lz, rep0, len)) if (!lzma_lz_out_repeat(&coder->lz, rep0, len))
goto error; return true;
} }
rc_normalize(); rc_normalize();
@ -649,12 +634,10 @@ decode_real(lzma_coder *restrict coder, const uint8_t *restrict in,
// Misc // Misc
coder->now_pos = now_pos; coder->now_pos = now_pos;
coder->has_produced_output = has_produced_output;
*in_pos = in_pos_local; *in_pos = in_pos_local;
return false; return false;
error:
return true;
} }
@ -766,20 +749,20 @@ lzma_lzma_decoder_init(lzma_next_coder *next, lzma_allocator *allocator,
bit_reset(next->coder->rep_match_len_decoder.choice2); bit_reset(next->coder->rep_match_len_decoder.choice2);
for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) { for (uint32_t pos_state = 0; pos_state < num_pos_states; ++pos_state) {
bittree_reset(next->coder->len_decoder.low[pos_state], bittree_reset(next->coder->len_decoder.low[pos_state], LEN_LOW_BITS);
LEN_LOW_BITS); bittree_reset(next->coder->len_decoder.mid[pos_state], LEN_MID_BITS);
bittree_reset(next->coder->len_decoder.mid[pos_state],
LEN_MID_BITS);
bittree_reset(next->coder->rep_match_len_decoder.low[ bittree_reset(next->coder->rep_match_len_decoder.low[pos_state],
pos_state], LEN_LOW_BITS); LEN_LOW_BITS);
bittree_reset(next->coder->rep_match_len_decoder.mid[ bittree_reset(next->coder->rep_match_len_decoder.mid[pos_state],
pos_state], LEN_MID_BITS); LEN_MID_BITS);
} }
bittree_reset(next->coder->len_decoder.high, LEN_HIGH_BITS); bittree_reset(next->coder->len_decoder.high, LEN_HIGH_BITS);
bittree_reset(next->coder->rep_match_len_decoder.high, LEN_HIGH_BITS); bittree_reset(next->coder->rep_match_len_decoder.high, LEN_HIGH_BITS);
next->coder->has_produced_output = false;
// Initialize the next decoder in the chain, if any. // Initialize the next decoder in the chain, if any.
{ {
const lzma_ret ret = lzma_next_filter_init(&next->coder->next, const lzma_ret ret = lzma_next_filter_init(&next->coder->next,