mirror of https://git.tukaani.org/xz.git
961 lines
29 KiB
C
961 lines
29 KiB
C
// SPDX-License-Identifier: 0BSD
|
|
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
//
|
|
/// \file range_decoder.h
|
|
/// \brief Range Decoder
|
|
///
|
|
// Authors: Igor Pavlov
|
|
// Lasse Collin
|
|
//
|
|
///////////////////////////////////////////////////////////////////////////////
|
|
|
|
#ifndef LZMA_RANGE_DECODER_H
|
|
#define LZMA_RANGE_DECODER_H
|
|
|
|
#include "range_common.h"
|
|
|
|
|
|
// Choose the range decoder variants to use using a bitmask.
|
|
// If no bits are set, only the basic version is used.
|
|
// If more than one version is selected for the same feature,
|
|
// the last one on the list below is used.
|
|
//
|
|
// Bitwise-or of the following enable branchless C versions:
|
|
// 0x01 normal bittrees
|
|
// 0x02 fixed-sized reverse bittrees
|
|
// 0x04 variable-sized reverse bittrees (disabled by default, not faster?)
|
|
// 0x08 matched literal (disabled by default, not faster?)
|
|
//
|
|
// GCC & Clang compatible x86-64 inline assembly:
|
|
// 0x010 normal bittrees
|
|
// 0x020 fixed-sized reverse bittrees
|
|
// 0x040 variable-sized reverse bittrees
|
|
// 0x080 matched literal
|
|
// 0x100 direct bits
|
|
//
|
|
// The default can be overridden at build time by defining
|
|
// LZMA_RANGE_DECODER_CONFIG to the desired mask.
|
|
#ifndef LZMA_RANGE_DECODER_CONFIG
|
|
# if defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__))
|
|
# define LZMA_RANGE_DECODER_CONFIG 0x1F0
|
|
# else
|
|
# define LZMA_RANGE_DECODER_CONFIG 0x03
|
|
# endif
|
|
#endif
|
|
|
|
|
|
// Negative RC_BIT_MODEL_TOTAL but the lowest RC_MOVE_BITS are flipped.
|
|
// This is useful for updating probability variables in branchless decoding:
|
|
//
|
|
// uint32_t decoded_bit = ...;
|
|
// probability tmp = RC_BIT_MODEL_OFFSET;
|
|
// tmp &= decoded_bit - 1;
|
|
// prob -= (prob + tmp) >> RC_MOVE_BITS;
|
|
#define RC_BIT_MODEL_OFFSET \
|
|
((UINT32_C(1) << RC_MOVE_BITS) - 1 - RC_BIT_MODEL_TOTAL)
|
|
|
|
|
|
typedef struct {
|
|
uint32_t range;
|
|
uint32_t code;
|
|
uint32_t init_bytes_left;
|
|
} lzma_range_decoder;
|
|
|
|
|
|
/// Reads the first five bytes to initialize the range decoder.
|
|
static inline lzma_ret
|
|
rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in,
|
|
size_t *restrict in_pos, size_t in_size)
|
|
{
|
|
while (rc->init_bytes_left > 0) {
|
|
if (*in_pos == in_size)
|
|
return LZMA_OK;
|
|
|
|
// The first byte is always 0x00. It could have been omitted
|
|
// in LZMA2 but it wasn't, so one byte is wasted in every
|
|
// LZMA2 chunk.
|
|
if (rc->init_bytes_left == 5 && in[*in_pos] != 0x00)
|
|
return LZMA_DATA_ERROR;
|
|
|
|
rc->code = (rc->code << 8) | in[*in_pos];
|
|
++*in_pos;
|
|
--rc->init_bytes_left;
|
|
}
|
|
|
|
return LZMA_STREAM_END;
|
|
}
|
|
|
|
|
|
/// Makes local copies of range decoder and *in_pos variables. Doing this
|
|
/// improves speed significantly. The range decoder macros expect also
|
|
/// variables 'in' and 'in_size' to be defined.
|
|
#define rc_to_local(range_decoder, in_pos, fast_mode_in_required) \
|
|
lzma_range_decoder rc = range_decoder; \
|
|
const uint8_t *rc_in_ptr = in + (in_pos); \
|
|
const uint8_t *rc_in_end = in + in_size; \
|
|
const uint8_t *rc_in_fast_end \
|
|
= (rc_in_end - rc_in_ptr) <= (fast_mode_in_required) \
|
|
? rc_in_ptr \
|
|
: rc_in_end - (fast_mode_in_required); \
|
|
(void)rc_in_fast_end; /* Silence a warning with HAVE_SMALL. */ \
|
|
uint32_t rc_bound
|
|
|
|
|
|
/// Evaluates to true if there is enough input remaining to use fast mode.
|
|
#define rc_is_fast_allowed() (rc_in_ptr < rc_in_fast_end)
|
|
|
|
|
|
/// Stores the local copes back to the range decoder structure.
|
|
#define rc_from_local(range_decoder, in_pos) \
|
|
do { \
|
|
range_decoder = rc; \
|
|
in_pos = (size_t)(rc_in_ptr - in); \
|
|
} while (0)
|
|
|
|
|
|
/// Resets the range decoder structure.
|
|
#define rc_reset(range_decoder) \
|
|
do { \
|
|
(range_decoder).range = UINT32_MAX; \
|
|
(range_decoder).code = 0; \
|
|
(range_decoder).init_bytes_left = 5; \
|
|
} while (0)
|
|
|
|
|
|
/// When decoding has been properly finished, rc.code is always zero unless
|
|
/// the input stream is corrupt. So checking this can catch some corrupt
|
|
/// files especially if they don't have any other integrity check.
|
|
#define rc_is_finished(range_decoder) \
|
|
((range_decoder).code == 0)
|
|
|
|
|
|
// Read the next input byte if needed.
|
|
#define rc_normalize() \
|
|
do { \
|
|
if (rc.range < RC_TOP_VALUE) { \
|
|
rc.range <<= RC_SHIFT_BITS; \
|
|
rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \
|
|
} \
|
|
} while (0)
|
|
|
|
|
|
/// If more input is needed but there is
|
|
/// no more input available, "goto out" is used to jump out of the main
|
|
/// decoder loop. The "_safe" macros are used in the Resumable decoder
|
|
/// mode in order to save the sequence to continue decoding from that
|
|
/// point later.
|
|
#define rc_normalize_safe(seq) \
|
|
do { \
|
|
if (rc.range < RC_TOP_VALUE) { \
|
|
if (rc_in_ptr == rc_in_end) { \
|
|
coder->sequence = seq; \
|
|
goto out; \
|
|
} \
|
|
rc.range <<= RC_SHIFT_BITS; \
|
|
rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \
|
|
} \
|
|
} while (0)
|
|
|
|
|
|
/// Start decoding a bit. This must be used together with rc_update_0()
|
|
/// and rc_update_1():
|
|
///
|
|
/// rc_if_0(prob) {
|
|
/// rc_update_0(prob);
|
|
/// // Do something
|
|
/// } else {
|
|
/// rc_update_1(prob);
|
|
/// // Do something else
|
|
/// }
|
|
///
|
|
#define rc_if_0(prob) \
|
|
rc_normalize(); \
|
|
rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \
|
|
if (rc.code < rc_bound)
|
|
|
|
|
|
#define rc_if_0_safe(prob, seq) \
|
|
rc_normalize_safe(seq); \
|
|
rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \
|
|
if (rc.code < rc_bound)
|
|
|
|
|
|
/// Update the range decoder state and the used probability variable to
|
|
/// match a decoded bit of 0.
|
|
///
|
|
/// The x86-64 assembly uses the commented method but it seems that,
|
|
/// at least on x86-64, the first version is slightly faster as C code.
|
|
#define rc_update_0(prob) \
|
|
do { \
|
|
rc.range = rc_bound; \
|
|
prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \
|
|
/* prob -= ((prob) + RC_BIT_MODEL_OFFSET) >> RC_MOVE_BITS; */ \
|
|
} while (0)
|
|
|
|
|
|
/// Update the range decoder state and the used probability variable to
|
|
/// match a decoded bit of 1.
|
|
#define rc_update_1(prob) \
|
|
do { \
|
|
rc.range -= rc_bound; \
|
|
rc.code -= rc_bound; \
|
|
prob -= (prob) >> RC_MOVE_BITS; \
|
|
} while (0)
|
|
|
|
|
|
/// Decodes one bit and runs action0 or action1 depending on the decoded bit.
|
|
/// This macro is used as the last step in bittree reverse decoders since
|
|
/// those don't use "symbol" for anything else than indexing the probability
|
|
/// arrays.
|
|
#define rc_bit_last(prob, action0, action1) \
|
|
do { \
|
|
rc_if_0(prob) { \
|
|
rc_update_0(prob); \
|
|
action0; \
|
|
} else { \
|
|
rc_update_1(prob); \
|
|
action1; \
|
|
} \
|
|
} while (0)
|
|
|
|
|
|
#define rc_bit_last_safe(prob, action0, action1, seq) \
|
|
do { \
|
|
rc_if_0_safe(prob, seq) { \
|
|
rc_update_0(prob); \
|
|
action0; \
|
|
} else { \
|
|
rc_update_1(prob); \
|
|
action1; \
|
|
} \
|
|
} while (0)
|
|
|
|
|
|
/// Decodes one bit, updates "symbol", and runs action0 or action1 depending
|
|
/// on the decoded bit.
|
|
#define rc_bit(prob, action0, action1) \
|
|
rc_bit_last(prob, \
|
|
symbol <<= 1; action0, \
|
|
symbol = (symbol << 1) + 1; action1);
|
|
|
|
|
|
#define rc_bit_safe(prob, action0, action1, seq) \
|
|
rc_bit_last_safe(prob, \
|
|
symbol <<= 1; action0, \
|
|
symbol = (symbol << 1) + 1; action1, \
|
|
seq);
|
|
|
|
// Unroll fixed-sized bittree decoding.
|
|
//
|
|
// A compile-time constant in final_add can be used to get rid of the high bit
|
|
// from symbol that is used for the array indexing (1U << bittree_bits).
|
|
// final_add may also be used to add offset to the result (LZMA length
|
|
// decoder does that).
|
|
//
|
|
// The reason to have final_add here is that in the asm code the addition
|
|
// can be done for free: in x86-64 there is SBB instruction with -1 as
|
|
// the immediate value, and final_add is combined with that value.
|
|
#define rc_bittree_bit(prob) \
|
|
rc_bit(prob, , )
|
|
|
|
#define rc_bittree3(probs, final_add) \
|
|
do { \
|
|
symbol = 1; \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
symbol += (uint32_t)(final_add); \
|
|
} while (0)
|
|
|
|
#define rc_bittree6(probs, final_add) \
|
|
do { \
|
|
symbol = 1; \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
symbol += (uint32_t)(final_add); \
|
|
} while (0)
|
|
|
|
#define rc_bittree8(probs, final_add) \
|
|
do { \
|
|
symbol = 1; \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
rc_bittree_bit(probs[symbol]); \
|
|
symbol += (uint32_t)(final_add); \
|
|
} while (0)
|
|
|
|
|
|
// Fixed-sized reverse bittree
|
|
#define rc_bittree_rev4(probs) \
|
|
do { \
|
|
symbol = 0; \
|
|
rc_bit_last(probs[symbol + 1], , symbol += 1); \
|
|
rc_bit_last(probs[symbol + 2], , symbol += 2); \
|
|
rc_bit_last(probs[symbol + 4], , symbol += 4); \
|
|
rc_bit_last(probs[symbol + 8], , symbol += 8); \
|
|
} while (0)
|
|
|
|
|
|
// Decode one bit from variable-sized reverse bittree. The loop is done
|
|
// in the code that uses this macro. This could be changed if the assembly
|
|
// version benefited from having the loop done in assembly but it didn't
|
|
// seem so in early 2024.
|
|
//
|
|
// Also, if the loop was done here, the loop counter would likely be local
|
|
// to the macro so that it wouldn't modify yet another input variable.
|
|
// If a _safe version of a macro with a loop was done then a modifiable
|
|
// input variable couldn't be avoided though.
|
|
#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \
|
|
rc_bit(probs[symbol], \
|
|
, \
|
|
dest += value_to_add_if_1);
|
|
|
|
|
|
// Matched literal
|
|
#define decode_with_match_bit \
|
|
t_match_byte <<= 1; \
|
|
t_match_bit = t_match_byte & t_offset; \
|
|
t_subcoder_index = t_offset + t_match_bit + symbol; \
|
|
rc_bit(probs[t_subcoder_index], \
|
|
t_offset &= ~t_match_bit, \
|
|
t_offset &= t_match_bit)
|
|
|
|
#define rc_matched_literal(probs_base_var, match_byte) \
|
|
do { \
|
|
uint32_t t_match_byte = (match_byte); \
|
|
uint32_t t_match_bit; \
|
|
uint32_t t_subcoder_index; \
|
|
uint32_t t_offset = 0x100; \
|
|
symbol = 1; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
decode_with_match_bit; \
|
|
} while (0)
|
|
|
|
|
|
/// Decode a bit without using a probability.
|
|
//
|
|
// NOTE: GCC 13 and Clang/LLVM 16 can, at least on x86-64, optimize the bound
|
|
// calculation to use an arithmetic right shift so there's no need to provide
|
|
// the alternative code which, according to C99/C11/C23 6.3.1.3-p3 isn't
|
|
// perfectly portable: rc_bound = (uint32_t)((int32_t)rc.code >> 31);
|
|
#define rc_direct(dest, count_var) \
|
|
do { \
|
|
dest = (dest << 1) + 1; \
|
|
rc_normalize(); \
|
|
rc.range >>= 1; \
|
|
rc.code -= rc.range; \
|
|
rc_bound = UINT32_C(0) - (rc.code >> 31); \
|
|
dest += rc_bound; \
|
|
rc.code += rc.range & rc_bound; \
|
|
} while (--count_var > 0)
|
|
|
|
|
|
|
|
#define rc_direct_safe(dest, count_var, seq) \
|
|
do { \
|
|
rc_normalize_safe(seq); \
|
|
rc.range >>= 1; \
|
|
rc.code -= rc.range; \
|
|
rc_bound = UINT32_C(0) - (rc.code >> 31); \
|
|
rc.code += rc.range & rc_bound; \
|
|
dest = (dest << 1) + (rc_bound + 1); \
|
|
} while (--count_var > 0)
|
|
|
|
|
|
//////////////////
|
|
// Branchless C //
|
|
//////////////////
|
|
|
|
/// Decode a bit using a branchless method. This reduces the number of
|
|
/// mispredicted branches and thus can improve speed.
|
|
#define rc_c_bit(prob, action_bit, action_neg) \
|
|
do { \
|
|
probability *p = &(prob); \
|
|
rc_normalize(); \
|
|
rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * *p; \
|
|
uint32_t rc_mask = rc.code >= rc_bound; /* rc_mask = decoded bit */ \
|
|
action_bit; /* action when rc_mask is 0 or 1 */ \
|
|
/* rc_mask becomes 0 if bit is 0 and 0xFFFFFFFF if bit is 1: */ \
|
|
rc_mask = 0U - rc_mask; \
|
|
rc.range &= rc_mask; /* If bit 0: set rc.range = 0 */ \
|
|
rc_bound ^= rc_mask; \
|
|
rc_bound -= rc_mask; /* If bit 1: rc_bound = 0U - rc_bound */ \
|
|
rc.range += rc_bound; \
|
|
rc_bound &= rc_mask; \
|
|
rc.code += rc_bound; \
|
|
action_neg; /* action when rc_mask is 0 or 0xFFFFFFFF */ \
|
|
rc_mask = ~rc_mask; /* If bit 0: all bits are set in rc_mask */ \
|
|
rc_mask &= RC_BIT_MODEL_OFFSET; \
|
|
*p -= (*p + rc_mask) >> RC_MOVE_BITS; \
|
|
} while (0)
|
|
|
|
|
|
// Testing on x86-64 give an impression that only the normal bittrees and
|
|
// the fixed-sized reverse bittrees are worth the branchless C code.
|
|
// It should be tested on other archs for which there isn't assembly code
|
|
// in this file.
|
|
|
|
// Using addition in "(symbol << 1) + rc_mask" allows use of x86 LEA
|
|
// or RISC-V SH1ADD instructions. Compilers might infer it from
|
|
// "(symbol << 1) | rc_mask" too if they see that mask is 0 or 1 but
|
|
// the use of addition doesn't require such analysis from compilers.
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x01
|
|
#undef rc_bittree_bit
|
|
#define rc_bittree_bit(prob) \
|
|
rc_c_bit(prob, \
|
|
symbol = (symbol << 1) + rc_mask, \
|
|
)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x01
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x02
|
|
#undef rc_bittree_rev4
|
|
#define rc_bittree_rev4(probs) \
|
|
do { \
|
|
symbol = 0; \
|
|
rc_c_bit(probs[symbol + 1], symbol += rc_mask, ); \
|
|
rc_c_bit(probs[symbol + 2], symbol += rc_mask << 1, ); \
|
|
rc_c_bit(probs[symbol + 4], symbol += rc_mask << 2, ); \
|
|
rc_c_bit(probs[symbol + 8], symbol += rc_mask << 3, ); \
|
|
} while (0)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x02
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x04
|
|
#undef rc_bit_add_if_1
|
|
#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \
|
|
rc_c_bit(probs[symbol], \
|
|
symbol = (symbol << 1) + rc_mask, \
|
|
dest += (value_to_add_if_1) & rc_mask)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x04
|
|
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x08
|
|
#undef decode_with_match_bit
|
|
#define decode_with_match_bit \
|
|
t_match_byte <<= 1; \
|
|
t_match_bit = t_match_byte & t_offset; \
|
|
t_subcoder_index = t_offset + t_match_bit + symbol; \
|
|
rc_c_bit(probs[t_subcoder_index], \
|
|
symbol = (symbol << 1) + rc_mask, \
|
|
t_offset &= ~t_match_bit ^ rc_mask)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x08
|
|
|
|
|
|
////////////
|
|
// x86-64 //
|
|
////////////
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x1F0
|
|
|
|
// rc_asm_y and rc_asm_n are used as arguments to macros to control which
|
|
// strings to include or omit.
|
|
#define rc_asm_y(str) str
|
|
#define rc_asm_n(str)
|
|
|
|
// There are a few possible variations for normalization.
|
|
// This is the smallest variant which is also used by LZMA SDK.
|
|
//
|
|
// - This has partial register write (the MOV from (%[in_ptr])).
|
|
//
|
|
// - INC saves one byte in code size over ADD. False dependency on
|
|
// partial flags from INC shouldn't become a problem on any processor
|
|
// because the instructions after normalization don't read the flags
|
|
// until SUB which sets all flags.
|
|
//
|
|
#define rc_asm_normalize \
|
|
"cmp %[top_value], %[range]\n\t" \
|
|
"jae 1f\n\t" \
|
|
"shl %[shift_bits], %[code]\n\t" \
|
|
"mov (%[in_ptr]), %b[code]\n\t" \
|
|
"shl %[shift_bits], %[range]\n\t" \
|
|
"inc %[in_ptr]\n" \
|
|
"1:\n"
|
|
|
|
// rc_asm_calc(prob) is roughly equivalent to the C version of rc_if_0(prob)...
|
|
//
|
|
// rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob);
|
|
// if (rc.code < rc_bound)
|
|
//
|
|
// ...but the bound is stored in "range":
|
|
//
|
|
// t0 = range;
|
|
// range = (range >> RC_BIT_MODEL_TOTAL_BITS) * (prob);
|
|
// t0 -= range;
|
|
// t1 = code;
|
|
// code -= range;
|
|
//
|
|
// The carry flag (CF) from the last subtraction holds the negation of
|
|
// the decoded bit (if CF==0 then the decoded bit is 1).
|
|
// The values in t0 and t1 are needed for rc_update_0(prob) and
|
|
// rc_update_1(prob). If the bit is 0, rc_update_0(prob)...
|
|
//
|
|
// rc.range = rc_bound;
|
|
//
|
|
// ...has already been done but the "code -= range" has to be reverted using
|
|
// the old value stored in t1. (Also, prob needs to be updated.)
|
|
//
|
|
// If the bit is 1, rc_update_1(prob)...
|
|
//
|
|
// rc.range -= rc_bound;
|
|
// rc.code -= rc_bound;
|
|
//
|
|
// ...is already done for "code" but the value for "range" needs to be taken
|
|
// from t0. (Also, prob needs to be updated here as well.)
|
|
//
|
|
// The assignments from t0 and t1 can be done in a branchless manner with CMOV
|
|
// after the instructions from this macro. The CF from SUB tells which moves
|
|
// are needed.
|
|
#define rc_asm_calc(prob) \
|
|
"mov %[range], %[t0]\n\t" \
|
|
"shr %[bit_model_total_bits], %[range]\n\t" \
|
|
"imul %[" prob "], %[range]\n\t" \
|
|
"sub %[range], %[t0]\n\t" \
|
|
"mov %[code], %[t1]\n\t" \
|
|
"sub %[range], %[code]\n\t"
|
|
|
|
// Also, prob needs to be updated: The update math depends on the decoded bit.
|
|
// It can be expressed in a few slightly different ways but this is fairly
|
|
// convenient here:
|
|
//
|
|
// prob -= (prob + (bit ? 0 : RC_BIT_MODEL_OFFSET)) >> RC_MOVE_BITS;
|
|
//
|
|
// To do it in branchless way when the negation of the decoded bit is in CF,
|
|
// both "prob" and "prob + RC_BIT_MODEL_OFFSET" are needed. Then the desired
|
|
// value can be picked with CMOV. The addition can be done using LEA without
|
|
// affecting CF.
|
|
//
|
|
// (This prob update method is a tiny bit different from LZMA SDK 23.01.
|
|
// In the LZMA SDK a single register is reserved solely for a constant to
|
|
// be used with CMOV when updating prob. That is fine since there are enough
|
|
// free registers to do so. The method used here uses one fewer register,
|
|
// which is valuable with inline assembly.)
|
|
//
|
|
// * * *
|
|
//
|
|
// In bittree decoding, each (unrolled) loop iteration decodes one bit
|
|
// and needs one prob variable. To make it faster, the prob variable of
|
|
// the iteration N+1 is loaded during iteration N. There are two possible
|
|
// prob variables to choose from for N+1. Both are loaded from memory and
|
|
// the correct one is chosen with CMOV using the same CF as is used for
|
|
// other things described above.
|
|
//
|
|
// This preloading/prefetching requires an extra register. To avoid
|
|
// useless moves from "preloaded prob register" to "current prob register",
|
|
// the macros swap between the two registers for odd and even iterations.
|
|
//
|
|
// * * *
|
|
//
|
|
// Finally, the decoded bit has to be stored in "symbol". Since the negation
|
|
// of the bit is in CF, this can be done with SBB: symbol -= CF - 1. That is,
|
|
// if the decoded bit is 0 (CF==1) the operation is a no-op "symbol -= 0"
|
|
// and when bit is 1 (CF==0) the operation is "symbol -= 0 - 1" which is
|
|
// the same as "symbol += 1".
|
|
//
|
|
// The instructions for all things are intertwined for a few reasons:
|
|
// - freeing temporary registers for new use
|
|
// - not modifying CF too early
|
|
// - instruction scheduling
|
|
//
|
|
// The first and last iterations can cheat a little. For example,
|
|
// on the first iteration "symbol" is known to start from 1 so it
|
|
// doesn't need to be read; it can even be immediately initialized
|
|
// to 2 to prepare for the second iteration of the loop.
|
|
//
|
|
// * * *
|
|
//
|
|
// a = number of the current prob variable (0 or 1)
|
|
// b = number of the next prob variable (1 or 0)
|
|
// *_only = rc_asm_y or _n to include or exclude code marked with them
|
|
#define rc_asm_bittree(a, b, first_only, middle_only, last_only) \
|
|
first_only( \
|
|
"movzw 2(%[probs_base]), %[prob" #a "]\n\t" \
|
|
"mov $2, %[symbol]\n\t" \
|
|
"movzw 4(%[probs_base]), %[prob" #b "]\n\t" \
|
|
) \
|
|
middle_only( \
|
|
/* Note the scaling of 4 instead of 2: */ \
|
|
"movzw (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \
|
|
) \
|
|
last_only( \
|
|
"add %[symbol], %[symbol]\n\t" \
|
|
) \
|
|
\
|
|
rc_asm_normalize \
|
|
rc_asm_calc("prob" #a) \
|
|
\
|
|
"cmovae %[t0], %[range]\n\t" \
|
|
\
|
|
first_only( \
|
|
"movzw 6(%[probs_base]), %[t0]\n\t" \
|
|
"cmovae %[t0], %[prob" #b "]\n\t" \
|
|
) \
|
|
middle_only( \
|
|
"movzw 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \
|
|
"lea (%q[symbol], %q[symbol]), %[symbol]\n\t" \
|
|
"cmovae %[t0], %[prob" #b "]\n\t" \
|
|
) \
|
|
last_only( \
|
|
/*"lea (%q[symbol], %q[symbol]), %[symbol]\n\t"*/ \
|
|
) \
|
|
\
|
|
"lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \
|
|
"cmovb %[t1], %[code]\n\t" \
|
|
"mov %[symbol], %[t1]\n\t" \
|
|
"cmovae %[prob" #a "], %[t0]\n\t" \
|
|
\
|
|
first_only( \
|
|
"sbb $-1, %[symbol]\n\t" \
|
|
) \
|
|
middle_only( \
|
|
"sbb $-1, %[symbol]\n\t" \
|
|
) \
|
|
last_only( \
|
|
"sbb %[last_sbb], %[symbol]\n\t" \
|
|
) \
|
|
\
|
|
"shr %[move_bits], %[t0]\n\t" \
|
|
"sub %[t0], %[prob" #a "]\n\t" \
|
|
/* Scaling of 1 instead of 2 because symbol <<= 1. */ \
|
|
"mov %w[prob" #a "], (%[probs_base], %q[t1], 1)\n\t"
|
|
|
|
// NOTE: The order of variables in __asm__ can affect speed and code size.
|
|
#define rc_asm_bittree_n(probs_base_var, final_add, asm_str) \
|
|
do { \
|
|
uint32_t t0; \
|
|
uint32_t t1; \
|
|
uint32_t t_prob0; \
|
|
uint32_t t_prob1; \
|
|
\
|
|
__asm__( \
|
|
asm_str \
|
|
: \
|
|
[range] "+&r"(rc.range), \
|
|
[code] "+&r"(rc.code), \
|
|
[t0] "=&r"(t0), \
|
|
[t1] "=&r"(t1), \
|
|
[prob0] "=&r"(t_prob0), \
|
|
[prob1] "=&r"(t_prob1), \
|
|
[symbol] "=&r"(symbol), \
|
|
[in_ptr] "+&r"(rc_in_ptr) \
|
|
: \
|
|
[probs_base] "r"(probs_base_var), \
|
|
[last_sbb] "n"(-1 - (final_add)), \
|
|
[top_value] "n"(RC_TOP_VALUE), \
|
|
[shift_bits] "n"(RC_SHIFT_BITS), \
|
|
[bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \
|
|
[bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \
|
|
[move_bits] "n"(RC_MOVE_BITS) \
|
|
: \
|
|
"cc", "memory"); \
|
|
} while (0)
|
|
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x010
|
|
#undef rc_bittree3
|
|
#define rc_bittree3(probs_base_var, final_add) \
|
|
rc_asm_bittree_n(probs_base_var, final_add, \
|
|
rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_n, rc_asm_y) \
|
|
)
|
|
|
|
#undef rc_bittree6
|
|
#define rc_bittree6(probs_base_var, final_add) \
|
|
rc_asm_bittree_n(probs_base_var, final_add, \
|
|
rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \
|
|
)
|
|
|
|
#undef rc_bittree8
|
|
#define rc_bittree8(probs_base_var, final_add) \
|
|
rc_asm_bittree_n(probs_base_var, final_add, \
|
|
rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \
|
|
)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x010
|
|
|
|
|
|
// Fixed-sized reverse bittree
|
|
//
|
|
// This uses the indexing that constructs the final value in symbol directly.
|
|
// add = 1, 2, 4, 8
|
|
// dcur = -, 4, 8, 16
|
|
// dnext0 = 4, 8, 16, -
|
|
// dnext0 = 6, 12, 24, -
|
|
#define rc_asm_bittree_rev(a, b, add, dcur, dnext0, dnext1, \
|
|
first_only, middle_only, last_only) \
|
|
first_only( \
|
|
"movzw 2(%[probs_base]), %[prob" #a "]\n\t" \
|
|
"xor %[symbol], %[symbol]\n\t" \
|
|
"movzw 4(%[probs_base]), %[prob" #b "]\n\t" \
|
|
) \
|
|
middle_only( \
|
|
"movzw " #dnext0 "(%[probs_base], %q[symbol], 2), " \
|
|
"%[prob" #b "]\n\t" \
|
|
) \
|
|
\
|
|
rc_asm_normalize \
|
|
rc_asm_calc("prob" #a) \
|
|
\
|
|
"cmovae %[t0], %[range]\n\t" \
|
|
\
|
|
first_only( \
|
|
"movzw 6(%[probs_base]), %[t0]\n\t" \
|
|
"cmovae %[t0], %[prob" #b "]\n\t" \
|
|
) \
|
|
middle_only( \
|
|
"movzw " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \
|
|
"cmovae %[t0], %[prob" #b "]\n\t" \
|
|
) \
|
|
\
|
|
"lea " #add "(%q[symbol]), %[t0]\n\t" \
|
|
"cmovb %[t1], %[code]\n\t" \
|
|
middle_only( \
|
|
"mov %[symbol], %[t1]\n\t" \
|
|
) \
|
|
last_only( \
|
|
"mov %[symbol], %[t1]\n\t" \
|
|
) \
|
|
"cmovae %[t0], %[symbol]\n\t" \
|
|
"lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \
|
|
"cmovae %[prob" #a "], %[t0]\n\t" \
|
|
\
|
|
"shr %[move_bits], %[t0]\n\t" \
|
|
"sub %[t0], %[prob" #a "]\n\t" \
|
|
first_only( \
|
|
"mov %w[prob" #a "], 2(%[probs_base])\n\t" \
|
|
) \
|
|
middle_only( \
|
|
"mov %w[prob" #a "], " \
|
|
#dcur "(%[probs_base], %q[t1], 2)\n\t" \
|
|
) \
|
|
last_only( \
|
|
"mov %w[prob" #a "], " \
|
|
#dcur "(%[probs_base], %q[t1], 2)\n\t" \
|
|
)
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x020
|
|
#undef rc_bittree_rev4
|
|
#define rc_bittree_rev4(probs_base_var) \
|
|
rc_asm_bittree_n(probs_base_var, 4, \
|
|
rc_asm_bittree_rev(0, 1, 1, -, 4, 6, rc_asm_y, rc_asm_n, rc_asm_n) \
|
|
rc_asm_bittree_rev(1, 0, 2, 4, 8, 12, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree_rev(0, 1, 4, 8, 16, 24, rc_asm_n, rc_asm_y, rc_asm_n) \
|
|
rc_asm_bittree_rev(1, 0, 8, 16, -, -, rc_asm_n, rc_asm_n, rc_asm_y) \
|
|
)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x020
|
|
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x040
|
|
#undef rc_bit_add_if_1
|
|
#define rc_bit_add_if_1(probs_base_var, dest_var, value_to_add_if_1) \
|
|
do { \
|
|
uint32_t t0; \
|
|
uint32_t t1; \
|
|
uint32_t t2 = (value_to_add_if_1); \
|
|
uint32_t t_prob; \
|
|
uint32_t t_index; \
|
|
\
|
|
__asm__( \
|
|
"movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \
|
|
"mov %[symbol], %[index]\n\t" \
|
|
\
|
|
"add %[dest], %[t2]\n\t" \
|
|
"add %[symbol], %[symbol]\n\t" \
|
|
\
|
|
rc_asm_normalize \
|
|
rc_asm_calc("prob") \
|
|
\
|
|
"cmovae %[t0], %[range]\n\t" \
|
|
"lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \
|
|
"cmovb %[t1], %[code]\n\t" \
|
|
"cmovae %[prob], %[t0]\n\t" \
|
|
\
|
|
"cmovae %[t2], %[dest]\n\t" \
|
|
"sbb $-1, %[symbol]\n\t" \
|
|
\
|
|
"sar %[move_bits], %[t0]\n\t" \
|
|
"sub %[t0], %[prob]\n\t" \
|
|
"mov %w[prob], (%[probs_base], %q[index], 2)" \
|
|
: \
|
|
[range] "+&r"(rc.range), \
|
|
[code] "+&r"(rc.code), \
|
|
[t0] "=&r"(t0), \
|
|
[t1] "=&r"(t1), \
|
|
[prob] "=&r"(t_prob), \
|
|
[index] "=&r"(t_index), \
|
|
[symbol] "+&r"(symbol), \
|
|
[t2] "+&r"(t2), \
|
|
[dest] "+&r"(dest_var), \
|
|
[in_ptr] "+&r"(rc_in_ptr) \
|
|
: \
|
|
[probs_base] "r"(probs_base_var), \
|
|
[top_value] "n"(RC_TOP_VALUE), \
|
|
[shift_bits] "n"(RC_SHIFT_BITS), \
|
|
[bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \
|
|
[bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \
|
|
[move_bits] "n"(RC_MOVE_BITS) \
|
|
: \
|
|
"cc", "memory"); \
|
|
} while (0)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x040
|
|
|
|
|
|
// Literal decoding uses a normal 8-bit bittree but literal with match byte
|
|
// is more complex in picking the probability variable from the correct
|
|
// subtree. This doesn't use preloading/prefetching of the next prob because
|
|
// there are four choices instead of two.
|
|
//
|
|
// FIXME? The first iteration starts with symbol = 1 so it could be optimized
|
|
// by a tiny amount.
|
|
#define rc_asm_matched_literal(nonlast_only) \
|
|
"add %[offset], %[symbol]\n\t" \
|
|
"and %[offset], %[match_bit]\n\t" \
|
|
"add %[match_bit], %[symbol]\n\t" \
|
|
\
|
|
"movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \
|
|
\
|
|
"add %[symbol], %[symbol]\n\t" \
|
|
\
|
|
nonlast_only( \
|
|
"xor %[match_bit], %[offset]\n\t" \
|
|
"add %[match_byte], %[match_byte]\n\t" \
|
|
) \
|
|
\
|
|
rc_asm_normalize \
|
|
rc_asm_calc("prob") \
|
|
\
|
|
"cmovae %[t0], %[range]\n\t" \
|
|
"lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \
|
|
"cmovb %[t1], %[code]\n\t" \
|
|
"mov %[symbol], %[t1]\n\t" \
|
|
"cmovae %[prob], %[t0]\n\t" \
|
|
\
|
|
nonlast_only( \
|
|
"cmovae %[match_bit], %[offset]\n\t" \
|
|
"mov %[match_byte], %[match_bit]\n\t" \
|
|
) \
|
|
\
|
|
"sbb $-1, %[symbol]\n\t" \
|
|
\
|
|
"shr %[move_bits], %[t0]\n\t" \
|
|
/* Undo symbol += match_bit + offset: */ \
|
|
"and $0x1FF, %[symbol]\n\t" \
|
|
"sub %[t0], %[prob]\n\t" \
|
|
\
|
|
/* Scaling of 1 instead of 2 because symbol <<= 1. */ \
|
|
"mov %w[prob], (%[probs_base], %q[t1], 1)\n\t"
|
|
|
|
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x080
|
|
#undef rc_matched_literal
|
|
#define rc_matched_literal(probs_base_var, match_byte_value) \
|
|
do { \
|
|
uint32_t t0; \
|
|
uint32_t t1; \
|
|
uint32_t t_prob; \
|
|
uint32_t t_match_byte = (uint32_t)(match_byte_value) << 1; \
|
|
uint32_t t_match_bit = t_match_byte; \
|
|
uint32_t t_offset = 0x100; \
|
|
symbol = 1; \
|
|
\
|
|
__asm__( \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_y) \
|
|
rc_asm_matched_literal(rc_asm_n) \
|
|
: \
|
|
[range] "+&r"(rc.range), \
|
|
[code] "+&r"(rc.code), \
|
|
[t0] "=&r"(t0), \
|
|
[t1] "=&r"(t1), \
|
|
[prob] "=&r"(t_prob), \
|
|
[match_bit] "+&r"(t_match_bit), \
|
|
[symbol] "+&r"(symbol), \
|
|
[match_byte] "+&r"(t_match_byte), \
|
|
[offset] "+&r"(t_offset), \
|
|
[in_ptr] "+&r"(rc_in_ptr) \
|
|
: \
|
|
[probs_base] "r"(probs_base_var), \
|
|
[top_value] "n"(RC_TOP_VALUE), \
|
|
[shift_bits] "n"(RC_SHIFT_BITS), \
|
|
[bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \
|
|
[bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \
|
|
[move_bits] "n"(RC_MOVE_BITS) \
|
|
: \
|
|
"cc", "memory"); \
|
|
} while (0)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x080
|
|
|
|
|
|
// Doing the loop in asm instead of C seems to help a little.
|
|
#if LZMA_RANGE_DECODER_CONFIG & 0x100
|
|
#undef rc_direct
|
|
#define rc_direct(dest_var, count_var) \
|
|
do { \
|
|
uint32_t t0; \
|
|
uint32_t t1; \
|
|
\
|
|
__asm__( \
|
|
"2:\n\t" \
|
|
"add %[dest], %[dest]\n\t" \
|
|
"lea 1(%q[dest]), %[t1]\n\t" \
|
|
\
|
|
rc_asm_normalize \
|
|
\
|
|
"shr $1, %[range]\n\t" \
|
|
"mov %[code], %[t0]\n\t" \
|
|
"sub %[range], %[code]\n\t" \
|
|
"cmovns %[t1], %[dest]\n\t" \
|
|
"cmovs %[t0], %[code]\n\t" \
|
|
"dec %[count]\n\t" \
|
|
"jnz 2b\n\t" \
|
|
: \
|
|
[range] "+&r"(rc.range), \
|
|
[code] "+&r"(rc.code), \
|
|
[t0] "=&r"(t0), \
|
|
[t1] "=&r"(t1), \
|
|
[dest] "+&r"(dest_var), \
|
|
[count] "+&r"(count_var), \
|
|
[in_ptr] "+&r"(rc_in_ptr) \
|
|
: \
|
|
[top_value] "n"(RC_TOP_VALUE), \
|
|
[shift_bits] "n"(RC_SHIFT_BITS) \
|
|
: \
|
|
"cc", "memory"); \
|
|
} while (0)
|
|
#endif // LZMA_RANGE_DECODER_CONFIG & 0x100
|
|
|
|
#endif // x86_64
|
|
|
|
#endif
|