liblzma: Index decoder: Reject an obviously-bad Number of Records better

File info decoder knows Backward Size, so pass it to the Index decoder.
Now the Index decoder, when called from the file info decoder, won't
allocate a large amount of memory when Backward Size is tiny and the
Number of Records field contains a huge invalid value.

If a reasonable memory usage limit was set, the old code returned
LZMA_MEMLIMIT_ERROR if the value in Number of Records was too huge.
Now it results in LZMA_DATA_ERROR which is better.

src/liblzma/common/file_info.c

@@ -438,7 +438,8 @@ file_info_decode(void *coder_ptr, const lzma_allocator *allocator,
 		return_if_error(lzma_index_decoder_init(
 				&coder->index_decoder, allocator,
 				&coder->this_index,
-				coder->memlimit - memused));
+				coder->memlimit - memused,
+				coder->footer_flags.backward_size));
 
 		coder->index_remaining = coder->footer_flags.backward_size;
 		coder->sequence = SEQ_INDEX_DECODE;
@@ -719,8 +720,8 @@ file_info_decoder_memconfig(void *coder_ptr, uint64_t *memusage,
 
 	// combined_index_memusage + this_index_memusage shouldn't overflow
 	// because combined_index_memusage is limited by how much can be
-	// successfully allocated, and this_index_memusage is limited by
+	// successfully allocated, and this_index_memusage is limited by the
-	// INDEX_RECORDS_MAX in index_decoder.c. Check for overflow anyway.
+	// input_size_max check in index_decoder.c. Check for overflow anyway.
 	if (UINT64_MAX - combined_index_memusage < this_index_memusage)
 		return LZMA_PROG_ERROR;
 

src/liblzma/common/index_decoder.c

@@ -39,6 +39,12 @@ typedef struct {
 	/// successful decoding.
 	lzma_index **index_ptr;
 
+	/// If Number of Records contains a value higher than this,
+	/// the Index is immediately considered to be corrupt.
+	/// This prevents us from allocating a large amount of memory
+	/// when we know that the Index isn't big.
+	lzma_vli number_of_records_max;
+
 	/// Number of Records left to decode.
 	lzma_vli count;
 
@@ -95,11 +101,8 @@ index_decode(void *coder_ptr, const lzma_allocator *allocator,
 		if (ret != LZMA_STREAM_END)
 			goto out;
 
-		// If the Number of Records field has so large value that the
+		// Reject Number of Records values that are clearly invalid.
-		// Index would have to be larger than LZMA_BACKWARD_SIZE_MAX,
+		if (coder->count > coder->number_of_records_max)
-		// don't even try to decode the Index because clearly it's
-		// not a valid .xz file.
-		if (coder->count > INDEX_RECORDS_MAX)
 			return LZMA_DATA_ERROR;
 
 		coder->pos = 0;
@@ -266,7 +269,7 @@ index_decoder_memconfig(void *coder_ptr, uint64_t *memusage,
 
 static lzma_ret
 index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator,
-		lzma_index **i, uint64_t memlimit)
+		lzma_index **i, uint64_t memlimit, uint64_t input_size_max)
 {
 	// Remember the pointer given by the application. We will set it
 	// to point to the decoded Index only if decoding is successful.
@@ -280,6 +283,15 @@ index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator,
 	if (coder->index == NULL)
 		return LZMA_MEM_ERROR;
 
+	// Approximate how many Records the Index may contain at most when
+	// we know that the Index won't be larger than input_size_max bytes.
+	// Each Record must consume at least two bytes. The CRC32 etc.
+	// consume a few bytes too, but we don't need an exact value.
+	//
+	// file_info.c knows Backward Size and passes it to us. When we have
+	// no information, input_size_max equals LZMA_BACKWARD_SIZE_MAX.
+	coder->number_of_records_max = input_size_max / 2;
+
 	// Initialize the rest.
 	coder->sequence = SEQ_INDICATOR;
 	coder->memlimit = my_max(1, memlimit);
@@ -293,7 +305,7 @@ index_decoder_reset(lzma_index_coder *coder, const lzma_allocator *allocator,
 
 extern lzma_ret
 lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
-		lzma_index **i, uint64_t memlimit)
+		lzma_index **i, uint64_t memlimit, uint64_t input_size_max)
 {
 	lzma_next_coder_init(&lzma_index_decoder_init, next, allocator);
 
@@ -315,7 +327,8 @@ lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator,
 		lzma_index_end(coder->index, allocator);
 	}
 
-	return index_decoder_reset(coder, allocator, i, memlimit);
+	return index_decoder_reset(
+			coder, allocator, i, memlimit, input_size_max);
 }
 
 
@@ -328,7 +341,8 @@ lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit)
 	if (i != NULL)
 		*i = NULL;
 
-	lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit);
+	lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit,
+			LZMA_BACKWARD_SIZE_MAX);
 
 	strm->internal->supported_actions[LZMA_RUN] = true;
 	strm->internal->supported_actions[LZMA_FINISH] = true;
@@ -354,7 +368,8 @@ lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit,
 
 	// Initialize the decoder.
 	lzma_index_coder coder;
-	return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit));
+	return_if_error(index_decoder_reset(&coder, allocator, i, *memlimit,
+			my_min(in_size, LZMA_BACKWARD_SIZE_MAX)));
 
 	// Store the input start position so that we can restore it in case
 	// of an error.

src/liblzma/common/index_decoder.h

@@ -18,7 +18,7 @@
 
 extern lzma_ret lzma_index_decoder_init(lzma_next_coder *next,
 		const lzma_allocator *allocator,
-		lzma_index **i, uint64_t memlimit);
+		lzma_index **i, uint64_t memlimit, uint64_t input_size_max);
 
 
 #endif