Tests: Make some memory allocations fail in fuzzers

tests/ossfuzz/fuzz_common.h

@@ -13,6 +13,7 @@
 #include <inttypes.h>
 #include <stdlib.h>
 #include <stdio.h>
+#include <stdatomic.h>
 #include "lzma.h"
 
 // Some header values can make liblzma allocate a lot of RAM
@@ -24,6 +25,42 @@
 #define IN_CHUNK_SIZE 2047
 
 
+static atomic_ulong alloc_fail;
+
+
+static void *
+my_alloc(void *opaque, size_t nmemb, size_t size)
+{
+	(void)opaque;
+	(void)nmemb;
+
+	if (--alloc_fail == 0)
+		return NULL;
+
+	return malloc(size);
+}
+
+
+static void
+prepare_stream(lzma_stream *stream, const uint8_t *inbuf, size_t inbuf_size)
+{
+	stream->next_in = inbuf;
+	stream->avail_in = inbuf_size;
+
+	// Unless the input is very tiny, make one allocation in 1024 fail
+	// based on the last two input bytes and inbuf_size.
+	if (inbuf_size >= 10) {
+		alloc_fail = inbuf[inbuf_size - 2]
+				^ ((inbuf[inbuf_size - 1]) << 2)
+				^ (inbuf_size & 0x3FF);
+
+		static const lzma_allocator my_allocator
+				= { &my_alloc, NULL, NULL };
+		stream->allocator = &my_allocator;
+	}
+}
+
+
 static void
 fuzz_code(lzma_stream *stream, const uint8_t *inbuf, size_t inbuf_size)
 {

tests/ossfuzz/fuzz_decode_alone.c

@@ -21,14 +21,16 @@ extern int
 LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 {
 	lzma_stream strm = LZMA_STREAM_INIT;
-	strm.next_in = inbuf;
-	strm.avail_in = inbuf_size;
+	prepare_stream(&strm, inbuf, inbuf_size);
 
 	lzma_ret ret;
 
 	for (int i = 0; i < 3; ++i) {
 		ret = lzma_alone_decoder(&strm, MEM_LIMIT);
 
+		if (ret == LZMA_MEM_ERROR)
+			continue;
+
 		if (ret != LZMA_OK) {
 			// This should never happen unless the system has
 			// no free memory or address space to allow the small

tests/ossfuzz/fuzz_decode_stream.c

@@ -21,8 +21,7 @@ extern int
 LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 {
 	lzma_stream strm = LZMA_STREAM_INIT;
-	strm.next_in = inbuf;
-	strm.avail_in = inbuf_size;
+	prepare_stream(&strm, inbuf, inbuf_size);
 
 	lzma_ret ret;
 
@@ -47,6 +46,9 @@ LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 		ret = lzma_stream_decoder(&strm, MEM_LIMIT, LZMA_IGNORE_CHECK
 				| (i >= 2 ? LZMA_CONCATENATED : 0));
 
+		if (ret == LZMA_MEM_ERROR)
+			continue;
+
 		if (ret != LZMA_OK) {
 			// This should never happen unless the system has
 			// no free memory or address space to allow the small

tests/ossfuzz/fuzz_decode_stream_mt.c

@@ -20,8 +20,7 @@ extern int
 LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 {
 	lzma_stream strm = LZMA_STREAM_INIT;
-	strm.next_in = inbuf;
-	strm.avail_in = inbuf_size;
+	prepare_stream(&strm, inbuf, inbuf_size);
 
 	lzma_mt mt = {
 		.flags = /*LZMA_CONCATENATED |*/ LZMA_IGNORE_CHECK,
@@ -39,6 +38,9 @@ LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 
 		ret = lzma_stream_decoder_mt(&strm, &mt);
 
+		if (ret == LZMA_MEM_ERROR)
+			continue;
+
 		if (ret != LZMA_OK) {
 			// This should never happen unless the system has
 			// no free memory or address space to allow the small

tests/ossfuzz/fuzz_encode_stream.c

@@ -69,11 +69,15 @@ LLVMFuzzerTestOneInput(const uint8_t *inbuf, size_t inbuf_size)
 
 	// initialize empty LZMA stream
 	lzma_stream strm = LZMA_STREAM_INIT;
+	prepare_stream(&strm, inbuf, inbuf_size);
 
 	// Initialize the stream encoder using the above
 	// stream, filter chain and CRC64.
 	lzma_ret ret = lzma_stream_encoder(&strm, filters, LZMA_CHECK_CRC64);
 	if (ret != LZMA_OK) {
+		if (ret == LZMA_MEM_ERROR)
+			return 0;
+
 		fprintf(stderr, "lzma_stream_encoder() failed (%d)\n", ret);
 		abort();
 	}