VLI encoder and decoder cleanups. Made encoder return

LZMA_PROG_ERROR in single-call mode if there's no output
space.

src/liblzma/common/vli_decoder.c

@@ -53,24 +53,27 @@ lzma_vli_decode(lzma_vli *restrict vli, size_t *restrict vli_pos,
 	}
 
 	do {
-		// Read the next byte.
-		*vli |= (lzma_vli)(in[*in_pos] & 0x7F) << (*vli_pos * 7);
+		// Read the next byte. Use a temporary variable so that we
+		// can update *in_pos immediatelly.
+		const uint8_t byte = in[*in_pos];
+		++*in_pos;
+
+		// Add the newly read byte to *vli.
+		*vli += (lzma_vli)(byte & 0x7F) << (*vli_pos * 7);
 		++*vli_pos;
 
 		// Check if this is the last byte of a multibyte integer.
-		if (!(in[*in_pos] & 0x80)) {
+		if ((byte & 0x80) == 0) {
 			// We don't allow using variable-length integers as
 			// padding i.e. the encoding must use the most the
 			// compact form.
-			if (in[(*in_pos)++] == 0x00 && *vli_pos > 1)
+			if (byte == 0x00 && *vli_pos > 1)
 				return LZMA_DATA_ERROR;
 
 			return vli_pos == &vli_pos_internal
 					? LZMA_OK : LZMA_STREAM_END;
 		}
 
-		++*in_pos;
-
 		// There is at least one more byte coming. If we have already
 		// read maximum number of bytes, the integer is considered
 		// corrupt.

src/liblzma/common/vli_encoder.c

@@ -27,30 +27,47 @@ lzma_vli_encode(lzma_vli vli, size_t *restrict vli_pos,
 {
 	// If we haven't been given vli_pos, work in single-call mode.
 	size_t vli_pos_internal = 0;
-	if (vli_pos == NULL)
+	if (vli_pos == NULL) {
 		vli_pos = &vli_pos_internal;
 
+		// In single-call mode, we expect that the caller has
+		// reserved enough output space.
+		if (*out_pos >= out_size)
+			return LZMA_PROG_ERROR;
+	} else {
+		// This never happens when we are called by liblzma, but
+		// may happen if called directly from an application.
+		if (*out_pos >= out_size)
+			return LZMA_BUF_ERROR;
+	}
+
 	// Validate the arguments.
 	if (*vli_pos >= LZMA_VLI_BYTES_MAX || vli > LZMA_VLI_MAX)
 		return LZMA_PROG_ERROR;
 
-	if (*out_pos >= out_size)
-		return LZMA_BUF_ERROR;
+	// Shift vli so that the next bits to encode are the lowest. In
+	// single-call mode this never changes vli since *vli_pos is zero.
+	vli >>= *vli_pos * 7;
 
 	// Write the non-last bytes in a loop.
-	while ((vli >> (*vli_pos * 7)) >= 0x80) {
-		out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7)) | 0x80;
-
+	while (vli >= 0x80) {
+		// We don't need *vli_pos during this function call anymore,
+		// but update it here so that it is ready if we need to
+		// return before the whole integer has been decoded.
 		++*vli_pos;
 		assert(*vli_pos < LZMA_VLI_BYTES_MAX);
 
+		// Write the next byte.
+		out[*out_pos] = (uint8_t)(vli) | 0x80;
+		vli >>= 7;
+
 		if (++*out_pos == out_size)
 			return vli_pos == &vli_pos_internal
 					? LZMA_PROG_ERROR : LZMA_OK;
 	}
 
 	// Write the last byte.
-	out[*out_pos] = (uint8_t)(vli >> (*vli_pos * 7));
+	out[*out_pos] = (uint8_t)(vli);
 	++*out_pos;
 	++*vli_pos;