From 17f4e83c0f0fc3bacf4b2bbacb01f987bb5aff5f Mon Sep 17 00:00:00 2001
From: Grzegorz Antoniak <ga@anadoxin.org>
Date: Fri, 12 Feb 2021 20:18:31 +0100
Subject: [PATCH] RAR5 reader: fix invalid memory access in some files

RAR5 reader uses several variables to manage the window buffer during
extraction: the buffer itself (`window_buf`), the current size of the
window buffer (`window_size`), and a helper variable (`window_mask`)
that is used to constrain read and write offsets to the window buffer.

Some specially crafted files can force the unpacker to update the
`window_mask` variable to a value that is out of sync with current
buffer size. If the `window_mask` will be bigger than the actual buffer
size, then an invalid access operation can happen (SIGSEGV).

This commit ensures that if the `window_size` and `window_mask` will be
changed, the window buffer will be reallocated to the proper size, so no
invalid memory operation should be possible.

This commit contains a test file from OSSFuzz #30442.

Reference:https://github.com/libarchive/libarchive/commit/17f4e83c0f0fc3bacf4b2bbacb01f987bb5aff5f
Conflict:NA
---
 libarchive/archive_read_support_format_rar5.c | 27 ++++++++++++++-----
 1 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/libarchive/archive_read_support_format_rar5.c b/libarchive/archive_read_support_format_rar5.c
index a91c73f8b..880ba6612 100644
--- a/libarchive/archive_read_support_format_rar5.c
+++ b/libarchive/archive_read_support_format_rar5.c
@@ -1772,14 +1772,29 @@ static int process_head_file(struct archive_read* a, struct rar5* rar,
 		}
 	}
 
-	/* If we're currently switching volumes, ignore the new definition of
-	 * window_size. */
-	if(rar->cstate.switch_multivolume == 0) {
-		/* Values up to 64M should fit into ssize_t on every
-		 * architecture. */
-		rar->cstate.window_size = (ssize_t) window_size;
+	if(rar->cstate.window_size < (ssize_t) window_size &&
+	    rar->cstate.window_buf)
+	{
+		/* If window_buf has been allocated before, reallocate it, so
+		 * that its size will match new window_size. */
+
+		uint8_t* new_window_buf =
+			realloc(rar->cstate.window_buf, window_size);
+
+		if(!new_window_buf) {
+			archive_set_error(&a->archive, ARCHIVE_ERRNO_PROGRAMMER,
+				"Not enough memory when trying to realloc the window "
+				"buffer.");
+			return ARCHIVE_FATAL;
+		}
+
+		rar->cstate.window_buf = new_window_buf;
 	}
 
+	/* Values up to 64M should fit into ssize_t on every
+	 * architecture. */
+	rar->cstate.window_size = (ssize_t) window_size;
+
 	if(rar->file.solid > 0 && rar->file.solid_window_size == 0) {
 		/* Solid files have to have the same window_size across
 		   whole archive. Remember the window_size parameter
