path: root/CVE-2018-8788.patch

Backport of:

From d1112c279bd1a327e8e4d0b5f371458bf2579659 Mon Sep 17 00:00:00 2001
From: Armin Novak <armin.novak@thincast.com>
Date: Mon, 22 Oct 2018 16:52:21 +0200
Subject: [PATCH] Fixed CVE-2018-8788

Thanks to Eyal Itkin from Check Point Software Technologies.
---
 include/freerdp/codec/nsc.h   |  4 +-
 libfreerdp/codec/nsc.c        | 94 +++++++++++++++++++++++++++++------
 libfreerdp/codec/nsc_encode.c | 62 ++++++++++++++++-------
 libfreerdp/codec/nsc_encode.h |  2 +-
 libfreerdp/codec/nsc_sse2.c   |  4 +-
 5 files changed, 130 insertions(+), 36 deletions(-)

Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/include/freerdp/codec/nsc.h
===================================================================
--- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/include/freerdp/codec/nsc.h
+++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/include/freerdp/codec/nsc.h
@@ -59,8 +59,8 @@ struct _NSC_CONTEXT
 	/* color palette allocated by the application */
 	const BYTE* palette;
 
-	void (*decode)(NSC_CONTEXT* context);
-	void (*encode)(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
+	BOOL (*decode)(NSC_CONTEXT* context);
+	BOOL (*encode)(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
 
 	NSC_CONTEXT_PRIV* priv;
 };
Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc.c
===================================================================
--- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc.c
+++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc.c
@@ -43,7 +43,7 @@
 #define NSC_INIT_SIMD(_nsc_context) do { } while (0)
 #endif
 
-static void nsc_decode(NSC_CONTEXT* context)
+static BOOL nsc_decode(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
@@ -60,11 +60,18 @@ static void nsc_decode(NSC_CONTEXT* cont
 	INT16 g_val;
 	INT16 b_val;
 	BYTE* bmpdata;
+	size_t pos = 0;
+
+	if (!context)
+		return FALSE;
 
 	bmpdata = context->bmpdata;
 	rw = ROUND_UP_TO(context->width, 8);
 	shift = context->nsc_stream.ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */
 
+	if (!bmpdata)
+		return FALSE;
+
 	for (y = 0; y < context->height; y++)
 	{
 		if (context->nsc_stream.ChromaSubSamplingLevel > 0)
@@ -88,6 +95,11 @@ static void nsc_decode(NSC_CONTEXT* cont
 			r_val = y_val + co_val - cg_val;
 			g_val = y_val + cg_val;
 			b_val = y_val - co_val - cg_val;
+
+			if (pos + 4 > context->bmpdata_length)
+				return FALSE;
+
+			pos += 4;
 			*bmpdata++ = MINMAX(b_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(g_val, 0, 0xFF);
 			*bmpdata++ = MINMAX(r_val, 0, 0xFF);
@@ -98,9 +110,11 @@ static void nsc_decode(NSC_CONTEXT* cont
 			aplane++;
 		}
 	}
+
+	return TRUE;
 }
 
-static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 origsz)
+static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 origsz)
 {
 	UINT32 len;
 	UINT32 left;
@@ -113,6 +127,10 @@ static void nsc_rle_decode(BYTE* in, BYT
 
 		if (left == 5)
 		{
+			if (outSize < 1)
+				return FALSE;
+
+			outSize--;
 			*out++ = value;
 			left--;
 		}
@@ -130,6 +148,10 @@ static void nsc_rle_decode(BYTE* in, BYT
 				len = *((UINT32*) in);
 				in += 4;
 			}
+			if (outSize < len)
+				return FALSE;
+
+			outSize -= len;
 			memset(out, value, len);
 			out += len;
 			left -= len;
@@ -141,16 +163,24 @@ static void nsc_rle_decode(BYTE* in, BYT
 		}
 	}
 
-	*((UINT32*)out) = *((UINT32*)in);
+	if ((outSize < 4) || (left < 4))
+		return FALSE;
+
+	memcpy(out, in, 4);
+	return TRUE;
 }
 
-static void nsc_rle_decompress_data(NSC_CONTEXT* context)
+static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context)
 {
 	UINT16 i;
 	BYTE* rle;
 	UINT32 origsize;
 	UINT32 planesize;
 
+
+	if (!context)
+		return FALSE;
+
 	rle = context->nsc_stream.Planes;
 
 	for (i = 0; i < 4; i++)
@@ -159,14 +189,30 @@ static void nsc_rle_decompress_data(NSC_
 		planesize = context->nsc_stream.PlaneByteCount[i];
 
 		if (planesize == 0)
+		{
+			if (context->priv->plane_buf_length < origsize)
+				return FALSE;
+
 			memset(context->priv->plane_buf[i], 0xff, origsize);
+		}
 		else if (planesize < origsize)
-			nsc_rle_decode(rle, context->priv->plane_buf[i], origsize);
+		{
+			if (!nsc_rle_decode(rle, context->priv->plane_buf[i], context->priv->plane_buf_length,
+			                    origsize))
+				return FALSE;
+		}
 		else
+		{
+			if (context->priv->plane_buf_length < origsize)
+				return FALSE;
+
 			memcpy(context->priv->plane_buf[i], rle, origsize);
+		}
 
 		rle += planesize;
 	}
+
+	return TRUE;
 }
 
 static void nsc_stream_initialize(NSC_CONTEXT* context, wStream* s)
@@ -337,12 +383,24 @@ void nsc_process_message(NSC_CONTEXT* co
 	Stream_Free(s, FALSE);
 
 	/* RLE decode */
-	PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
-	nsc_rle_decompress_data(context);
-	PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+	{
+		BOOL rc;
+		PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data);
+		rc = nsc_rle_decompress_data(context);
+		PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data);
+
+		if (!rc)
+                  return;
+	}
 
 	/* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */
-	PROFILER_ENTER(context->priv->prof_nsc_decode);
-	context->decode(context);
-	PROFILER_EXIT(context->priv->prof_nsc_decode);
+	{
+		BOOL rc;
+		PROFILER_ENTER(context->priv->prof_nsc_decode);
+		rc = context->decode(context);
+		PROFILER_EXIT(context->priv->prof_nsc_decode);
+
+		if (!rc)
+                  return;
+	}
 }
Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.c
===================================================================
--- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_encode.c
+++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.c
@@ -67,7 +67,7 @@ static void nsc_context_initialize_encod
 	}
 }
 
-static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
+static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
 {
 	UINT16 x;
 	UINT16 y;
@@ -85,10 +85,20 @@ static void nsc_encode_argb_to_aycocg(NS
 	UINT32 tempWidth;
 	UINT32 tempHeight;
 
+	if (!context || bmpdata || (rowstride == 0))
+		return FALSE;
+
 	tempWidth = ROUND_UP_TO(context->width, 8);
 	tempHeight = ROUND_UP_TO(context->height, 2);
 	rw = (context->nsc_stream.ChromaSubSamplingLevel > 0 ? tempWidth : context->width);
 	ccl = context->nsc_stream.ColorLossLevel;
+
+	if (context->priv->plane_buf_length < rw * rowstride)
+		return FALSE;
+
+	if (rw < rowstride * 2)
+		return FALSE;
+
 	yplane = context->priv->plane_buf[0];
 	coplane = context->priv->plane_buf[1];
 	cgplane = context->priv->plane_buf[2];
@@ -196,32 +206,38 @@ static void nsc_encode_argb_to_aycocg(NS
 		memcpy(coplane + rw, coplane, rw);
 		memcpy(cgplane + rw, cgplane, rw);
 	}
+
+	return TRUE;
 }
 
-static void nsc_encode_subsampling(NSC_CONTEXT* context)
+static BOOL nsc_encode_subsampling(NSC_CONTEXT* context)
 {
 	UINT16 x;
 	UINT16 y;
-	BYTE* co_dst;
-	BYTE* cg_dst;
-	INT8* co_src0;
-	INT8* co_src1;
-	INT8* cg_src0;
-	INT8* cg_src1;
 	UINT32 tempWidth;
 	UINT32 tempHeight;
 
+
+	if (!context)
+		return FALSE;
+
 	tempWidth = ROUND_UP_TO(context->width, 8);
 	tempHeight = ROUND_UP_TO(context->height, 2);
 
+	if (tempHeight == 0)
+		return FALSE;
+
+	if (tempWidth > context->priv->plane_buf_length / tempHeight)
+		return FALSE;
+
 	for (y = 0; y < tempHeight >> 1; y++)
 	{
-		co_dst = context->priv->plane_buf[1] + y * (tempWidth >> 1);
-		cg_dst = context->priv->plane_buf[2] + y * (tempWidth >> 1);
-		co_src0 = (INT8*) context->priv->plane_buf[1] + (y << 1) * tempWidth;
-		co_src1 = co_src0 + tempWidth;
-		cg_src0 = (INT8*) context->priv->plane_buf[2] + (y << 1) * tempWidth;
-		cg_src1 = cg_src0 + tempWidth;
+		BYTE* co_dst = context->priv->plane_buf[1] + y * (tempWidth >> 1);
+		BYTE* cg_dst = context->priv->plane_buf[2] + y * (tempWidth >> 1);
+		const INT8* co_src0 = (INT8*) context->priv->plane_buf[1] + (y << 1) * tempWidth;
+		const INT8* co_src1 = co_src0 + tempWidth;
+		const INT8* cg_src0 = (INT8*) context->priv->plane_buf[2] + (y << 1) * tempWidth;
+		const INT8* cg_src1 = cg_src0 + tempWidth;
 		for (x = 0; x < tempWidth >> 1; x++)
 		{
 			*co_dst++ = (BYTE) (((INT16) *co_src0 + (INT16) *(co_src0 + 1) +
@@ -234,18 +250,28 @@ static void nsc_encode_subsampling(NSC_C
 			cg_src1 += 2;
 		}
 	}
+
+	return TRUE;
 }
 
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
 {
-	nsc_encode_argb_to_aycocg(context, bmpdata, rowstride);
+	if (!context || !bmpdata || (rowstride == 0))
+		return FALSE;
+
+	if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride))
+		return FALSE;
+
 	if (context->nsc_stream.ChromaSubSamplingLevel > 0)
 	{
-		nsc_encode_subsampling(context);
+		if (!nsc_encode_subsampling(context))
+			return FALSE;
 	}
+
+	return TRUE;
 }
 
-static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 origsz)
+static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 origsz)
 {
 	UINT32 left;
 	UINT32 runlength = 1;
Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_sse2.c
===================================================================
--- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_sse2.c
+++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_sse2.c
@@ -333,13 +333,15 @@ static void nsc_encode_subsampling_sse2(
 	}
 }
 
-static void nsc_encode_sse2(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
+static BOOL nsc_encode_sse2(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride)
 {
 	nsc_encode_argb_to_aycocg_sse2(context, bmpdata, rowstride);
 	if (context->nsc_stream.ChromaSubSamplingLevel > 0)
 	{
 		nsc_encode_subsampling_sse2(context);
 	}
+
+	return TRUE;
 }
 
 void nsc_init_sse2(NSC_CONTEXT* context)
Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.h
===================================================================
--- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_encode.h
+++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.h
@@ -20,6 +20,6 @@
 #ifndef __NSC_ENCODE_H
 #define __NSC_ENCODE_H
 
-void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
+BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride);
 
 #endif