Backport of: From d1112c279bd1a327e8e4d0b5f371458bf2579659 Mon Sep 17 00:00:00 2001 From: Armin Novak Date: Mon, 22 Oct 2018 16:52:21 +0200 Subject: [PATCH] Fixed CVE-2018-8788 Thanks to Eyal Itkin from Check Point Software Technologies. --- include/freerdp/codec/nsc.h | 4 +- libfreerdp/codec/nsc.c | 94 +++++++++++++++++++++++++++++------ libfreerdp/codec/nsc_encode.c | 62 ++++++++++++++++------- libfreerdp/codec/nsc_encode.h | 2 +- libfreerdp/codec/nsc_sse2.c | 4 +- 5 files changed, 130 insertions(+), 36 deletions(-) Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/include/freerdp/codec/nsc.h =================================================================== --- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/include/freerdp/codec/nsc.h +++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/include/freerdp/codec/nsc.h @@ -59,8 +59,8 @@ struct _NSC_CONTEXT /* color palette allocated by the application */ const BYTE* palette; - void (*decode)(NSC_CONTEXT* context); - void (*encode)(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride); + BOOL (*decode)(NSC_CONTEXT* context); + BOOL (*encode)(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride); NSC_CONTEXT_PRIV* priv; }; Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc.c =================================================================== --- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc.c +++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc.c @@ -43,7 +43,7 @@ #define NSC_INIT_SIMD(_nsc_context) do { } while (0) #endif -static void nsc_decode(NSC_CONTEXT* context) +static BOOL nsc_decode(NSC_CONTEXT* context) { UINT16 x; UINT16 y; @@ -60,11 +60,18 @@ static void nsc_decode(NSC_CONTEXT* cont INT16 g_val; INT16 b_val; BYTE* bmpdata; + size_t pos = 0; + + if (!context) + return FALSE; bmpdata = context->bmpdata; rw = ROUND_UP_TO(context->width, 8); shift = context->nsc_stream.ColorLossLevel - 1; /* colorloss recovery + YCoCg shift */ + if (!bmpdata) + return FALSE; + for (y = 0; y < context->height; y++) { if (context->nsc_stream.ChromaSubSamplingLevel > 0) @@ -88,6 +95,11 @@ static void nsc_decode(NSC_CONTEXT* cont r_val = y_val + co_val - cg_val; g_val = y_val + cg_val; b_val = y_val - co_val - cg_val; + + if (pos + 4 > context->bmpdata_length) + return FALSE; + + pos += 4; *bmpdata++ = MINMAX(b_val, 0, 0xFF); *bmpdata++ = MINMAX(g_val, 0, 0xFF); *bmpdata++ = MINMAX(r_val, 0, 0xFF); @@ -98,9 +110,11 @@ static void nsc_decode(NSC_CONTEXT* cont aplane++; } } + + return TRUE; } -static void nsc_rle_decode(BYTE* in, BYTE* out, UINT32 origsz) +static BOOL nsc_rle_decode(BYTE* in, BYTE* out, UINT32 outSize, UINT32 origsz) { UINT32 len; UINT32 left; @@ -113,6 +127,10 @@ static void nsc_rle_decode(BYTE* in, BYT if (left == 5) { + if (outSize < 1) + return FALSE; + + outSize--; *out++ = value; left--; } @@ -130,6 +148,10 @@ static void nsc_rle_decode(BYTE* in, BYT len = *((UINT32*) in); in += 4; } + if (outSize < len) + return FALSE; + + outSize -= len; memset(out, value, len); out += len; left -= len; @@ -141,16 +163,24 @@ static void nsc_rle_decode(BYTE* in, BYT } } - *((UINT32*)out) = *((UINT32*)in); + if ((outSize < 4) || (left < 4)) + return FALSE; + + memcpy(out, in, 4); + return TRUE; } -static void nsc_rle_decompress_data(NSC_CONTEXT* context) +static BOOL nsc_rle_decompress_data(NSC_CONTEXT* context) { UINT16 i; BYTE* rle; UINT32 origsize; UINT32 planesize; + + if (!context) + return FALSE; + rle = context->nsc_stream.Planes; for (i = 0; i < 4; i++) @@ -159,14 +189,30 @@ static void nsc_rle_decompress_data(NSC_ planesize = context->nsc_stream.PlaneByteCount[i]; if (planesize == 0) + { + if (context->priv->plane_buf_length < origsize) + return FALSE; + memset(context->priv->plane_buf[i], 0xff, origsize); + } else if (planesize < origsize) - nsc_rle_decode(rle, context->priv->plane_buf[i], origsize); + { + if (!nsc_rle_decode(rle, context->priv->plane_buf[i], context->priv->plane_buf_length, + origsize)) + return FALSE; + } else + { + if (context->priv->plane_buf_length < origsize) + return FALSE; + memcpy(context->priv->plane_buf[i], rle, origsize); + } rle += planesize; } + + return TRUE; } static void nsc_stream_initialize(NSC_CONTEXT* context, wStream* s) @@ -337,12 +383,24 @@ void nsc_process_message(NSC_CONTEXT* co Stream_Free(s, FALSE); /* RLE decode */ - PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data); - nsc_rle_decompress_data(context); - PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data); + { + BOOL rc; + PROFILER_ENTER(context->priv->prof_nsc_rle_decompress_data); + rc = nsc_rle_decompress_data(context); + PROFILER_EXIT(context->priv->prof_nsc_rle_decompress_data); + + if (!rc) + return; + } /* Colorloss recover, Chroma supersample and AYCoCg to ARGB Conversion in one step */ - PROFILER_ENTER(context->priv->prof_nsc_decode); - context->decode(context); - PROFILER_EXIT(context->priv->prof_nsc_decode); + { + BOOL rc; + PROFILER_ENTER(context->priv->prof_nsc_decode); + rc = context->decode(context); + PROFILER_EXIT(context->priv->prof_nsc_decode); + + if (!rc) + return; + } } Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.c =================================================================== --- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_encode.c +++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.c @@ -67,7 +67,7 @@ static void nsc_context_initialize_encod } } -static void nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) +static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) { UINT16 x; UINT16 y; @@ -85,10 +85,20 @@ static void nsc_encode_argb_to_aycocg(NS UINT32 tempWidth; UINT32 tempHeight; + if (!context || bmpdata || (rowstride == 0)) + return FALSE; + tempWidth = ROUND_UP_TO(context->width, 8); tempHeight = ROUND_UP_TO(context->height, 2); rw = (context->nsc_stream.ChromaSubSamplingLevel > 0 ? tempWidth : context->width); ccl = context->nsc_stream.ColorLossLevel; + + if (context->priv->plane_buf_length < rw * rowstride) + return FALSE; + + if (rw < rowstride * 2) + return FALSE; + yplane = context->priv->plane_buf[0]; coplane = context->priv->plane_buf[1]; cgplane = context->priv->plane_buf[2]; @@ -196,32 +206,38 @@ static void nsc_encode_argb_to_aycocg(NS memcpy(coplane + rw, coplane, rw); memcpy(cgplane + rw, cgplane, rw); } + + return TRUE; } -static void nsc_encode_subsampling(NSC_CONTEXT* context) +static BOOL nsc_encode_subsampling(NSC_CONTEXT* context) { UINT16 x; UINT16 y; - BYTE* co_dst; - BYTE* cg_dst; - INT8* co_src0; - INT8* co_src1; - INT8* cg_src0; - INT8* cg_src1; UINT32 tempWidth; UINT32 tempHeight; + + if (!context) + return FALSE; + tempWidth = ROUND_UP_TO(context->width, 8); tempHeight = ROUND_UP_TO(context->height, 2); + if (tempHeight == 0) + return FALSE; + + if (tempWidth > context->priv->plane_buf_length / tempHeight) + return FALSE; + for (y = 0; y < tempHeight >> 1; y++) { - co_dst = context->priv->plane_buf[1] + y * (tempWidth >> 1); - cg_dst = context->priv->plane_buf[2] + y * (tempWidth >> 1); - co_src0 = (INT8*) context->priv->plane_buf[1] + (y << 1) * tempWidth; - co_src1 = co_src0 + tempWidth; - cg_src0 = (INT8*) context->priv->plane_buf[2] + (y << 1) * tempWidth; - cg_src1 = cg_src0 + tempWidth; + BYTE* co_dst = context->priv->plane_buf[1] + y * (tempWidth >> 1); + BYTE* cg_dst = context->priv->plane_buf[2] + y * (tempWidth >> 1); + const INT8* co_src0 = (INT8*) context->priv->plane_buf[1] + (y << 1) * tempWidth; + const INT8* co_src1 = co_src0 + tempWidth; + const INT8* cg_src0 = (INT8*) context->priv->plane_buf[2] + (y << 1) * tempWidth; + const INT8* cg_src1 = cg_src0 + tempWidth; for (x = 0; x < tempWidth >> 1; x++) { *co_dst++ = (BYTE) (((INT16) *co_src0 + (INT16) *(co_src0 + 1) + @@ -234,18 +250,28 @@ static void nsc_encode_subsampling(NSC_C cg_src1 += 2; } } + + return TRUE; } -void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) +BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) { - nsc_encode_argb_to_aycocg(context, bmpdata, rowstride); + if (!context || !bmpdata || (rowstride == 0)) + return FALSE; + + if (!nsc_encode_argb_to_aycocg(context, bmpdata, rowstride)) + return FALSE; + if (context->nsc_stream.ChromaSubSamplingLevel > 0) { - nsc_encode_subsampling(context); + if (!nsc_encode_subsampling(context)) + return FALSE; } + + return TRUE; } -static UINT32 nsc_rle_encode(BYTE* in, BYTE* out, UINT32 origsz) +static UINT32 nsc_rle_encode(const BYTE* in, BYTE* out, UINT32 origsz) { UINT32 left; UINT32 runlength = 1; Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_sse2.c =================================================================== --- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_sse2.c +++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_sse2.c @@ -333,13 +333,15 @@ static void nsc_encode_subsampling_sse2( } } -static void nsc_encode_sse2(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) +static BOOL nsc_encode_sse2(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride) { nsc_encode_argb_to_aycocg_sse2(context, bmpdata, rowstride); if (context->nsc_stream.ChromaSubSamplingLevel > 0) { nsc_encode_subsampling_sse2(context); } + + return TRUE; } void nsc_init_sse2(NSC_CONTEXT* context) Index: freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.h =================================================================== --- freerdp-1.1.0~git20140921.1.440916e+dfsg1.orig/libfreerdp/codec/nsc_encode.h +++ freerdp-1.1.0~git20140921.1.440916e+dfsg1/libfreerdp/codec/nsc_encode.h @@ -20,6 +20,6 @@ #ifndef __NSC_ENCODE_H #define __NSC_ENCODE_H -void nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride); +BOOL nsc_encode(NSC_CONTEXT* context, BYTE* bmpdata, int rowstride); #endif