diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c index bf62e58c0..589a24bd3 100644 --- a/libfreerdp/codec/planar.c +++ b/libfreerdp/codec/planar.c @@ -309,7 +309,7 @@ static INLINE BOOL planar_decompress_planes_raw(const BYTE* pSrcData[4], for (y = beg; y != end; y += inc) { BYTE* pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * GetBytesPerPixel( - DstFormat))]; + DstFormat))]; if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA)) return FALSE; @@ -320,7 +320,7 @@ static INLINE BOOL planar_decompress_planes_raw(const BYTE* pSrcData[4], for (y = beg; y != end; y += inc) { BYTE* pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * GetBytesPerPixel( - DstFormat))]; + DstFormat))]; if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA)) return FALSE; @@ -998,7 +998,7 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane, return outPlane; } -static INLINE UINT32 freerdp_bitmap_planar_compress_planes_rle( +static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle( BYTE* inPlanes[4], UINT32 width, UINT32 height, BYTE* outPlanes, UINT32* dstSizes, BOOL skipAlpha) { @@ -1015,7 +1015,7 @@ static INLINE UINT32 freerdp_bitmap_planar_compress_planes_rle( if (!freerdp_bitmap_planar_compress_plane_rle( inPlanes[0], width, height, outPlanes, &dstSizes[0])) - return 0; + return FALSE; outPlanes += dstSizes[0]; outPlanesSize -= dstSizes[0]; @@ -1026,7 +1026,7 @@ static INLINE UINT32 freerdp_bitmap_planar_compress_planes_rle( if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes, &dstSizes[1])) - return 0; + return FALSE; outPlanes += dstSizes[1]; outPlanesSize -= dstSizes[1]; @@ -1035,7 +1035,7 @@ static INLINE UINT32 freerdp_bitmap_planar_compress_planes_rle( if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes, &dstSizes[2])) - return 0; + return FALSE; outPlanes += dstSizes[2]; outPlanesSize -= dstSizes[2]; @@ -1044,9 +1044,9 @@ static INLINE UINT32 freerdp_bitmap_planar_compress_planes_rle( if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes, &dstSizes[3])) - return 0; + return FALSE; - return 1; + return TRUE; } BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* inPlane, @@ -1136,10 +1136,12 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* context, context->deltaPlanes)) return NULL; - if (freerdp_bitmap_planar_compress_planes_rle( + if (!freerdp_bitmap_planar_compress_planes_rle( context->deltaPlanes, width, height, context->rlePlanesBuffer, dstSizes, - context->AllowSkipAlpha) > 0) + context->AllowSkipAlpha)) + return NULL; + { int offset = 0; FormatHeader |= PLANAR_FORMAT_HEADER_RLE; @@ -1155,6 +1157,24 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* context, } } + if (FormatHeader & PLANAR_FORMAT_HEADER_RLE) + { + if (!context->AllowRunLengthEncoding) + return NULL; + + if (context->rlePlanes[0] == NULL) + return NULL; + + if (context->rlePlanes[1] == NULL) + return NULL; + + if (context->rlePlanes[2] == NULL) + return NULL; + + if (context->rlePlanes[3] == NULL) + return NULL; + } + if (!dstData) { size = 1;