Merge pull request #2825 from akallabeth/pointer_decoding_fix

Fixed color pointer decoding.

client/Mac/MRDPView.m

@@ -1075,9 +1075,17 @@ BOOL mf_Pointer_New(rdpContext* context, rdpPointer* pointer)
 		return FALSE;
 	mrdpCursor->cursor_data = cursor_data;
 
-	freerdp_image_copy_from_pointer_data(cursor_data, PIXEL_FORMAT_ARGB32,
+	if (freerdp_image_copy_from_pointer_data(
+		    cursor_data, PIXEL_FORMAT_ARGB32,
 		    pointer->width * 4, 0, 0, pointer->width, pointer->height,
-					     pointer->xorMaskData, pointer->andMaskData, pointer->xorBpp, NULL);
+		    pointer->xorMaskData, pointer->lengthXorMask,
+		    pointer->andMaskData, pointer->lengthAndMask,
+		    pointer->xorBpp, NULL) < 0)
+	{
+		free(cursor_data);
+		mrdpCursor->cursor_data = NULL;
+		return FALSE;
+	}
 
 	/* store cursor bitmap image in representation - required by NSImage */
 	bmiRep = [[NSBitmapImageRep alloc] initWithBitmapDataPlanes:(unsigned char **) &cursor_data

client/X11/xf_graphics.c

@@ -237,11 +237,16 @@ BOOL xf_Pointer_New(rdpContext* context, rdpPointer* pointer)
 		return FALSE;
 	}
 
-	if ((pointer->andMaskData != 0) && (pointer->xorMaskData != 0))
+	if (freerdp_image_copy_from_pointer_data(
-	{
+		    (BYTE*) ci.pixels, PIXEL_FORMAT_ARGB32,
-		freerdp_image_copy_from_pointer_data((BYTE*) ci.pixels, PIXEL_FORMAT_ARGB32,
 		    pointer->width * 4, 0, 0, pointer->width, pointer->height,
-				pointer->xorMaskData, pointer->andMaskData, pointer->xorBpp, xfc->palette);
+		    pointer->xorMaskData, pointer->lengthXorMask,
+		    pointer->andMaskData, pointer->lengthAndMask,
+		    pointer->xorBpp, xfc->palette) < 0)
+	{
+		free(ci.pixels);
+		xf_unlock_x11(xfc, FALSE);
+		return FALSE;
 	}
 
 	((xfPointer*) pointer)->cursor = XcursorImageLoadCursor(xfc->display, &ci);

include/freerdp/codec/color.h

@@ -452,8 +452,11 @@ FREERDP_API void freerdp_clrconv_free(HCLRCONV clrconv);
 FREERDP_API int freerdp_image_copy_from_monochrome(BYTE* pDstData, UINT32 DstFormat, int nDstStep, int nXDst, int nYDst,
 		int nWidth, int nHeight, BYTE* pSrcData, UINT32 backColor, UINT32 foreColor, BYTE* palette);
 
-FREERDP_API int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat, int nDstStep, int nXDst, int nYDst,
+FREERDP_API int freerdp_image_copy_from_pointer_data(
-		int nWidth, int nHeight, BYTE* xorMask, BYTE* andMask, UINT32 xorBpp, BYTE* palette);
+		BYTE* pDstData, UINT32 DstFormat, int nDstStep,
+		int nXDst, int nYDst, int nWidth, int nHeight, BYTE* xorMask,
+		UINT32 xorMaskLength, BYTE* andMask, UINT32 andMaskLength,
+		UINT32 xorBpp, BYTE* palette);
 
 FREERDP_API int freerdp_image8_copy(BYTE* pDstData, DWORD DstFormat, int nDstStep, int nXDst, int nYDst,
 		int nWidth, int nHeight, BYTE* pSrcData, DWORD SrcFormat, int nSrcStep, int nXSrc, int nYSrc, BYTE* palette);

libfreerdp/codec/color.c

@@ -1426,8 +1426,12 @@ void freerdp_alpha_cursor_convert(BYTE* alphaData, BYTE* xorMask, BYTE* andMask,
  * http://msdn.microsoft.com/en-us/library/windows/hardware/ff556138/
  */
 
-int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat, int nDstStep, int nXDst, int nYDst,
+int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat,
-		int nWidth, int nHeight, BYTE* xorMask, BYTE* andMask, UINT32 xorBpp, BYTE* palette)
+					 int nDstStep, int nXDst, int nYDst,
+					 int nWidth, int nHeight, BYTE* xorMask,
+					 UINT32 xorMaskLength, BYTE* andMask,
+					 UINT32 andMaskLength, UINT32 xorBpp,
+					 BYTE* palette)
 {
 	int x, y;
 	BOOL vFlip;
@@ -1463,15 +1467,27 @@ int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat, int n
 	andStep = (nWidth + 7) / 8;
 	andStep += (andStep % 2);
 
+	if (!xorMask || (xorMaskLength == 0))
+		return -1;
+
 	if (dstBytesPerPixel == 4)
 	{
 		UINT32* pDstPixel;
 
 		if (xorBpp == 1)
 		{
+			if (!andMask || (andMaskLength == 0))
+				return -1;
+
 			xorStep = (nWidth + 7) / 8;
 			xorStep += (xorStep % 2);
 
+			if (xorStep * nHeight > xorMaskLength)
+				return -1;
+
+			if (andStep * nHeight > andMaskLength)
+				return -1;
+
 			pDstPixel = (UINT32*) &pDstData[(nYDst * nDstStep) + (nXDst * 4)];
 
 			for (y = 0; y < nHeight; y++)
@@ -1525,26 +1541,41 @@ int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat, int n
 				return -1;
 			}
 
+			if (xorStep * nHeight > xorMaskLength)
+				return -1;
+
+			if (andMask)
+			{
+				if (andStep * nHeight > andMaskLength)
+					return -1;
+			}
+
 			for (y = 0; y < nHeight; y++)
 			{
 				andBit = 0x80;
 
 				if (!vFlip)
 				{
+					if (andMask)
 						andBits = &andMask[andStep * y];
 					xorBits = &xorMask[xorStep * y];
 				}
 				else
 				{
+					if (andMask)
 						andBits = &andMask[andStep * (nHeight - y - 1)];
 					xorBits = &xorMask[xorStep * (nHeight - y - 1)];
 				}
 
 				for (x = 0; x < nWidth; x++)
 				{
+					BOOL ignoreAndMask = FALSE;
+
 					if (xorBpp == 32)
 					{
 						xorPixel = *((UINT32*) xorBits);
+						if (xorPixel & 0xFF000000)
+							ignoreAndMask = TRUE;
 					}
 					else if (xorBpp == 16)
 					{
@@ -1563,14 +1594,21 @@ int freerdp_image_copy_from_pointer_data(BYTE* pDstData, UINT32 DstFormat, int n
 
 					xorBits += xorBytesPerPixel;
 
+					andPixel = 0;
+					if (andMask)
+					{
 						andPixel = (*andBits & andBit) ? 1 : 0;
 						if (!(andBit >>= 1)) { andBits++; andBit = 0x80; }
+					}
 
-					if (andPixel)
+					/* Ignore the AND mask, if the color format already supplies alpha data. */
+					if (andPixel && !ignoreAndMask)
 					{
-						if (xorPixel == 0xFF000000) /* black */
+						const UINT32 xorPixelMasked = xorPixel | 0xFF000000;
+
+						if (xorPixelMasked == 0xFF000000) /* black */
 							*pDstPixel++ = 0x00000000; /* transparent */
-						else if (xorPixel == 0xFFFFFFFF) /* white */
+						else if (xorPixelMasked == 0xFFFFFFFF) /* white */
 							*pDstPixel++ = 0xFF000000; /* inverted (set as black) */
 						else
 							*pDstPixel++ = xorPixel;