server/shadow: Fix alpha pointer message so that it allows pixel with both 'xor' and 'and'.

Original pointer alpha update message is too specific for x11 implementation.
It doesn't allow pointer alpha mask with both 'xor' color and 'and' mask, e.g.: the 'edit' pointer in windows text box.

1. Move shadow_client_convert_alpha_pointer_data to x11 implementation as it is specific for x11.
2. Update message definition to be more generic: to accept 'xor/and' mask directly.
Implementation of subsystem can implement its own way to convert pointer mask data.
3. Fixed fault handling to free the resource allocated.

include/freerdp/server/shadow.h

@@ -231,9 +231,10 @@ struct _SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE
 	UINT32 yHot;
 	UINT32 width;
 	UINT32 height;
-	BYTE* pixels;
+	UINT32 lengthAndMask;
-	int scanline;
+	UINT32 lengthXorMask;
-	BOOL premultiplied;
+	BYTE* xorMaskData;
+	BYTE* andMaskData;
 };
 typedef struct _SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE;
 

server/shadow/X11/x11_shadow.c

@@ -368,7 +368,8 @@ static void x11_shadow_message_free(UINT32 id, SHADOW_MSG_OUT* msg)
 			break;
 
 		case SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE_ID:
-			free(((SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE*)msg)->pixels);
+			free(((SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE*)msg)->xorMaskData);
+			free(((SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE*)msg)->andMaskData);
 			free(msg);
 			break;
 
@@ -396,6 +397,89 @@ int x11_shadow_pointer_position_update(x11ShadowSubsystem* subsystem)
 	return shadow_client_boardcast_msg(subsystem->server, NULL, msgId, (SHADOW_MSG_OUT*) msg, NULL) ? 1 : -1;
 }
 
+static int x11_shadow_pointer_convert_alpha_pointer_data(BYTE* pixels, BOOL premultiplied,
+		UINT32 width, UINT32 height, SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE* pointerColor)
+{
+	UINT32 x, y;
+	BYTE* pSrc8;
+	BYTE* pDst8;
+	int xorStep;
+	int andStep;
+	UINT32 andBit;
+	BYTE* andBits;
+	UINT32 andPixel;
+	BYTE A, R, G, B;
+
+	xorStep = (width * 3);
+	xorStep += (xorStep % 2);
+
+	andStep = ((width + 7) / 8);
+	andStep += (andStep % 2);
+
+	pointerColor->lengthXorMask = height * xorStep;
+	pointerColor->xorMaskData = (BYTE*) calloc(1, pointerColor->lengthXorMask);
+
+	if (!pointerColor->xorMaskData)
+		return -1;
+
+	pointerColor->lengthAndMask = height * andStep;
+	pointerColor->andMaskData = (BYTE*) calloc(1, pointerColor->lengthAndMask);
+
+	if (!pointerColor->andMaskData)
+	{
+		free(pointerColor->xorMaskData);
+		pointerColor->xorMaskData = NULL;
+		return -1;
+	}
+
+	for (y = 0; y < height; y++)
+	{
+		pSrc8 = &pixels[(width * 4) * (height - 1 - y)];
+		pDst8 = &(pointerColor->xorMaskData[y * xorStep]);
+
+		andBit = 0x80;
+		andBits = &(pointerColor->andMaskData[andStep * y]);
+
+		for (x = 0; x < width; x++)
+		{
+			B = *pSrc8++;
+			G = *pSrc8++;
+			R = *pSrc8++;
+			A = *pSrc8++;
+
+			andPixel = 0;
+
+			if (A < 64)
+				A = 0; /* pixel cannot be partially transparent */
+
+			if (!A)
+			{
+				/* transparent pixel: XOR = black, AND = 1 */
+				andPixel = 1;
+				B = G = R = 0;
+			}
+			else
+			{
+				if (premultiplied)
+				{
+					B = (B * 0xFF ) / A;
+					G = (G * 0xFF ) / A;
+					R = (R * 0xFF ) / A;
+				}
+			}
+
+			*pDst8++ = B;
+			*pDst8++ = G;
+			*pDst8++ = R;
+
+			if (andPixel) *andBits |= andBit;
+			if (!(andBit >>= 1)) { andBits++; andBit = 0x80; }
+		}
+	}
+
+	return 1;
+}
+
 int x11_shadow_pointer_alpha_update(x11ShadowSubsystem* subsystem)
 {
 	SHADOW_MSG_OUT_POINTER_ALPHA_UPDATE* msg;
@@ -410,18 +494,14 @@ int x11_shadow_pointer_alpha_update(x11ShadowSubsystem* subsystem)
 	msg->yHot = subsystem->cursorHotY;
 	msg->width = subsystem->cursorWidth;
 	msg->height = subsystem->cursorHeight;
-	msg->scanline = msg->width * 4;
 
-	msg->pixels = (BYTE*) malloc(msg->scanline * msg->height);
+	if (x11_shadow_pointer_convert_alpha_pointer_data(subsystem->cursorPixels, TRUE,
-
+			msg->width, msg->height, msg) < 0)
-	if (!msg->pixels)
 	{
 		free (msg);
 		return -1;
 	}
 
-	CopyMemory(msg->pixels, subsystem->cursorPixels, msg->scanline * msg->height);
-	msg->premultiplied = TRUE;
 	msg->Free = x11_shadow_message_free;
 
 	return shadow_client_boardcast_msg(subsystem->server, NULL, msgId, (SHADOW_MSG_OUT*) msg, NULL) ? 1 : -1;

server/shadow/shadow_client.c

@@ -819,85 +819,6 @@ int shadow_client_surface_update(rdpShadowClient* client, REGION16* region)
 	return 1;
 }
 
-int shadow_client_convert_alpha_pointer_data(BYTE* pixels, BOOL premultiplied,
-		UINT32 width, UINT32 height, POINTER_COLOR_UPDATE* pointerColor)
-{
-	UINT32 x, y;
-	BYTE* pSrc8;
-	BYTE* pDst8;
-	int xorStep;
-	int andStep;
-	UINT32 andBit;
-	BYTE* andBits;
-	UINT32 andPixel;
-	BYTE A, R, G, B;
-
-	xorStep = (width * 3);
-	xorStep += (xorStep % 2);
-
-	andStep = ((width + 7) / 8);
-	andStep += (andStep % 2);
-
-	pointerColor->lengthXorMask = height * xorStep;
-	pointerColor->xorMaskData = (BYTE*) calloc(1, pointerColor->lengthXorMask);
-
-	if (!pointerColor->xorMaskData)
-		return -1;
-
-	pointerColor->lengthAndMask = height * andStep;
-	pointerColor->andMaskData = (BYTE*) calloc(1, pointerColor->lengthAndMask);
-
-	if (!pointerColor->andMaskData)
-		return -1;
-
-	for (y = 0; y < height; y++)
-	{
-		pSrc8 = &pixels[(width * 4) * (height - 1 - y)];
-		pDst8 = &(pointerColor->xorMaskData[y * xorStep]);
-
-		andBit = 0x80;
-		andBits = &(pointerColor->andMaskData[andStep * y]);
-
-		for (x = 0; x < width; x++)
-		{
-			B = *pSrc8++;
-			G = *pSrc8++;
-			R = *pSrc8++;
-			A = *pSrc8++;
-
-			andPixel = 0;
-
-			if (A < 64)
-				A = 0; /* pixel cannot be partially transparent */
-
-			if (!A)
-			{
-				/* transparent pixel: XOR = black, AND = 1 */
-				andPixel = 1;
-				B = G = R = 0;
-			}
-			else
-			{
-				if (premultiplied)
-				{
-					B = (B * 0xFF ) / A;
-					G = (G * 0xFF ) / A;
-					R = (R * 0xFF ) / A;
-				}
-			}
-
-			*pDst8++ = B;
-			*pDst8++ = G;
-			*pDst8++ = R;
-
-			if (andPixel) *andBits |= andBit;
-			if (!(andBit >>= 1)) { andBits++; andBit = 0x80; }
-		}
-	}
-
-	return 1;
-}
-
 int shadow_client_subsystem_process_message(rdpShadowClient* client, wMessage* message)
 {
 	rdpContext* context = (rdpContext*) client;
@@ -944,19 +865,17 @@ int shadow_client_subsystem_process_message(rdpShadowClient* client, wMessage* m
 			pointerColor->yPos = msg->yHot;
 			pointerColor->width = msg->width;
 			pointerColor->height = msg->height;
+			pointerColor->lengthAndMask = msg->lengthAndMask;
+			pointerColor->lengthXorMask = msg->lengthXorMask;
+			pointerColor->xorMaskData = msg->xorMaskData;
+			pointerColor->andMaskData = msg->andMaskData;
 
 			pointerCached.cacheIndex = pointerColor->cacheIndex;
 
 			if (client->activated)
 			{
-				shadow_client_convert_alpha_pointer_data(msg->pixels, msg->premultiplied,
-						msg->width, msg->height, pointerColor);
-
 				IFCALL(update->pointer->PointerNew, context, &pointerNew);
 				IFCALL(update->pointer->PointerCached, context, &pointerCached);
-
-				free(pointerColor->xorMaskData);
-				free(pointerColor->andMaskData);
 			}
 			break;
 		}