From b1dee0f56c5de0a9b76b798cbc494356a87a089b Mon Sep 17 00:00:00 2001
From: Armin Novak <anovak@thincast.com>
Date: Thu, 27 Jul 2023 20:38:51 +0200
Subject: [PATCH] [codec,rfx] fix sign warnings

---
 libfreerdp/codec/rfx.c      | 45 +++++++++++++++++++++++--------------
 libfreerdp/codec/rfx_rlgr.c | 18 +++++++++++----
 2 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/libfreerdp/codec/rfx.c b/libfreerdp/codec/rfx.c
index 74bd429ae..e1aaa9285 100644
--- a/libfreerdp/codec/rfx.c
+++ b/libfreerdp/codec/rfx.c
@@ -820,7 +820,7 @@ static BOOL rfx_process_message_tileset(RFX_CONTEXT* context, RFX_MESSAGE* messa
                                         UINT16* pExpectedBlockType)
 {
 	BOOL rc;
-	int close_cnt;
+	size_t close_cnt = 0;
 	BYTE quant;
 	RFX_TILE* tile;
 	UINT32* quants;
@@ -1237,38 +1237,49 @@ BOOL rfx_process_message(RFX_CONTEXT* context, const BYTE* data, UINT32 length,
 
 	if (ok)
 	{
-		UINT32 i, j;
-		UINT32 nbUpdateRects;
-		REGION16 clippingRects;
-		const RECTANGLE_16* updateRects;
+		UINT32 nbUpdateRects = 0;
+		REGION16 clippingRects = { 0 };
+		const RECTANGLE_16* updateRects = NULL;
 		const DWORD formatSize = FreeRDPGetBytesPerPixel(context->pixel_format);
 		const UINT32 dstWidth = dstStride / FreeRDPGetBytesPerPixel(dstFormat);
 		region16_init(&clippingRects);
 
-		for (i = 0; i < message->numRects; i++)
+		WINPR_ASSERT(dstWidth <= UINT16_MAX);
+		WINPR_ASSERT(dstHeight <= UINT16_MAX);
+		for (UINT32 i = 0; i < message->numRects; i++)
 		{
-			RECTANGLE_16 clippingRect;
+			RECTANGLE_16 clippingRect = { 0 };
 			const RFX_RECT* rect = &(message->rects[i]);
-			clippingRect.left = MIN(left + rect->x, dstWidth);
-			clippingRect.top = MIN(top + rect->y, dstHeight);
-			clippingRect.right = MIN(clippingRect.left + rect->width, dstWidth);
-			clippingRect.bottom = MIN(clippingRect.top + rect->height, dstHeight);
+
+			WINPR_ASSERT(left + rect->x <= UINT16_MAX);
+			WINPR_ASSERT(top + rect->y <= UINT16_MAX);
+			WINPR_ASSERT(clippingRect.left + rect->width <= UINT16_MAX);
+			WINPR_ASSERT(clippingRect.top + rect->height <= UINT16_MAX);
+
+			clippingRect.left = (UINT16)MIN(left + rect->x, dstWidth);
+			clippingRect.top = (UINT16)MIN(top + rect->y, dstHeight);
+			clippingRect.right = (UINT16)MIN(clippingRect.left + rect->width, dstWidth);
+			clippingRect.bottom = (UINT16)MIN(clippingRect.top + rect->height, dstHeight);
 			region16_union_rect(&clippingRects, &clippingRects, &clippingRect);
 		}
 
-		for (i = 0; i < message->numTiles; i++)
+		for (UINT32 i = 0; i < message->numTiles; i++)
 		{
-			RECTANGLE_16 updateRect;
+			RECTANGLE_16 updateRect = { 0 };
 			const RFX_TILE* tile = rfx_message_get_tile(message, i);
-			updateRect.left = left + tile->x;
-			updateRect.top = top + tile->y;
+
+			WINPR_ASSERT(left + tile->x <= UINT16_MAX);
+			WINPR_ASSERT(top + tile->y <= UINT16_MAX);
+
+			updateRect.left = (UINT16)left + tile->x;
+			updateRect.top = (UINT16)top + tile->y;
 			updateRect.right = updateRect.left + 64;
 			updateRect.bottom = updateRect.top + 64;
 			region16_init(&updateRegion);
 			region16_intersect_rect(&updateRegion, &clippingRects, &updateRect);
 			updateRects = region16_rects(&updateRegion, &nbUpdateRects);
 
-			for (j = 0; j < nbUpdateRects; j++)
+			for (UINT32 j = 0; j < nbUpdateRects; j++)
 			{
 				const UINT32 stride = 64 * formatSize;
 				const UINT32 nXDst = updateRects[j].left;
@@ -1964,7 +1975,7 @@ static BOOL rfx_write_message_tileset(RFX_CONTEXT* context, wStream* s, const RF
 	Stream_Write_UINT32(s, message->tilesDataSize); /* tilesDataSize (4 bytes) */
 
 	UINT32* quantVals = message->quantVals;
-	for (size_t i = 0; i < message->numQuant * 5; i++)
+	for (size_t i = 0; i < message->numQuant * 5ul; i++)
 	{
 		WINPR_ASSERT(quantVals);
 		Stream_Write_UINT8(s, quantVals[0] + (quantVals[1] << 4));
diff --git a/libfreerdp/codec/rfx_rlgr.c b/libfreerdp/codec/rfx_rlgr.c
index 07072c55a..4d918a29e 100644
--- a/libfreerdp/codec/rfx_rlgr.c
+++ b/libfreerdp/codec/rfx_rlgr.c
@@ -92,34 +92,43 @@ static INLINE UINT32 lzcnt_s(UINT32 x)
 	if (!g_LZCNT)
 	{
 		UINT32 y;
-		int n = 32;
+		UINT32 n = 32;
 		y = x >> 16;
 		if (y != 0)
 		{
+			WINPR_ASSERT(n >= 16);
 			n = n - 16;
 			x = y;
 		}
 		y = x >> 8;
 		if (y != 0)
 		{
+			WINPR_ASSERT(n >= 8);
 			n = n - 8;
 			x = y;
 		}
 		y = x >> 4;
 		if (y != 0)
 		{
+			WINPR_ASSERT(n >= 4);
 			n = n - 4;
 			x = y;
 		}
 		y = x >> 2;
 		if (y != 0)
 		{
+			WINPR_ASSERT(n >= 2);
 			n = n - 2;
 			x = y;
 		}
 		y = x >> 1;
 		if (y != 0)
+		{
+			WINPR_ASSERT(n >= 2);
 			return n - 2;
+		}
+
+		WINPR_ASSERT(n >= x);
 		return n - x;
 	}
 
@@ -127,7 +136,7 @@ static INLINE UINT32 lzcnt_s(UINT32 x)
 }
 
 int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16* pDstData,
-                    UINT32 DstSize)
+                    UINT32 rDstSize)
 {
 	int vk;
 	int run;
@@ -148,6 +157,7 @@ int rfx_rlgr_decode(RLGR_MODE mode, const BYTE* pSrcData, UINT32 SrcSize, INT16*
 	INT16* pOutput;
 	wBitStream* bs;
 	wBitStream s_bs;
+	const SSIZE_T DstSize = rDstSize;
 
 	InitOnceExecuteOnce(&rfx_rlgr_init_once, rfx_rlgr_init, NULL, NULL);
 
@@ -646,7 +656,6 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* data, UINT32 data_size, BYTE* b
 		{
 			int numZeros;
 			int runmax;
-			int mag;
 			int sign;
 
 			/* RUN-LENGTH MODE */
@@ -680,7 +689,8 @@ int rfx_rlgr_encode(RLGR_MODE mode, const INT16* data, UINT32 data_size, BYTE* b
 			   need to output the last two bits, otherwise mstsc will crash */
 
 			/* encode the nonzero value using GR coding */
-			mag = (input < 0 ? -input : input); /* absolute value of input coefficient */
+			const UINT32 mag =
+			    (UINT32)(input < 0 ? -input : input); /* absolute value of input coefficient */
 			sign = (input < 0 ? 1 : 0);         /* sign of input coefficient */
 
 			OutputBit(1, sign);              /* output the sign bit */