mirrors
/
FreeRDP
mirror of https://github.com/FreeRDP/FreeRDP
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[codec] encode messages considering endianness 

The byte order of the pixels is affected by endianness, use bitwise
operations to access those bytes so the endiannes won't affect the final
result.
This commit is contained in:
Joan Torres 2023-07-31 19:07:08 +02:00
parent 8d6c27ba10
commit 6ba4aad9ab
2 changed files with 92 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
libfreerdp/codec/nsc_encode.c
View File

@ -117,6 +117,8 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
UINT16 rw; UINT16 rw;
BYTE ccl; BYTE ccl;
const BYTE* src; const BYTE* src;
const UINT32* src_32;
const UINT16* src_16;
BYTE* yplane = NULL; BYTE* yplane = NULL;
BYTE* coplane = NULL; BYTE* coplane = NULL;
BYTE* cgplane = NULL; BYTE* cgplane = NULL;
@ -138,69 +140,85 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
coplane = context->priv->PlaneBuffers[1] + y * rw; coplane = context->priv->PlaneBuffers[1] + y * rw;
cgplane = context->priv->PlaneBuffers[2] + y * rw; cgplane = context->priv->PlaneBuffers[2] + y * rw;
aplane = context->priv->PlaneBuffers[3] + y * context->width; aplane = context->priv->PlaneBuffers[3] + y * context->width;
src_32 = (UINT32*)src;
src_16 = (UINT16*)src;
for (x = 0; x < context->width; x++) for (x = 0; x < context->width; x++)
{ {
switch (context->format) switch (context->format)
{ {
case PIXEL_FORMAT_BGRX32: case PIXEL_FORMAT_BGRX32:
b_val = *src++; b_val = (INT16)(*src_32 & 0xFF);
g_val = *src++; g_val = (INT16)((*src_32 >> 8) & 0xFF);
r_val = *src++; r_val = (INT16)((*src_32 >> 16) & 0xFF);
src++;
a_val = 0xFF; a_val = 0xFF;
src_32++;
break; break;
case PIXEL_FORMAT_BGRA32: case PIXEL_FORMAT_BGRA32:
b_val = *src++; b_val = (INT16)(*src_32 & 0xFF);
g_val = *src++; g_val = (INT16)((*src_32 >> 8) & 0xFF);
r_val = *src++; r_val = (INT16)((*src_32 >> 16) & 0xFF);
a_val = *src++; a_val = (INT16)((*src_32 >> 24) & 0xFF);
src_32++;
break; break;
case PIXEL_FORMAT_RGBX32: case PIXEL_FORMAT_RGBX32:
r_val = *src++; r_val = (INT16)(*src_32 & 0xFF);
g_val = *src++; g_val = (INT16)((*src_32 >> 8) & 0xFF);
b_val = *src++; b_val = (INT16)((*src_32 >> 16) & 0xFF);
src++;
a_val = 0xFF; a_val = 0xFF;
src_32++;
break; break;
case PIXEL_FORMAT_RGBA32: case PIXEL_FORMAT_RGBA32:
r_val = *src++; r_val = (INT16)(*src_32 & 0xFF);
g_val = *src++; g_val = (INT16)((*src_32 >> 8) & 0xFF);
b_val = *src++; b_val = (INT16)((*src_32 >> 16) & 0xFF);
a_val = *src++; a_val = (INT16)((*src_32 >> 24) & 0xFF);
src_32++;
break; break;
case PIXEL_FORMAT_BGR24: case PIXEL_FORMAT_BGR24:
#ifdef __LITTLE_ENDIAN__
b_val = *src++; b_val = *src++;
g_val = *src++; g_val = *src++;
r_val = *src++; r_val = *src++;
#else
r_val = *src++;
g_val = *src++;
b_val = *src++;
#endif
a_val = 0xFF; a_val = 0xFF;
break; break;
case PIXEL_FORMAT_RGB24: case PIXEL_FORMAT_RGB24:
#ifdef __LITTLE_ENDIAN__
r_val = *src++; r_val = *src++;
g_val = *src++; g_val = *src++;
b_val = *src++; b_val = *src++;
#else
b_val = *src++;
g_val = *src++;
r_val = *src++;
#endif
a_val = 0xFF; a_val = 0xFF;
break; break;
case PIXEL_FORMAT_BGR16: case PIXEL_FORMAT_BGR16:
b_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); b_val = (INT16)((*src_16) & 0x1F);
g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); g_val = (INT16)((*src_16 >> 5) & 0x3F);
r_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); r_val = (INT16)((*src_16 >> 11) & 0x1F);
a_val = 0xFF; a_val = 0xFF;
src += 2; src_16++;
break; break;
case PIXEL_FORMAT_RGB16: case PIXEL_FORMAT_RGB16:
r_val = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); r_val = (INT16)((*src_16) & 0x1F);
g_val = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); g_val = (INT16)((*src_16 >> 5) & 0x3F);
b_val = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); b_val = (INT16)((*src_16 >> 11) & 0x1F);
a_val = 0xFF; a_val = 0xFF;
src += 2; src_16++;
break; break;
case PIXEL_FORMAT_A4: case PIXEL_FORMAT_A4:
@ -208,17 +226,17 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, UI
int shift; int shift;
BYTE idx; BYTE idx;
shift = (7 - (x % 8)); shift = (7 - (x % 8));
idx = ((*src) >> shift) & 1; idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1);
idx |= (((*(src + 1)) >> shift) & 1) << 1; idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1);
idx |= (((*(src + 2)) >> shift) & 1) << 2; idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2);
idx |= (((*(src + 3)) >> shift) & 1) << 3; idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3);
idx *= 3; idx *= 3;
r_val = (INT16)context->palette[idx]; r_val = (INT16)context->palette[idx];
g_val = (INT16)context->palette[idx + 1]; g_val = (INT16)context->palette[idx + 1];
b_val = (INT16)context->palette[idx + 2]; b_val = (INT16)context->palette[idx + 2];
if (shift == 0) if (shift == 0)
src += 4; src_32++;
} }
a_val = 0xFF; a_val = 0xFF;

74
libfreerdp/codec/rfx_encode.c
View File

@ -47,6 +47,8 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
int x_exceed; int x_exceed;
int y_exceed; int y_exceed;
const BYTE* src; const BYTE* src;
const UINT32* src_32;
const UINT16* src_16;
INT16 r, g, b; INT16 r, g, b;
INT16 *r_last, *g_last, *b_last; INT16 *r_last, *g_last, *b_last;
x_exceed = 64 - width; x_exceed = 64 - width;
@ -55,6 +57,8 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
for (y = 0; y < height; y++) for (y = 0; y < height; y++)
{ {
src = rgb_data + y * rowstride; src = rgb_data + y * rowstride;
src_32 = (UINT32*)src;
src_16 = (UINT16*)src;
switch (pixel_format) switch (pixel_format)
{ {
@ -62,10 +66,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGRA32: case PIXEL_FORMAT_BGRA32:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
*b_buf++ = (INT16)(*src++); *b_buf++ = (INT16)(*src_32 & 0xFF);
*g_buf++ = (INT16)(*src++); *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
*r_buf++ = (INT16)(*src++); *r_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
src++; src_32++;
} }
break; break;
@ -74,10 +78,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_ABGR32: case PIXEL_FORMAT_ABGR32:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
src++; *b_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
*b_buf++ = (INT16)(*src++); *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
*g_buf++ = (INT16)(*src++); *r_buf++ = (INT16)((*src_32 >> 24) & 0xFF);
*r_buf++ = (INT16)(*src++); src_32++;
} }
break; break;
@ -86,10 +90,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGBA32: case PIXEL_FORMAT_RGBA32:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
*r_buf++ = (INT16)(*src++); *r_buf++ = (INT16)(*src_32 & 0xFF);
*g_buf++ = (INT16)(*src++); *g_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
*b_buf++ = (INT16)(*src++); *b_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
src++; src_32++;
} }
break; break;
@ -98,10 +102,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_ARGB32: case PIXEL_FORMAT_ARGB32:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
src++; *r_buf++ = (INT16)((*src_32 >> 8) & 0xFF);
*r_buf++ = (INT16)(*src++); *g_buf++ = (INT16)((*src_32 >> 16) & 0xFF);
*g_buf++ = (INT16)(*src++); *b_buf++ = (INT16)((*src_32 >> 24) & 0xFF);
*b_buf++ = (INT16)(*src++); src_32++;
} }
break; break;
@ -109,9 +113,15 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGR24: case PIXEL_FORMAT_BGR24:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
#ifdef __LITTLE_ENDIAN__
*b_buf++ = (INT16)(*src++); *b_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++);
*r_buf++ = (INT16)(*src++); *r_buf++ = (INT16)(*src++);
#else
*r_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++);
*b_buf++ = (INT16)(*src++);
#endif
} }
break; break;
@ -119,9 +129,15 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGB24: case PIXEL_FORMAT_RGB24:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
#ifdef __LITTLE_ENDIAN__
*r_buf++ = (INT16)(*src++); *r_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++); *g_buf++ = (INT16)(*src++);
*b_buf++ = (INT16)(*src++); *b_buf++ = (INT16)(*src++);
#else
*b_buf++ = (INT16)(*src++);
*g_buf++ = (INT16)(*src++);
*r_buf++ = (INT16)(*src++);
#endif
} }
break; break;
@ -129,10 +145,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_BGR16: case PIXEL_FORMAT_BGR16:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
*b_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); *b_buf++ = (INT16)((*src_16) & 0x1F);
*g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F);
*r_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); *r_buf++ = (INT16)((*src_16 >> 11) & 0x1F);
src += 2; src_16++;
} }
break; break;
@ -140,10 +156,10 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
case PIXEL_FORMAT_RGB16: case PIXEL_FORMAT_RGB16:
for (x = 0; x < width; x++) for (x = 0; x < width; x++)
{ {
*r_buf++ = (INT16)(((*(src + 1)) & 0xF8) | ((*(src + 1)) >> 5)); *r_buf++ = (INT16)((*src_16 & 0x1F));
*g_buf++ = (INT16)((((*(src + 1)) & 0x07) << 5) | (((*src) & 0xE0) >> 3)); *g_buf++ = (INT16)((*src_16 >> 5) & 0x3F);
*b_buf++ = (INT16)((((*src) & 0x1F) << 3) | (((*src) >> 2) & 0x07)); *b_buf++ = (INT16)((*src_16 >> 11) & 0x1F);
src += 2; src_16++;
} }
break; break;
@ -157,17 +173,17 @@ static void rfx_encode_format_rgb(const BYTE* rgb_data, int width, int height, i
int shift; int shift;
BYTE idx; BYTE idx;
shift = (7 - (x % 8)); shift = (7 - (x % 8));
idx = ((*src) >> shift) & 1; idx = (BYTE)(((*src_32 & 0xFF) >> shift) & 1);
idx |= (((*(src + 1)) >> shift) & 1) << 1; idx |= (BYTE)(((((*src_32 >> 8) & 0xFF) >> shift) & 1) << 1);
idx |= (((*(src + 2)) >> shift) & 1) << 2; idx |= (BYTE)(((((*src_32 >> 16) & 0xFF) >> shift) & 1) << 2);
idx |= (((*(src + 3)) >> shift) & 1) << 3; idx |= (BYTE)(((((*src_32 >> 24) & 0xFF) >> shift) & 1) << 3);
idx *= 3; idx *= 3;
*r_buf++ = (INT16)palette[idx]; *r_buf++ = (INT16)palette[idx];
*g_buf++ = (INT16)palette[idx + 1]; *g_buf++ = (INT16)palette[idx + 1];
*b_buf++ = (INT16)palette[idx + 2]; *b_buf++ = (INT16)palette[idx + 2];
if (shift == 0) if (shift == 0)
src += 4; src_32++;
} }
break; break;