mirrors/xrdp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fixes for rle bitmap compress limits and raw bitmaps

Browse Source
This commit is contained in:
Jay Sorg 2016-12-14 23:55:13 -08:00
parent 4462d22bf1
commit 492116d535
3 changed files with 159 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
libxrdp/xrdp_bitmap32_compress.c
View File

@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e,
}
start_line--;
cy++;
if (out_index > 64 * 64)
if (out_index + width + e > 64 * 64)
{
break;
}
@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e,
}
start_line--;
cy++;
if (out_index > 64 * 64)
if (out_index + width + e > 64 * 64)
{
break;
}
@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
char *sr_data;
char *sg_data;
char *sb_data;
char *hold_p;
int a_bytes;
int r_bytes;
int g_bytes;
@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
r_data = a_data + max_bytes;
g_data = r_data + max_bytes;
b_data = g_data + max_bytes;
hold_p = s->p;
if (header & FLAGS_NOALPHA)
{
@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy);
out_uint8(s, header);
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
total_bytes = r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
while (cy > 0)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb "
"bytes %d %d %d total_bytes %d cx %d cy %d "
"byte_limit %d", r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
max_bytes = cx * cy * 3;
if (total_bytes > max_bytes)
{
/* raw is better */
LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb "
"bytes %d %d %d total_bytes %d cx %d cy %d "
"max_bytes %d", r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
s->p = hold_p;
out_uint8(s, header);
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
max_bytes = cx * cy * 3;
total_bytes = r_bytes + g_bytes + b_bytes;
if (total_bytes > max_bytes)
{
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
break;
}
}
if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
}
}
else
{
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
while (cy > 0)
{
max_bytes = cx * cy * 3;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
break;
}
cy--;
}
}
}
else
@ -500,36 +511,45 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy);
out_uint8(s, header);
a_bytes = fpack(a_data, cx, cy, s);
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
max_bytes = cx * cy * 4;
total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
while (cy > 0)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb "
"bytes %d %d %d %d total_bytes %d cx %d cy %d "
"byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
if (total_bytes > max_bytes)
{
/* raw is better */
LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
"bytes %d %d %d %d total_bytes %d cx %d cy %d "
"max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
s->p = hold_p;
out_uint8(s, header);
a_bytes = fpack(a_data, cx, cy, s);
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
max_bytes = cx * cy * 4;
total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
if (total_bytes > max_bytes)
{
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
break;
}
}
if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
}
}
else
{
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
while (cy > 0)
{
max_bytes = cx * cy * 4;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
break;
}
cy--;
}
}
}
return cy;

8
libxrdp/xrdp_bitmap_compress.c
View File

@ -22,6 +22,8 @@
#include "libxrdp.h"
#define BC_MAX_BYTES (16 * 1024)
/*****************************************************************************/
#define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \
do { \
@ -695,7 +697,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end;
line = in_data + width * start_line;
while (start_line >= 0 && out_count < 32768)
while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{
i = (s->p - s->data) + count;
@ -987,7 +989,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 2;
line = in_data + width * start_line * 2;
while (start_line >= 0 && out_count < 32768)
while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{
i = (s->p - s->data) + count * 2;
@ -1279,7 +1281,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 3;
line = in_data + width * start_line * 4;
while (start_line >= 0 && out_count < 32768)
while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{
i = (s->p - s->data) + count * 3;

127
libxrdp/xrdp_orders.c
View File

@ -2235,6 +2235,11 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp;
while (bufsize + 16 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 16) != 0)
{
return 1;
@ -2254,33 +2259,58 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
out_uint16_le(self->out_s, bufsize);
out_uint16_le(self->out_s, cache_idx);
for (i = height - 1; i >= 0; i--)
if (Bpp == 4)
{
for (j = 0; j < width; j++)
for (i = height - 1; i >= 0; i--)
{
if (Bpp == 3)
for (j = 0; j < width; j++)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
}
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 3)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
}
else if (Bpp == 2)
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
}
else if (Bpp == 1)
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel);
}
}
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
out_uint8s(self->out_s, Bpp * e);
}
}
@ -2334,20 +2364,19 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
if (bpp > 24)
{
lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10);
}
else
{
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e);
}
if (lines_sending != height)
{
g_writeln("error in xrdp_orders_send_bitmap, lines_sending(%d) != \
height(%d)", lines_sending, height);
return 1;
height = lines_sending;
}
bufsize = (int)(s->p - p);
@ -2458,6 +2487,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
int pixel = 0;
int e = 0;
g_writeln("xrdp_orders_send_raw_bitmap2:");
if (width > 64)
{
g_writeln("error, width > 64");
@ -2479,6 +2509,11 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp;
while (bufsize + 14 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 14) != 0)
{
return 1;
@ -2499,7 +2534,22 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
i = cache_idx & 0xff;
out_uint8(self->out_s, i);
if (1 && Bpp == 3)
if (Bpp == 4)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
}
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 3)
{
for (i = height - 1; i >= 0; i--)
{
@ -2510,43 +2560,33 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
}
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
}
out_uint8s(self->out_s, Bpp * e);
}
}
else
else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
for (i = height - 1; i >= 0; i--)
{
if (Bpp == 3)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
}
else if (Bpp == 2)
for (j = 0; j < width; j++)
{
pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
}
else if (Bpp == 1)
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel);
}
out_uint8s(self->out_s, Bpp * e);
}
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
}
}
}
return 0;
@ -2599,20 +2639,19 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self,
if (bpp > 24)
{
lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10);
}
else
{
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384,
lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e);
}
if (lines_sending != height)
{
g_writeln("error in xrdp_orders_send_bitmap2, lines_sending(%d) != \
height(%d)", lines_sending, height);
return 1;
height = lines_sending;
}
bufsize = (int)(s->p - p);