mirrors/FreeRDP
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

libfreerdp-codec: detect planar overflow per set of planes, not individual planes

Browse Source
This commit is contained in:
Marc-André Moreau 2013-11-27 16:23:20 -05:00
parent 7d3ce08e96
commit a5c1d0bbaa
1 changed files with 222 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
libfreerdp/codec/planar.c
View File

@ -87,10 +87,16 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
int nControlBytes;
cSegments = 0;
outPlaneSize = width * height;
if (!outPlane)
{
outPlaneSize = width * height * 2;
outPlane = malloc(outPlaneSize);
}
else
{
outPlaneSize = *dstSize;
}
dstp = outPlane;
@ -112,9 +118,9 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
if ((j != width) && (inPlane[(i * width) + j] == symbol))
{
nRunLength++;
continue;
}
else
{
if (nRunLength >= 3)
{
cRawBytes += 1;
@ -151,8 +157,6 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
cRawBytes -= 15;
rawValues += 15;
dstp += 15;
return NULL;
}
if (nRunLength > 47)
@ -220,7 +224,9 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
dstp++;
CopyMemory(dstp, rawValues, cRawBytes);
rawValues += cRawBytes;
dstp += cRawBytes;
cRawBytes = 0;
nRunLength -= 32;
*dstp = PLANAR_CONTROL_BYTE(2, nRunLength);
@ -241,7 +247,9 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
dstp++;
CopyMemory(dstp, rawValues, cRawBytes);
rawValues += cRawBytes;
dstp += cRawBytes;
cRawBytes = 0;
nRunLength -= 16;
*dstp = PLANAR_CONTROL_BYTE(1, nRunLength);
@ -262,18 +270,19 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
dstp++;
CopyMemory(dstp, rawValues, cRawBytes);
rawValues += cRawBytes;
dstp += cRawBytes;
}
rawValues = &inPlane[(i * width) + j];
cRawBytes = 0;
}
}
else
{
cRawBytes += (nRunLength + 1);
if (j == width)
{
printf("end of scanline: cRawBytes: %d nRunLength: %d\n", cRawBytes, nRunLength);
nRunLength = 0;
if (cRawBytes > 15)
@ -316,8 +325,6 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
nRunLength = 0;
}
}
}
//printf("---\n");
}
@ -328,20 +335,39 @@ BYTE* freerdp_bitmap_planar_compress_plane_rle(BYTE* inPlane, int width, int hei
return outPlane;
}
int freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[5], int width, int height, BYTE* outPlanes[5], int* dstSizes)
int freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[5], int width, int height, BYTE* outPlanes, int* dstSizes)
{
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes[0], &dstSizes[0]))
int outPlanesSize = width * height;
dstSizes[0] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes, &dstSizes[0]))
return 0;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes[1], &dstSizes[1]))
outPlanes += dstSizes[0];
outPlanesSize -= dstSizes[0];
dstSizes[1] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes, &dstSizes[1]))
return 0;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes[2], &dstSizes[2]))
outPlanes += dstSizes[1];
outPlanesSize -= dstSizes[1];
dstSizes[2] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes, &dstSizes[2]))
return 0;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes[3], &dstSizes[3]))
outPlanes += dstSizes[2];
outPlanesSize -= dstSizes[2];
dstSizes[3] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes, &dstSizes[3]))
return 0;
outPlanes += dstSizes[3];
outPlanesSize -= dstSizes[3];
return 1;
}
@ -404,12 +430,12 @@ BYTE* freerdp_bitmap_compress_planar(BYTE* data, UINT32 format, int width, int h
int size;
BYTE* dstp;
int planeSize;
int dstSizes[5];
int dstSizes[4];
BYTE* planes[5];
BYTE* planesBuffer;
BYTE* deltaPlanes[5];
BYTE* deltaPlanesBuffer;
BYTE* rlePlanes[5];
BYTE* rlePlanes[4];
BYTE* rlePlanesBuffer;
BYTE FormatHeader = 0;
@ -431,21 +457,21 @@ BYTE* freerdp_bitmap_compress_planar(BYTE* data, UINT32 format, int width, int h
deltaPlanes[3] = &deltaPlanesBuffer[planeSize * 3];
deltaPlanes[4] = &deltaPlanesBuffer[planeSize * 4];
rlePlanesBuffer = malloc(planeSize * 5);
rlePlanes[0] = &rlePlanesBuffer[planeSize * 0];
rlePlanes[1] = &rlePlanesBuffer[planeSize * 1];
rlePlanes[2] = &rlePlanesBuffer[planeSize * 2];
rlePlanes[3] = &rlePlanesBuffer[planeSize * 3];
rlePlanes[4] = &rlePlanesBuffer[planeSize * 4];
rlePlanesBuffer = malloc(planeSize * 4);
freerdp_split_color_planes(data, format, width, height, scanline, planes);
freerdp_bitmap_planar_delta_encode_planes(planes, width, height, deltaPlanes);
if (freerdp_bitmap_planar_compress_planes_rle(deltaPlanes, width, height, rlePlanes, (int*) &dstSizes) > 0)
if (freerdp_bitmap_planar_compress_planes_rle(deltaPlanes, width, height, rlePlanesBuffer, (int*) &dstSizes) > 0)
{
FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
rlePlanes[0] = &rlePlanesBuffer[0];
rlePlanes[1] = &rlePlanesBuffer[dstSizes[0]];
rlePlanes[2] = &rlePlanesBuffer[dstSizes[0] + dstSizes[1]];
rlePlanes[3] = &rlePlanesBuffer[dstSizes[0] + dstSizes[1] + dstSizes[2]];
printf("R: [%d/%d] G: [%d/%d] B: [%d/%d]\n",
dstSizes[1], planeSize, dstSizes[2], planeSize, dstSizes[3], planeSize);
}