libfreerdp-codec: ClearCodec fix error codes and wrapping around of cursors
This commit is contained in:
Marc-André Moreau
parent
0ed7d3e6be
commit
9474784070
@ -97,7 +97,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
offset += 2;
|
||||
|
||||
if (seqNumber != clear->seqNumber)
|
||||
return -1;
|
||||
return -1004;
|
||||
|
||||
clear->seqNumber = (seqNumber + 1) % 256;
|
||||
|
||||
@ -110,21 +110,21 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
//printf("glyphFlags: 0x%02X seqNumber: %d\n", glyphFlags, seqNumber);
|
||||
|
||||
if ((glyphFlags & CLEARCODEC_FLAG_GLYPH_HIT) && !(glyphFlags & CLEARCODEC_FLAG_GLYPH_INDEX))
|
||||
return -1004;
|
||||
return -1005;
|
||||
|
||||
if (glyphFlags & CLEARCODEC_FLAG_GLYPH_INDEX)
|
||||
{
|
||||
if ((nWidth * nHeight) > (1024 * 1024))
|
||||
return -1005;
|
||||
return -1006;
|
||||
|
||||
if (SrcSize < 4)
|
||||
return -1006;
|
||||
return -1007;
|
||||
|
||||
glyphIndex = *((UINT16*) &pSrcData[2]);
|
||||
offset += 2;
|
||||
|
||||
if (glyphIndex >= 4000)
|
||||
return -1007;
|
||||
return -1008;
|
||||
|
||||
if (glyphFlags & CLEARCODEC_FLAG_GLYPH_HIT)
|
||||
{
|
||||
@ -136,7 +136,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
glyphData = clear->GlyphCache[glyphIndex];
|
||||
|
||||
if (!glyphData)
|
||||
return -1008;
|
||||
return -1009;
|
||||
|
||||
nSrcStep = nWidth * 4;
|
||||
pSrcPixel8 = glyphData;
|
||||
@ -156,7 +156,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
/* Read composition payload header parameters */
|
||||
|
||||
if ((SrcSize - offset) < 12)
|
||||
return -1009;
|
||||
return -1010;
|
||||
|
||||
residualByteCount = *((UINT32*) &pSrcData[offset]);
|
||||
bandsByteCount = *((UINT32*) &pSrcData[offset + 4]);
|
||||
@ -172,7 +172,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
BYTE* residualData;
|
||||
|
||||
if ((SrcSize - offset) < residualByteCount)
|
||||
return -1010;
|
||||
return -1011;
|
||||
|
||||
suboffset = 0;
|
||||
residualData = &pSrcData[offset];
|
||||
@ -182,7 +182,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
while (suboffset < residualByteCount)
|
||||
{
|
||||
if ((residualByteCount - suboffset) < 4)
|
||||
return -1011;
|
||||
return -1012;
|
||||
|
||||
color = RGB32(residualData[suboffset + 2], residualData[suboffset + 1], residualData[suboffset + 0]);
|
||||
suboffset += 3;
|
||||
@ -193,7 +193,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
if (runLengthFactor >= 0xFF)
|
||||
{
|
||||
if ((residualByteCount - suboffset) < 2)
|
||||
return -1012;
|
||||
return -1013;
|
||||
|
||||
runLengthFactor = (UINT32) *((UINT16*) &residualData[suboffset]);
|
||||
suboffset += 2;
|
||||
@ -201,7 +201,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
if (runLengthFactor >= 0xFFFF)
|
||||
{
|
||||
if ((residualByteCount - suboffset) < 4)
|
||||
return -1013;
|
||||
return -1014;
|
||||
|
||||
runLengthFactor = *((UINT32*) &residualData[suboffset]);
|
||||
suboffset += 4;
|
||||
@ -235,7 +235,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
}
|
||||
|
||||
if (pixelIndex != (nWidth * nHeight))
|
||||
return -1014;
|
||||
return -1015;
|
||||
|
||||
offset += residualByteCount;
|
||||
}
|
||||
@ -246,7 +246,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
UINT32 suboffset;
|
||||
|
||||
if ((SrcSize - offset) < bandsByteCount)
|
||||
return -1015;
|
||||
return -1016;
|
||||
|
||||
suboffset = 0;
|
||||
bandsData = &pSrcData[offset];
|
||||
@ -272,7 +272,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
CLEAR_VBAR_ENTRY* vBarShortEntry;
|
||||
|
||||
if ((bandsByteCount - suboffset) < 11)
|
||||
return -1016;
|
||||
return -1017;
|
||||
|
||||
xStart = *((UINT16*) &bandsData[suboffset]);
|
||||
xEnd = *((UINT16*) &bandsData[suboffset + 2]);
|
||||
@ -284,10 +284,10 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
suboffset += 3;
|
||||
|
||||
if (xEnd < xStart)
|
||||
return -1017;
|
||||
return -1018;
|
||||
|
||||
if (yEnd < yStart)
|
||||
return -1018;
|
||||
return -1019;
|
||||
|
||||
vBarCount = (xEnd - xStart) + 1;
|
||||
|
||||
@ -300,7 +300,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBar = &bandsData[suboffset];
|
||||
|
||||
if ((bandsByteCount - suboffset) < 2)
|
||||
return -1019;
|
||||
return -1020;
|
||||
|
||||
vBarHeader = *((UINT16*) &vBar[0]);
|
||||
suboffset += 2;
|
||||
@ -308,7 +308,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBarHeight = (yEnd - yStart + 1);
|
||||
|
||||
if (vBarHeight > 52)
|
||||
return -1020;
|
||||
return -1021;
|
||||
|
||||
if ((vBarHeader & 0xC000) == 0x4000) /* SHORT_VBAR_CACHE_HIT */
|
||||
{
|
||||
@ -329,12 +329,12 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBarShortPixelCount = (yEnd - yStart + 1 - vBarYOn); /* should be maximum value */
|
||||
|
||||
if (clear->ShortVBarStorageCursor >= 16384)
|
||||
return -1;
|
||||
return -1024;
|
||||
|
||||
vBarShortEntry = &(clear->ShortVBarStorage[clear->ShortVBarStorageCursor]);
|
||||
|
||||
if (!vBarShortEntry)
|
||||
return -1024;
|
||||
return -1025;
|
||||
|
||||
vBarShortPixelCount = vBarShortEntry->count;
|
||||
|
||||
@ -346,22 +346,22 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBarYOff = ((vBarHeader >> 8) & 0x3F);
|
||||
|
||||
if (vBarYOff < vBarYOn)
|
||||
return -1025;
|
||||
return -1026;
|
||||
|
||||
pSrcPixel8 = &vBar[2];
|
||||
vBarShortPixelCount = (vBarYOff - vBarYOn);
|
||||
|
||||
if (vBarShortPixelCount > 52)
|
||||
return -1026;
|
||||
return -1027;
|
||||
|
||||
//printf("SHORT_VBAR_CACHE_MISS: vBarYOn: %d vBarYOff: %d vBarPixelCount: %d Cursor: %d / %d\n",
|
||||
// vBarYOn, vBarYOff, vBarShortPixelCount, clear->VBarStorageCursor, clear->ShortVBarStorageCursor);
|
||||
|
||||
if ((bandsByteCount - suboffset) < (vBarShortPixelCount * 3))
|
||||
return -1027;
|
||||
return -1028;
|
||||
|
||||
if (clear->ShortVBarStorageCursor >= 16384)
|
||||
return -1;
|
||||
return -1029;
|
||||
|
||||
vBarShortEntry = &(clear->ShortVBarStorage[clear->ShortVBarStorageCursor]);
|
||||
|
||||
@ -369,7 +369,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBarShortEntry->pixels = (UINT32*) malloc(52 * 4);
|
||||
|
||||
if (!vBarShortEntry->pixels)
|
||||
return -1028;
|
||||
return -1030;
|
||||
|
||||
pDstPixel32 = vBarShortEntry->pixels;
|
||||
|
||||
@ -383,7 +383,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
suboffset += (vBarShortPixelCount * 3);
|
||||
|
||||
vBarShortEntry->count = vBarShortPixelCount;
|
||||
clear->ShortVBarStorageCursor++;
|
||||
clear->ShortVBarStorageCursor = (clear->ShortVBarStorageCursor + 1) % 16384;
|
||||
|
||||
vBarUpdate = TRUE;
|
||||
}
|
||||
@ -395,19 +395,19 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
// vBarIndex, clear->VBarStorageCursor, clear->ShortVBarStorageCursor);
|
||||
|
||||
if (vBarIndex >= 32768)
|
||||
return -1021;
|
||||
return -1031;
|
||||
|
||||
vBarEntry = &(clear->VBarStorage[vBarIndex]);
|
||||
}
|
||||
else
|
||||
{
|
||||
return -1029; /* invalid vBarHeader */
|
||||
return -1032; /* invalid vBarHeader */
|
||||
}
|
||||
|
||||
if (vBarUpdate)
|
||||
{
|
||||
if (clear->VBarStorageCursor >= 32768)
|
||||
return -1;
|
||||
return -1033;
|
||||
|
||||
vBarEntry = &(clear->VBarStorage[clear->VBarStorageCursor]);
|
||||
|
||||
@ -415,7 +415,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
vBarEntry->pixels = (UINT32*) malloc(52 * 4);
|
||||
|
||||
if (!vBarEntry->pixels)
|
||||
return -1030;
|
||||
return -1034;
|
||||
|
||||
vBarPixelCount = vBarHeight;
|
||||
pDstPixel32 = vBarEntry->pixels;
|
||||
@ -461,7 +461,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
}
|
||||
|
||||
vBarEntry->count = vBarPixelCount;
|
||||
clear->VBarStorageCursor++;
|
||||
clear->VBarStorageCursor = (clear->VBarStorageCursor + 1) % 32768;
|
||||
}
|
||||
|
||||
nXDstRel = nXDst + xStart;
|
||||
@ -523,7 +523,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
UINT32 suboffset;
|
||||
|
||||
if ((SrcSize - offset) < subcodecByteCount)
|
||||
return -1031;
|
||||
return -1035;
|
||||
|
||||
suboffset = 0;
|
||||
subcodecs = &pSrcData[offset];
|
||||
@ -531,7 +531,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
while (suboffset < subcodecByteCount)
|
||||
{
|
||||
if ((subcodecByteCount - suboffset) < 13)
|
||||
return -1032;
|
||||
return -1036;
|
||||
|
||||
xStart = *((UINT16*) &subcodecs[suboffset]);
|
||||
yStart = *((UINT16*) &subcodecs[suboffset + 2]);
|
||||
@ -545,7 +545,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
// bitmapDataByteCount, subcodecByteCount, suboffset, subcodecId);
|
||||
|
||||
if ((subcodecByteCount - suboffset) < bitmapDataByteCount)
|
||||
return -1033;
|
||||
return -1037;
|
||||
|
||||
nXDstRel = nXDst + xStart;
|
||||
nYDstRel = nYDst + yStart;
|
||||
@ -555,7 +555,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
if (subcodecId == 0) /* Uncompressed */
|
||||
{
|
||||
if (bitmapDataByteCount != (width * height * 3))
|
||||
return -1034;
|
||||
return -1038;
|
||||
|
||||
pSrcPixel8 = bitmapData;
|
||||
|
||||
@ -574,7 +574,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
else if (subcodecId == 1) /* NSCodec */
|
||||
{
|
||||
if (nsc_process_message(clear->nsc, 32, width, height, bitmapData, bitmapDataByteCount) < 0)
|
||||
return -1035;
|
||||
return -1039;
|
||||
|
||||
nSrcStep = width * 4;
|
||||
pSrcPixel8 = clear->nsc->BitmapData;
|
||||
@ -616,7 +616,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
while (bitmapDataOffset < bitmapDataByteCount)
|
||||
{
|
||||
if ((bitmapDataByteCount - bitmapDataOffset) < 2)
|
||||
return -1;
|
||||
return -1040;
|
||||
|
||||
stopIndex = bitmapData[bitmapDataOffset] & CLEAR_8BIT_MASKS[numBits];
|
||||
suiteDepth = (bitmapData[bitmapDataOffset] >> numBits) & CLEAR_8BIT_MASKS[numBits];
|
||||
@ -629,7 +629,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
if (runLengthFactor >= 0xFF)
|
||||
{
|
||||
if ((bitmapDataByteCount - bitmapDataOffset) < 2)
|
||||
return -1036;
|
||||
return -1041;
|
||||
|
||||
runLengthFactor = (UINT32) *((UINT16*) &bitmapData[bitmapDataOffset]);
|
||||
bitmapDataOffset += 2;
|
||||
@ -637,7 +637,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
if (runLengthFactor >= 0xFFFF)
|
||||
{
|
||||
if ((bitmapDataByteCount - bitmapDataOffset) < 4)
|
||||
return -1034;
|
||||
return -1042;
|
||||
|
||||
runLengthFactor = *((UINT32*) &bitmapData[bitmapDataOffset]);
|
||||
bitmapDataOffset += 4;
|
||||
@ -645,10 +645,10 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
}
|
||||
|
||||
if (startIndex > paletteCount)
|
||||
return -1037;
|
||||
return -1043;
|
||||
|
||||
if (stopIndex > paletteCount)
|
||||
return -1038;
|
||||
return -1044;
|
||||
|
||||
suiteIndex = startIndex;
|
||||
|
||||
@ -678,7 +678,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
}
|
||||
else
|
||||
{
|
||||
return -1039;
|
||||
return -1045;
|
||||
}
|
||||
|
||||
suboffset += bitmapDataByteCount;
|
||||
@ -700,7 +700,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
glyphData = clear->GlyphCache[glyphIndex];
|
||||
|
||||
if (!glyphData)
|
||||
return -1040;
|
||||
return -1046;
|
||||
|
||||
nSrcStep = nWidth * 4;
|
||||
pDstPixel8 = glyphData;
|
||||
@ -715,7 +715,7 @@ int clear_decompress(CLEAR_CONTEXT* clear, BYTE* pSrcData, UINT32 SrcSize,
|
||||
}
|
||||
|
||||
if (offset != SrcSize)
|
||||
return -1041;
|
||||
return -1047;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user