mirrors/FreeRDP
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
d2f73136d3
FreeRDP/libfreerdp/codec/planar.c
Martin Fleisz 1112e2fffb codec: Set alpha channel to FF if not used (#5431) 
* Formatting

* codec: Set alpha channel to FF if not used

This PR initializes the alpha channel to FF. Before the alpha channel
contained uninitialized memory potentially causing problems when the
image data was used by an alpha channel aware frontend.
2019-06-13 09:19:42 +02:00

1285 lines
31 KiB
C
Raw Blame History

/**
* FreeRDP: A Remote Desktop Protocol Implementation
* RDP6 Planar Codec
*
* Copyright 2013 Marc-Andre Moreau <marcandre.moreau@gmail.com>
* Copyright 2016 Armin Novak <armin.novak@thincast.com>
* Copyright 2016 Thincast Technologies GmbH
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifdef HAVE_CONFIG_H
# include "config.h"
#endif
#include <winpr/crt.h>
#include <winpr/print.h>
#include <freerdp/primitives.h>
#include <freerdp/log.h>
#include <freerdp/codec/bitmap.h>
#include <freerdp/codec/planar.h>
#define TAG FREERDP_TAG("codec")
static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* plane, UINT32 width,
UINT32 height, BYTE* outPlane,
UINT32* dstSize);
static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* inPlane, UINT32 width,
UINT32 height, BYTE* outPlane);
static INLINE INT32 planar_skip_plane_rle(const BYTE* pSrcData, UINT32 SrcSize, UINT32 nWidth,
UINT32 nHeight)
{
UINT32 x, y;
BYTE controlByte;
const BYTE* pRLE = pSrcData;
const BYTE* pEnd = &pSrcData[SrcSize];
for (y = 0; y < nHeight; y++)
{
for (x = 0; x < nWidth;)
{
int cRawBytes;
int nRunLength;
if (pRLE >= pEnd)
return -1;
controlByte = *pRLE++;
nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
if (nRunLength == 1)
{
nRunLength = cRawBytes + 16;
cRawBytes = 0;
}
else if (nRunLength == 2)
{
nRunLength = cRawBytes + 32;
cRawBytes = 0;
}
pRLE += cRawBytes;
x += cRawBytes;
x += nRunLength;
if (x > nWidth)
return -1;
if (pRLE > pEnd)
return -1;
}
}
return (INT32)(pRLE - pSrcData);
}
static INLINE INT32 planar_decompress_plane_rle(const BYTE* pSrcData, UINT32 SrcSize,
BYTE* pDstData, INT32 nDstStep, UINT32 nXDst,
UINT32 nYDst, UINT32 nWidth, UINT32 nHeight,
UINT32 nChannel, BOOL vFlip)
{
INT32 x, y;
UINT32 pixel;
UINT32 cRawBytes;
UINT32 nRunLength;
INT32 deltaValue;
INT32 beg, end, inc;
BYTE controlByte;
BYTE* currentScanline;
BYTE* previousScanline;
const BYTE* srcp = pSrcData;
if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX) || (nDstStep > INT32_MAX))
return -1;
previousScanline = NULL;
if (vFlip)
{
beg = (INT32)nHeight - 1;
end = -1;
inc = -1;
}
else
{
beg = 0;
end = (INT32)nHeight;
inc = 1;
}
for (y = beg; y != end; y += inc)
{
BYTE* dstp = &pDstData[((nYDst + y) * (INT32)nDstStep) + (nXDst * 4) + nChannel];
pixel = 0;
currentScanline = dstp;
for (x = 0; x < (INT32)nWidth;)
{
controlByte = *srcp;
srcp++;
if ((srcp - pSrcData) > SrcSize)
{
WLog_ERR(TAG, "error reading input buffer");
return -1;
}
nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
if (nRunLength == 1)
{
nRunLength = cRawBytes + 16;
cRawBytes = 0;
}
else if (nRunLength == 2)
{
nRunLength = cRawBytes + 32;
cRawBytes = 0;
}
if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4)
{
WLog_ERR(TAG, "too many pixels in scanline");
return -1;
}
if (!previousScanline)
{
/* first scanline, absolute values */
while (cRawBytes > 0)
{
pixel = *srcp;
srcp++;
*dstp = pixel;
dstp += 4;
x++;
cRawBytes--;
}
while (nRunLength > 0)
{
*dstp = pixel;
dstp += 4;
x++;
nRunLength--;
}
}
else
{
/* delta values relative to previous scanline */
while (cRawBytes > 0)
{
deltaValue = *srcp;
srcp++;
if (deltaValue & 1)
{
deltaValue = deltaValue >> 1;
deltaValue = deltaValue + 1;
pixel = -deltaValue;
}
else
{
deltaValue = deltaValue >> 1;
pixel = deltaValue;
}
deltaValue = previousScanline[x * 4] + pixel;
*dstp = deltaValue;
dstp += 4;
x++;
cRawBytes--;
}
while (nRunLength > 0)
{
deltaValue = previousScanline[x * 4] + pixel;
*dstp = deltaValue;
dstp += 4;
x++;
nRunLength--;
}
}
}
previousScanline = currentScanline;
}
return (INT32)(srcp - pSrcData);
}
static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, INT32 nDstStep, UINT32 nXDst,
UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
BOOL vFlip)
{
INT32 x, y;
INT32 beg, end, inc;
if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX) || (nDstStep > INT32_MAX))
return -1;
if (vFlip)
{
beg = (INT32)nHeight - 1;
end = -1;
inc = -1;
}
else
{
beg = 0;
end = (INT32)nHeight;
inc = 1;
}
for (y = beg; y != end; y += inc)
{
BYTE* dstp = &pDstData[((nYDst + y) * (INT32)nDstStep) + (nXDst * 4) + nChannel];
for (x = 0; x < (INT32)nWidth; ++x)
{
*dstp = bValue;
dstp += 4;
}
}
return 0;
}
static INLINE BOOL writeLine(BYTE** ppRgba, UINT32 DstFormat, UINT32 width, const BYTE** ppR,
const BYTE** ppG, const BYTE** ppB, const BYTE** ppA)
{
UINT32 x;
if (!ppRgba || !ppR || !ppG || !ppB)
return FALSE;
switch (DstFormat)
{
case PIXEL_FORMAT_BGRA32:
for (x = 0; x < width; x++)
{
*(*ppRgba)++ = *(*ppB)++;
*(*ppRgba)++ = *(*ppG)++;
*(*ppRgba)++ = *(*ppR)++;
*(*ppRgba)++ = *(*ppA)++;
}
return TRUE;
case PIXEL_FORMAT_BGRX32:
for (x = 0; x < width; x++)
{
*(*ppRgba)++ = *(*ppB)++;
*(*ppRgba)++ = *(*ppG)++;
*(*ppRgba)++ = *(*ppR)++;
*(*ppRgba)++ = 0xFF;
}
return TRUE;
default:
if (ppA)
{
for (x = 0; x < width; x++)
{
BYTE alpha = *(*ppA)++;
UINT32 color = FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
WriteColor(*ppRgba, DstFormat, color);
*ppRgba += GetBytesPerPixel(DstFormat);
}
}
else
{
const BYTE alpha = 0xFF;
for (x = 0; x < width; x++)
{
UINT32 color = FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
WriteColor(*ppRgba, DstFormat, color);
*ppRgba += GetBytesPerPixel(DstFormat);
}
}
return TRUE;
}
}
static INLINE BOOL planar_decompress_planes_raw(const BYTE* pSrcData[4], BYTE* pDstData,
UINT32 DstFormat, UINT32 nDstStep, UINT32 nXDst,
UINT32 nYDst, UINT32 nWidth, UINT32 nHeight,
BOOL vFlip)
{
INT32 y;
INT32 beg, end, inc;
const BYTE* pR = pSrcData[0];
const BYTE* pG = pSrcData[1];
const BYTE* pB = pSrcData[2];
const BYTE* pA = pSrcData[3];
if (vFlip)
{
beg = nHeight - 1;
end = -1;
inc = -1;
}
else
{
beg = 0;
end = nHeight;
inc = 1;
}
for (y = beg; y != end; y += inc)
{
BYTE* pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * GetBytesPerPixel(DstFormat))];
if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
return FALSE;
}
return TRUE;
}
BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* planar, const BYTE* pSrcData, UINT32 SrcSize,
UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE* pDstData, UINT32 DstFormat,
UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
UINT32 nDstHeight, BOOL vFlip)
{
BOOL cs;
BOOL rle;
UINT32 cll;
BOOL alpha;
BOOL useAlpha = FALSE;
INT32 status;
const BYTE* srcp;
UINT32 subSize;
UINT32 subWidth;
UINT32 subHeight;
UINT32 planeSize;
INT32 rleSizes[4] = { 0, 0, 0, 0 };
UINT32 rawSizes[4];
UINT32 rawWidths[4];
UINT32 rawHeights[4];
BYTE FormatHeader;
const BYTE* planes[4] = { 0 };
const UINT32 w = MIN(nSrcWidth, nDstWidth);
const UINT32 h = MIN(nSrcHeight, nDstHeight);
const primitives_t* prims = primitives_get();
if (nDstStep <= 0)
nDstStep = nDstWidth * GetBytesPerPixel(DstFormat);
srcp = pSrcData;
if (!pDstData)
{
WLog_ERR(TAG, "Invalid argument pDstData=NULL");
return FALSE;
}
FormatHeader = *srcp++;
cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE;
rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE;
alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE;
if (alpha)
useAlpha = ColorHasAlpha(DstFormat);
// WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
// alpha);
if (!cll && cs)
return FALSE; /* Chroma subsampling requires YCoCg */
subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
planeSize = nSrcWidth * nSrcHeight;
subSize = subWidth * subHeight;
if (!cs)
{
rawSizes[0] = planeSize; /* LumaOrRedPlane */
rawWidths[0] = nSrcWidth;
rawHeights[0] = nSrcHeight;
rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
rawWidths[1] = nSrcWidth;
rawHeights[1] = nSrcHeight;
rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
rawWidths[2] = nSrcWidth;
rawHeights[2] = nSrcHeight;
rawSizes[3] = planeSize; /* AlphaPlane */
rawWidths[3] = nSrcWidth;
rawHeights[3] = nSrcHeight;
}
else /* Chroma Subsampling */
{
rawSizes[0] = planeSize; /* LumaOrRedPlane */
rawWidths[0] = nSrcWidth;
rawHeights[0] = nSrcHeight;
rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
rawWidths[1] = subWidth;
rawHeights[1] = subHeight;
rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
rawWidths[2] = subWidth;
rawHeights[2] = subHeight;
rawSizes[3] = planeSize; /* AlphaPlane */
rawWidths[3] = nSrcWidth;
rawHeights[3] = nSrcHeight;
}
if (!rle) /* RAW */
{
if (alpha)
{
if ((SrcSize - (srcp - pSrcData)) < (planeSize * 4))
return FALSE;
planes[3] = srcp; /* AlphaPlane */
planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
return FALSE;
}
else
{
if ((SrcSize - (srcp - pSrcData)) < (planeSize * 3))
return FALSE;
planes[0] = srcp; /* LumaOrRedPlane */
planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
return FALSE;
}
}
else /* RLE */
{
if (alpha)
{
planes[3] = srcp;
rleSizes[3] = planar_skip_plane_rle(planes[3], SrcSize - (planes[3] - pSrcData),
rawWidths[3], rawHeights[3]); /* AlphaPlane */
if (rleSizes[3] < 0)
return FALSE;
planes[0] = planes[3] + rleSizes[3];
}
else
planes[0] = srcp;
rleSizes[0] = planar_skip_plane_rle(planes[0], SrcSize - (planes[0] - pSrcData),
rawWidths[0], rawHeights[0]); /* RedPlane */
if (rleSizes[0] < 0)
return FALSE;
planes[1] = planes[0] + rleSizes[0];
rleSizes[1] = planar_skip_plane_rle(planes[1], SrcSize - (planes[1] - pSrcData),
rawWidths[1], rawHeights[1]); /* GreenPlane */
if (rleSizes[1] < 1)
return FALSE;
planes[2] = planes[1] + rleSizes[1];
rleSizes[2] = planar_skip_plane_rle(planes[2], SrcSize - (planes[2] - pSrcData),
rawWidths[2], rawHeights[2]); /* BluePlane */
if (rleSizes[2] < 1)
return FALSE;
}
if (!cll) /* RGB */
{
UINT32 TempFormat;
BYTE* pTempData = pDstData;
UINT32 nTempStep = nDstStep;
if (useAlpha)
TempFormat = PIXEL_FORMAT_BGRA32;
else
TempFormat = PIXEL_FORMAT_BGRX32;
if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
{
pTempData = planar->pTempData;
nTempStep = planar->nTempStep;
}
if (!rle) /* RAW */
{
if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
nYDst, nSrcWidth, nSrcHeight, vFlip))
return FALSE;
if (alpha)
srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
else /* NoAlpha */
srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
if ((SrcSize - (srcp - pSrcData)) == 1)
srcp++; /* pad */
}
else /* RLE */
{
status =
planar_decompress_plane_rle(planes[0], rleSizes[0], pTempData, nTempStep, nXDst,
nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
if (status < 0)
return FALSE;
status = planar_decompress_plane_rle(planes[1], rleSizes[1], pTempData, nTempStep,
nXDst, nYDst, nSrcWidth, nSrcHeight, 1,
vFlip); /* GreenPlane */
if (status < 0)
return FALSE;
status =
planar_decompress_plane_rle(planes[2], rleSizes[2], pTempData, nTempStep, nXDst,
nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
if (status < 0)
return FALSE;
srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
if (useAlpha)
{
status = planar_decompress_plane_rle(planes[3], rleSizes[3], pTempData, nTempStep,
nXDst, nYDst, nSrcWidth, nSrcHeight, 3,
vFlip); /* AlphaPlane */
}
else
status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
nSrcHeight, 3, vFlip);
if (status < 0)
return FALSE;
if (alpha)
srcp += rleSizes[3];
}
if (pTempData != pDstData)
{
if (!freerdp_image_copy(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h, pTempData,
TempFormat, nTempStep, nXDst, nYDst, NULL, FREERDP_FLIP_NONE))
return FALSE;
}
}
else /* YCoCg */
{
UINT32 TempFormat;
BYTE* pTempData = planar->pTempData;
UINT32 nTempStep = planar->nTempStep;
if (useAlpha)
TempFormat = PIXEL_FORMAT_BGRA32;
else
TempFormat = PIXEL_FORMAT_BGRX32;
if (!pTempData)
return FALSE;
if (cs)
{
WLog_ERR(TAG, "Chroma subsampling unimplemented");
return FALSE;
}
if (!rle) /* RAW */
{
if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
nYDst, nSrcWidth, nSrcHeight, vFlip))
return FALSE;
if (alpha)
srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
else /* NoAlpha */
srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
if ((SrcSize - (srcp - pSrcData)) == 1)
srcp++; /* pad */
}
else /* RLE */
{
if (useAlpha)
{
status = planar_decompress_plane_rle(planes[3], rleSizes[3], pTempData, nTempStep,
nXDst, nYDst, nSrcWidth, nSrcHeight, 3,
vFlip); /* AlphaPlane */
if (status < 0)
return FALSE;
}
if (alpha)
srcp += rleSizes[3];
status =
planar_decompress_plane_rle(planes[0], rleSizes[0], pTempData, nTempStep, nXDst,
nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* LumaPlane */
if (status < 0)
return FALSE;
status = planar_decompress_plane_rle(planes[1], rleSizes[1], pTempData, nTempStep,
nXDst, nYDst, nSrcWidth, nSrcHeight, 1,
vFlip); /* OrangeChromaPlane */
if (status < 0)
return FALSE;
status = planar_decompress_plane_rle(planes[2], rleSizes[2], pTempData, nTempStep,
nXDst, nYDst, nSrcWidth, nSrcHeight, 0,
vFlip); /* GreenChromaPlane */
if (status < 0)
return FALSE;
srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
}
if (prims->YCoCgToRGB_8u_AC4R(pTempData, nTempStep, pDstData, DstFormat, nDstStep, w, h,
cll, useAlpha) != PRIMITIVES_SUCCESS)
return FALSE;
}
return (SrcSize == (srcp - pSrcData)) ? TRUE : FALSE;
}
static INLINE BOOL freerdp_split_color_planes(const BYTE* data, UINT32 format, UINT32 width,
UINT32 height, UINT32 scanline, BYTE* planes[4])
{
INT32 i, j, k;
if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
return FALSE;
k = 0;
if (scanline == 0)
scanline = width * GetBytesPerPixel(format);
for (i = (INT32)height - 1; i >= 0; i--)
{
const BYTE* pixel = &data[(INT32)scanline * i];
for (j = 0; j < (INT32)width; j++)
{
const UINT32 color = ReadColor(pixel, format);
pixel += GetBytesPerPixel(format);
SplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k], &planes[0][k],
NULL);
k++;
}
}
return TRUE;
}
static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* pInBuffer, UINT32 cRawBytes,
UINT32 nRunLength, BYTE* pOutBuffer,
UINT32 outBufferSize)
{
const BYTE* pInput;
BYTE* pOutput;
BYTE controlByte;
UINT32 nBytesToWrite;
pInput = pInBuffer;
pOutput = pOutBuffer;
if (!cRawBytes && !nRunLength)
return 0;
if (nRunLength < 3)
{
cRawBytes += nRunLength;
nRunLength = 0;
}
while (cRawBytes)
{
if (cRawBytes < 16)
{
if (nRunLength > 15)
{
if (nRunLength < 18)
{
controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
nRunLength -= 13;
cRawBytes = 0;
}
else
{
controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
nRunLength -= 15;
cRawBytes = 0;
}
}
else
{
controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
nRunLength = 0;
cRawBytes = 0;
}
}
else
{
controlByte = PLANAR_CONTROL_BYTE(0, 15);
cRawBytes -= 15;
}
if (outBufferSize < 1)
return 0;
outBufferSize--;
*pOutput = controlByte;
pOutput++;
nBytesToWrite = (int)(controlByte >> 4);
if (nBytesToWrite)
{
if (outBufferSize < nBytesToWrite)
return 0;
outBufferSize -= nBytesToWrite;
CopyMemory(pOutput, pInput, nBytesToWrite);
pOutput += nBytesToWrite;
pInput += nBytesToWrite;
}
}
while (nRunLength)
{
if (nRunLength > 47)
{
if (nRunLength < 50)
{
controlByte = PLANAR_CONTROL_BYTE(2, 13);
nRunLength -= 45;
}
else
{
controlByte = PLANAR_CONTROL_BYTE(2, 15);
nRunLength -= 47;
}
}
else if (nRunLength > 31)
{
controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
nRunLength = 0;
}
else if (nRunLength > 15)
{
controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
nRunLength = 0;
}
else
{
controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
nRunLength = 0;
}
if (outBufferSize < 1)
return 0;
--outBufferSize;
*pOutput = controlByte;
pOutput++;
}
return (pOutput - pOutBuffer);
}
static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* pInBuffer,
UINT32 inBufferSize, BYTE* pOutBuffer,
UINT32 outBufferSize)
{
BYTE symbol;
const BYTE* pInput;
BYTE* pOutput;
const BYTE* pBytes;
UINT32 cRawBytes;
UINT32 nRunLength;
UINT32 bSymbolMatch;
UINT32 nBytesWritten;
UINT32 nTotalBytesWritten;
symbol = 0;
cRawBytes = 0;
nRunLength = 0;
pInput = pInBuffer;
pOutput = pOutBuffer;
nTotalBytesWritten = 0;
if (!outBufferSize)
return 0;
do
{
if (!inBufferSize)
break;
bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
symbol = *pInput;
pInput++;
inBufferSize--;
if (nRunLength && !bSymbolMatch)
{
if (nRunLength < 3)
{
cRawBytes += nRunLength;
nRunLength = 0;
}
else
{
pBytes = pInput - (cRawBytes + nRunLength + 1);
nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
pOutput, outBufferSize);
nRunLength = 0;
if (!nBytesWritten || (nBytesWritten > outBufferSize))
return nRunLength;
nTotalBytesWritten += nBytesWritten;
outBufferSize -= nBytesWritten;
pOutput += nBytesWritten;
cRawBytes = 0;
}
}
nRunLength += bSymbolMatch;
cRawBytes += (!bSymbolMatch) ? TRUE : FALSE;
} while (outBufferSize);
if (cRawBytes || nRunLength)
{
pBytes = pInput - (cRawBytes + nRunLength);
nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
pOutput, outBufferSize);
if (!nBytesWritten)
return 0;
nTotalBytesWritten += nBytesWritten;
}
if (inBufferSize)
return 0;
return nTotalBytesWritten;
}
BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane, UINT32 width, UINT32 height,
BYTE* outPlane, UINT32* dstSize)
{
UINT32 index;
const BYTE* pInput;
BYTE* pOutput;
UINT32 outBufferSize;
UINT32 nBytesWritten;
UINT32 nTotalBytesWritten;
if (!outPlane)
return FALSE;
index = 0;
pInput = inPlane;
pOutput = outPlane;
outBufferSize = *dstSize;
nTotalBytesWritten = 0;
while (outBufferSize)
{
nBytesWritten =
freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
return FALSE;
outBufferSize -= nBytesWritten;
nTotalBytesWritten += nBytesWritten;
pOutput += nBytesWritten;
pInput += width;
index++;
if (index >= height)
break;
}
*dstSize = nTotalBytesWritten;
return TRUE;
}
static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[4], UINT32 width,
UINT32 height, BYTE* outPlanes,
UINT32* dstSizes, BOOL skipAlpha)
{
UINT32 outPlanesSize = width * height * 4;
/* AlphaPlane */
if (skipAlpha)
{
dstSizes[0] = 0;
}
else
{
dstSizes[0] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
&dstSizes[0]))
return FALSE;
outPlanes += dstSizes[0];
outPlanesSize -= dstSizes[0];
}
/* LumaOrRedPlane */
dstSizes[1] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
&dstSizes[1]))
return FALSE;
outPlanes += dstSizes[1];
outPlanesSize -= dstSizes[1];
/* OrangeChromaOrGreenPlane */
dstSizes[2] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
&dstSizes[2]))
return FALSE;
outPlanes += dstSizes[2];
outPlanesSize -= dstSizes[2];
/* GreenChromeOrBluePlane */
dstSizes[3] = outPlanesSize;
if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
&dstSizes[3]))
return FALSE;
return TRUE;
}
BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* inPlane, UINT32 width, UINT32 height,
BYTE* outPlane)
{
char s2c;
INT32 delta;
UINT32 y, x;
BYTE* outPtr;
const BYTE *srcPtr, *prevLinePtr;
if (!outPlane)
{
if (width * height == 0)
return NULL;
if (!(outPlane = (BYTE*)calloc(height, width)))
return NULL;
}
// first line is copied as is
CopyMemory(outPlane, inPlane, width);
outPtr = outPlane + width;
srcPtr = inPlane + width;
prevLinePtr = inPlane;
for (y = 1; y < height; y++)
{
for (x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
{
delta = *srcPtr - *prevLinePtr;
s2c = (delta >= 0) ? (char)delta : (char)(~((BYTE)(-delta)) + 1);
s2c = (s2c >= 0) ? (s2c << 1) : (char)(((~((BYTE)s2c) + 1) << 1) - 1);
*outPtr = (BYTE)s2c;
}
}
return outPlane;
}
static INLINE BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* inPlanes[4], UINT32 width,
UINT32 height, BYTE* outPlanes[4])
{
UINT32 i;
for (i = 0; i < 4; i++)
{
outPlanes[i] =
freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
if (!outPlanes[i])
return FALSE;
}
return TRUE;
}
BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* context, const BYTE* data,
UINT32 format, UINT32 width, UINT32 height, UINT32 scanline,
BYTE* dstData, UINT32* pDstSize)
{
UINT32 size;
BYTE* dstp;
UINT32 planeSize;
UINT32 dstSizes[4] = { 0 };
BYTE FormatHeader = 0;
if (!context || !context->rlePlanesBuffer)
return NULL;
if (context->AllowSkipAlpha)
FormatHeader |= PLANAR_FORMAT_HEADER_NA;
planeSize = width * height;
if (!freerdp_split_color_planes(data, format, width, height, scanline, context->planes))
return NULL;
if (context->AllowRunLengthEncoding)
{
if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
context->deltaPlanes))
return NULL;
if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
context->rlePlanesBuffer, dstSizes,
context->AllowSkipAlpha))
return NULL;
{
int offset = 0;
FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
offset += dstSizes[0];
context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
offset += dstSizes[1];
context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
offset += dstSizes[2];
context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
// WLog_DBG(TAG, "R: [%"PRIu32"/%"PRIu32"] G: [%"PRIu32"/%"PRIu32"] B:
// [%"PRIu32"/%"PRIu32"]", dstSizes[1], planeSize, dstSizes[2], planeSize,
// dstSizes[3],
// planeSize);
}
}
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
{
if (!context->AllowRunLengthEncoding)
return NULL;
if (context->rlePlanes[0] == NULL)
return NULL;
if (context->rlePlanes[1] == NULL)
return NULL;
if (context->rlePlanes[2] == NULL)
return NULL;
if (context->rlePlanes[3] == NULL)
return NULL;
}
if (!dstData)
{
size = 1;
if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
{
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
size += dstSizes[0];
else
size += planeSize;
}
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
else
size += (planeSize * 3);
if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
size++;
dstData = malloc(size);
if (!dstData)
return NULL;
*pDstSize = size;
}
dstp = dstData;
*dstp = FormatHeader; /* FormatHeader */
dstp++;
/* AlphaPlane */
if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
{
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
{
CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
dstp += dstSizes[0];
}
else
{
CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
dstp += planeSize;
}
}
/* LumaOrRedPlane */
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
{
CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
dstp += dstSizes[1];
}
else
{
CopyMemory(dstp, context->planes[1], planeSize); /* Red */
dstp += planeSize;
}
/* OrangeChromaOrGreenPlane */
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
{
CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
dstp += dstSizes[2];
}
else
{
CopyMemory(dstp, context->planes[2], planeSize); /* Green */
dstp += planeSize;
}
/* GreenChromeOrBluePlane */
if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
{
CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
dstp += dstSizes[3];
}
else
{
CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
dstp += planeSize;
}
/* Pad1 (1 byte) */
if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
{
*dstp = 0;
dstp++;
}
size = (dstp - dstData);
*pDstSize = size;
return dstData;
}
BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* context, UINT32 width,
UINT32 height)
{
if (!context)
return FALSE;
context->maxWidth = width;
context->maxHeight = height;
context->maxPlaneSize = context->maxWidth * context->maxHeight;
context->nTempStep = context->maxWidth * 4;
free(context->planesBuffer);
free(context->pTempData);
free(context->deltaPlanesBuffer);
free(context->rlePlanesBuffer);
context->planesBuffer = calloc(context->maxPlaneSize, 4);
context->pTempData = calloc(context->maxPlaneSize, 4);
context->deltaPlanesBuffer = calloc(context->maxPlaneSize, 4);
context->rlePlanesBuffer = calloc(context->maxPlaneSize, 4);
if (!context->planesBuffer || !context->pTempData || !context->deltaPlanesBuffer ||
!context->rlePlanesBuffer)
return FALSE;
context->planes[0] = &context->planesBuffer[context->maxPlaneSize * 0];
context->planes[1] = &context->planesBuffer[context->maxPlaneSize * 1];
context->planes[2] = &context->planesBuffer[context->maxPlaneSize * 2];
context->planes[3] = &context->planesBuffer[context->maxPlaneSize * 3];
context->deltaPlanes[0] = &context->deltaPlanesBuffer[context->maxPlaneSize * 0];
context->deltaPlanes[1] = &context->deltaPlanesBuffer[context->maxPlaneSize * 1];
context->deltaPlanes[2] = &context->deltaPlanesBuffer[context->maxPlaneSize * 2];
context->deltaPlanes[3] = &context->deltaPlanesBuffer[context->maxPlaneSize * 3];
return TRUE;
}
BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
UINT32 maxHeight)
{
BITMAP_PLANAR_CONTEXT* context;
context = (BITMAP_PLANAR_CONTEXT*)calloc(1, sizeof(BITMAP_PLANAR_CONTEXT));
if (!context)
return NULL;
if (flags & PLANAR_FORMAT_HEADER_NA)
context->AllowSkipAlpha = TRUE;
if (flags & PLANAR_FORMAT_HEADER_RLE)
context->AllowRunLengthEncoding = TRUE;
if (flags & PLANAR_FORMAT_HEADER_CS)
context->AllowColorSubsampling = TRUE;
context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
if (context->ColorLossLevel)
context->AllowDynamicColorFidelity = TRUE;
if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
{
freerdp_bitmap_planar_context_free(context);
return NULL;
}
return context;
}
void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
{
if (!context)
return;
free(context->pTempData);
free(context->planesBuffer);
free(context->deltaPlanesBuffer);
free(context->rlePlanesBuffer);
free(context);
}
Reference in New Issue View Git Blame Copy Permalink