mirrors/FreeRDP
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[codec,color] fix overlapping check

Browse Source 
only consider images that do not contain the same lines not overlapping.
Ignore any x offsets as this may lead to alignment problems resulting in
invalid overlapping areas.
This commit is contained in:
akallabeth 2024-10-22 11:02:11 +02:00
parent 5ad826a503
commit d8dc2956e5
No known key found for this signature in database
GPG Key ID: A49454A3FC909FD5
1 changed files with 13 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
libfreerdp/codec/color.c
View File

@ -549,22 +549,20 @@ BOOL freerdp_image_copy_from_pointer_data(BYTE* WINPR_RESTRICT pDstData, UINT32
} }
} }
static INLINE BOOL overlapping(const BYTE* pDstData, UINT32 nXDst, UINT32 nYDst, UINT32 nDstStep, static INLINE BOOL overlapping(const BYTE* pDstData, UINT32 nYDst, UINT32 nDstStep,
UINT32 dstBytesPerPixel, const BYTE* pSrcData, UINT32 nXSrc, const BYTE* pSrcData, UINT32 nYSrc, UINT32 nSrcStep, UINT32 nHeight)
UINT32 nYSrc, UINT32 nSrcStep, UINT32 srcBytesPerPixel,
UINT32 nWidth, UINT32 nHeight)
{ {
const BYTE* pDstStart = &pDstData[1ULL * nXDst * dstBytesPerPixel + 1ULL * nYDst * nDstStep]; const uintptr_t src = (uintptr_t)pSrcData;
const BYTE* pDstEnd = pDstStart + 1ULL * nHeight * nDstStep; const uintptr_t srcstart = src + 1ULL * nSrcStep * nYSrc;
const BYTE* pSrcStart = &pSrcData[1ULL * nXSrc * srcBytesPerPixel + 1ULL * nYSrc * nSrcStep]; const uintptr_t srcend = srcstart + 1ULL * nSrcStep * nHeight;
const BYTE* pSrcEnd = pSrcStart + 1ULL * nHeight * nSrcStep; const uintptr_t dst = (uintptr_t)pDstData;
const uintptr_t dststart = dst + 1ULL * nDstStep * nYDst;
const uintptr_t dstend = dststart + 1ULL * nDstStep * nHeight;
WINPR_UNUSED(nWidth); if ((dststart >= srcstart) && (dststart <= srcend))
if ((pDstStart >= pSrcStart) && (pDstStart <= pSrcEnd))
return TRUE; return TRUE;
if ((pDstEnd >= pSrcStart) && (pDstEnd <= pSrcEnd)) if ((dstend >= srcstart) && (dstend <= srcend))
return TRUE; return TRUE;
return FALSE; return FALSE;
@ -733,8 +731,7 @@ BOOL freerdp_image_copy_overlap(BYTE* pDstData, DWORD DstFormat, UINT32 nDstStep
SSIZE_T dstVOffset = 0; SSIZE_T dstVOffset = 0;
SSIZE_T dstVMultiplier = 1; SSIZE_T dstVMultiplier = 1;
WINPR_ASSERT(overlapping(pDstData, nXDst, nYDst, nDstStep, dstByte, pSrcData, nXSrc, nYSrc, WINPR_ASSERT(overlapping(pDstData, nYDst, nDstStep, pSrcData, nYSrc, nSrcStep, nHeight));
nSrcStep, srcByte, nWidth, nHeight));
if ((nWidth == 0) || (nHeight == 0)) if ((nWidth == 0) || (nHeight == 0))
return TRUE; return TRUE;
@ -851,9 +848,6 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat, UINT32 nDstStep, UINT32
DWORD SrcFormat, UINT32 nSrcStep, UINT32 nXSrc, UINT32 nYSrc, DWORD SrcFormat, UINT32 nSrcStep, UINT32 nXSrc, UINT32 nYSrc,
const gdiPalette* WINPR_RESTRICT palette, UINT32 flags) const gdiPalette* WINPR_RESTRICT palette, UINT32 flags)
{ {
const UINT32 dstByte = FreeRDPGetBytesPerPixel(DstFormat);
const UINT32 srcByte = FreeRDPGetBytesPerPixel(SrcFormat);
if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX)) if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX))
return FALSE; return FALSE;
@ -869,8 +863,7 @@ BOOL freerdp_image_copy(BYTE* pDstData, DWORD DstFormat, UINT32 nDstStep, UINT32
if (nSrcStep == 0) if (nSrcStep == 0)
nSrcStep = nWidth * FreeRDPGetBytesPerPixel(SrcFormat); nSrcStep = nWidth * FreeRDPGetBytesPerPixel(SrcFormat);
const BOOL ovl = overlapping(pDstData, nXDst, nYDst, nDstStep, dstByte, pSrcData, nXSrc, nYSrc, const BOOL ovl = overlapping(pDstData, nYDst, nDstStep, pSrcData, nYSrc, nSrcStep, nHeight);
nSrcStep, srcByte, nWidth, nHeight);
if (ovl) if (ovl)
return freerdp_image_copy_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, nWidth, return freerdp_image_copy_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, nWidth,
nHeight, pSrcData, SrcFormat, nSrcStep, nXSrc, nYSrc, nHeight, pSrcData, SrcFormat, nSrcStep, nXSrc, nYSrc,
@ -1611,9 +1604,7 @@ BOOL freerdp_image_copy_no_overlap(BYTE* WINPR_RESTRICT pDstData, DWORD DstForma
if (!prims) if (!prims)
prims = primitives_get(); prims = primitives_get();
WINPR_ASSERT(!overlapping(pDstData, nXDst, nYDst, nDstStep, FreeRDPGetBytesPerPixel(DstFormat), WINPR_ASSERT(!overlapping(pDstData, nYDst, nDstStep, pSrcData, nYSrc, nSrcStep, nHeight));
pSrcData, nXSrc, nYSrc, nSrcStep, FreeRDPGetBytesPerPixel(SrcFormat),
nWidth, nHeight));
WINPR_ASSERT(prims); WINPR_ASSERT(prims);
WINPR_ASSERT(prims->copy_no_overlap); WINPR_ASSERT(prims->copy_no_overlap);
return prims->copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, nWidth, nHeight, return prims->copy_no_overlap(pDstData, DstFormat, nDstStep, nXDst, nYDst, nWidth, nHeight,