diff --git a/include/freerdp/codec/nsc.h b/include/freerdp/codec/nsc.h index cb16570db..c1b663841 100644 --- a/include/freerdp/codec/nsc.h +++ b/include/freerdp/codec/nsc.h @@ -30,86 +30,40 @@ #include #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif -struct _NSC_MESSAGE -{ - UINT32 x; - UINT32 y; - UINT32 width; - UINT32 height; - const BYTE* data; - UINT32 scanline; - BYTE* PlaneBuffer; - UINT32 MaxPlaneSize; - BYTE* PlaneBuffers[5]; - UINT32 OrgByteCount[4]; + typedef enum + { + NSC_COLOR_LOSS_LEVEL, + NSC_ALLOW_SUBSAMPLING, + NSC_DYNAMIC_COLOR_FIDELITY, + NSC_COLOR_FORMAT + } NSC_PARAMETER; - UINT32 LumaPlaneByteCount; - UINT32 OrangeChromaPlaneByteCount; - UINT32 GreenChromaPlaneByteCount; - UINT32 AlphaPlaneByteCount; - UINT32 ColorLossLevel; - UINT32 ChromaSubsamplingLevel; -}; -typedef struct _NSC_MESSAGE NSC_MESSAGE; + typedef struct _NSC_CONTEXT NSC_CONTEXT; -typedef struct _NSC_CONTEXT_PRIV NSC_CONTEXT_PRIV; + FREERDP_API WINPR_DEPRECATED(BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, + UINT32 pixel_format)); + FREERDP_API BOOL nsc_context_set_parameters(NSC_CONTEXT* context, NSC_PARAMETER what, + UINT32 value); -typedef struct _NSC_CONTEXT NSC_CONTEXT; + FREERDP_API BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp, UINT32 width, + UINT32 height, const BYTE* data, UINT32 length, + BYTE* pDstData, UINT32 DstFormat, UINT32 nDstStride, + UINT32 nXDst, UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, + UINT32 flip); + FREERDP_API BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, const BYTE* bmpdata, + UINT32 width, UINT32 height, UINT32 rowstride); + FREERDP_API BOOL nsc_decompose_message(NSC_CONTEXT* context, wStream* s, BYTE* bmpdata, + UINT32 x, UINT32 y, UINT32 width, UINT32 height, + UINT32 rowstride, UINT32 format, UINT32 flip); -struct _NSC_CONTEXT -{ - UINT32 OrgByteCount[4]; - UINT32 format; - UINT16 width; - UINT16 height; - BYTE* BitmapData; - UINT32 BitmapDataLength; + FREERDP_API BOOL nsc_context_reset(NSC_CONTEXT* context, UINT32 width, UINT32 height); - BYTE* Planes; - UINT32 PlaneByteCount[4]; - UINT32 ColorLossLevel; - UINT32 ChromaSubsamplingLevel; - BOOL DynamicColorFidelity; - - /* color palette allocated by the application */ - const BYTE* palette; - - BOOL (*decode)(NSC_CONTEXT* context); - BOOL (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData, - UINT32 rowstride); - - NSC_CONTEXT_PRIV* priv; -}; - -FREERDP_API BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, - UINT32 pixel_format); -FREERDP_API BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp, - UINT32 width, UINT32 height, - const BYTE* data, UINT32 length, - BYTE* pDstData, UINT32 DstFormat, - UINT32 nDstStride, UINT32 nXDst, UINT32 nYDst, - UINT32 nWidth, UINT32 nHeight, UINT32 flip); -FREERDP_API BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, - const BYTE* bmpdata, - UINT32 width, UINT32 height, UINT32 rowstride); - -FREERDP_API NSC_MESSAGE* nsc_encode_messages(NSC_CONTEXT* context, - const BYTE* data, - UINT32 x, UINT32 y, - UINT32 width, UINT32 height, UINT32 scanline, - UINT32* numMessages, UINT32 maxDataSize); -FREERDP_API BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, - NSC_MESSAGE* message); -FREERDP_API void nsc_message_free(NSC_CONTEXT* context, NSC_MESSAGE* message); - -FREERDP_API BOOL nsc_context_reset(NSC_CONTEXT* context, UINT32 width, - UINT32 height); - -FREERDP_API NSC_CONTEXT* nsc_context_new(void); -FREERDP_API void nsc_context_free(NSC_CONTEXT* context); + FREERDP_API NSC_CONTEXT* nsc_context_new(void); + FREERDP_API void nsc_context_free(NSC_CONTEXT* context); #ifdef __cplusplus } diff --git a/libfreerdp/codec/nsc.c b/libfreerdp/codec/nsc.c index bad0aec1c..9b39af196 100644 --- a/libfreerdp/codec/nsc.c +++ b/libfreerdp/codec/nsc.c @@ -317,8 +317,11 @@ BOOL nsc_context_reset(NSC_CONTEXT* context, UINT32 width, UINT32 height) if (!context) return FALSE; - context->width = width; - context->height = height; + if ((width > UINT16_MAX) || (height > UINT16_MAX)) + return FALSE; + + context->width = (UINT16)width; + context->height = (UINT16)height; return TRUE; } @@ -340,10 +343,6 @@ NSC_CONTEXT* nsc_context_new(void) context->BitmapData = NULL; context->decode = nsc_decode; context->encode = nsc_encode; - context->priv->PlanePool = BufferPool_New(TRUE, 0, 16); - - if (!context->priv->PlanePool) - goto error; PROFILER_CREATE(context->priv->prof_nsc_rle_decompress_data, "nsc_rle_decompress_data") @@ -374,7 +373,6 @@ void nsc_context_free(NSC_CONTEXT* context) for (i = 0; i < 4; i++) free(context->priv->PlaneBuffers[i]); - BufferPool_Free(context->priv->PlanePool); nsc_profiler_print(context->priv); PROFILER_FREE(context->priv->prof_nsc_rle_decompress_data) PROFILER_FREE(context->priv->prof_nsc_decode) @@ -388,11 +386,33 @@ void nsc_context_free(NSC_CONTEXT* context) } BOOL nsc_context_set_pixel_format(NSC_CONTEXT* context, UINT32 pixel_format) +{ + return nsc_context_set_parameters(context, NSC_COLOR_FORMAT, pixel_format); +} + +BOOL nsc_context_set_parameters(NSC_CONTEXT* context, NSC_PARAMETER what, + UINT32 value) { if (!context) return FALSE; - context->format = pixel_format; + switch(what) + { + case NSC_COLOR_LOSS_LEVEL: + context->ColorLossLevel = value; + break; + case NSC_ALLOW_SUBSAMPLING: + context->ChromaSubsamplingLevel = value; + break; + case NSC_DYNAMIC_COLOR_FIDELITY: + context->DynamicColorFidelity = value != 0; + break; + case NSC_COLOR_FORMAT: + context->format = value; + break; + default: + return FALSE; + } return TRUE; } @@ -406,6 +426,9 @@ BOOL nsc_process_message(NSC_CONTEXT* context, UINT16 bpp, { wStream* s; BOOL ret; + if (!context || !data || !pDstData) + return FALSE; + s = Stream_New((BYTE*)data, length); if (!s) diff --git a/libfreerdp/codec/nsc_encode.c b/libfreerdp/codec/nsc_encode.c index 2155260dd..54ee91ca2 100644 --- a/libfreerdp/codec/nsc_encode.c +++ b/libfreerdp/codec/nsc_encode.c @@ -35,6 +35,31 @@ #include "nsc_types.h" #include "nsc_encode.h" +struct _NSC_MESSAGE +{ + UINT32 x; + UINT32 y; + UINT32 width; + UINT32 height; + const BYTE* data; + UINT32 scanline; + BYTE* PlaneBuffer; + UINT32 MaxPlaneSize; + BYTE* PlaneBuffers[5]; + UINT32 OrgByteCount[4]; + + UINT32 LumaPlaneByteCount; + UINT32 OrangeChromaPlaneByteCount; + UINT32 GreenChromaPlaneByteCount; + UINT32 AlphaPlaneByteCount; + UINT8 ColorLossLevel; + UINT8 ChromaSubsamplingLevel; +}; +typedef struct _NSC_MESSAGE NSC_MESSAGE; + +static BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, + const NSC_MESSAGE* message); + static BOOL nsc_context_initialize_encode(NSC_CONTEXT* context) { int i; @@ -110,10 +135,12 @@ static BOOL nsc_encode_argb_to_aycocg(NSC_CONTEXT* context, const BYTE* data, rw = (context->ChromaSubsamplingLevel ? tempWidth : context->width); ccl = context->ColorLossLevel; - if (context->priv->PlaneBuffersLength < rw * scanline) + /* Internal buffer must conatin height * width pixels (aligned) */ + if (context->priv->PlaneBuffersLength < context->height * rw) return FALSE; - if (rw < scanline * 2) + /* Input stride must contain enough data for width */ + if (rw * GetBytesPerPixel(context->format) < scanline) return FALSE; for (y = 0; y < context->height; y++) @@ -427,136 +454,7 @@ UINT32 nsc_compute_byte_count(NSC_CONTEXT* context, UINT32* ByteCount, return maxPlaneSize; } -NSC_MESSAGE* nsc_encode_messages(NSC_CONTEXT* context, const BYTE* data, - UINT32 x, UINT32 y, UINT32 width, UINT32 height, - UINT32 scanline, UINT32* numMessages, - UINT32 maxDataSize) -{ - UINT32 i, j, k; - UINT32 dataOffset; - UINT32 rows, cols; - UINT32 BytesPerPixel; - UINT32 MaxRegionWidth; - UINT32 MaxRegionHeight; - UINT32 ByteCount[4]; - UINT32 MaxPlaneSize; - UINT32 MaxMessageSize; - NSC_MESSAGE* messages; - UINT32 PaddedMaxPlaneSize; - - if (!context || !data || !numMessages) - return NULL; - - if (maxDataSize < 1024) - return NULL; - - k = 0; - MaxRegionWidth = 64 * 4; - MaxRegionHeight = 64 * 2; - BytesPerPixel = GetBytesPerPixel(context->format); - rows = (width + (MaxRegionWidth - (width % MaxRegionWidth))) / MaxRegionWidth; - cols = (height + (MaxRegionHeight - (height % MaxRegionHeight))) / - MaxRegionHeight; - *numMessages = rows * cols; - MaxPlaneSize = nsc_compute_byte_count(context, (UINT32*) ByteCount, width, - height); - MaxMessageSize = ByteCount[0] + ByteCount[1] + ByteCount[2] + ByteCount[3] + 20; - maxDataSize -= 1024; /* reserve enough space for headers */ - if (maxDataSize < (*numMessages) * sizeof(NSC_MESSAGE)) - return NULL; - - messages = (NSC_MESSAGE*) calloc(*numMessages, sizeof(NSC_MESSAGE)); - - if (!messages) - return NULL; - - for (i = 0; i < rows; i++) - { - for (j = 0; j < cols; j++) - { - messages[k].x = x + (i * MaxRegionWidth); - messages[k].y = y + (j * MaxRegionHeight); - messages[k].width = (i < (rows - 1)) ? MaxRegionWidth : width - - (i * MaxRegionWidth); - messages[k].height = (j < (cols - 1)) ? MaxRegionHeight : height - - (j * MaxRegionHeight); - messages[k].data = data; - messages[k].scanline = scanline; - messages[k].MaxPlaneSize = nsc_compute_byte_count(context, - (UINT32*) messages[k].OrgByteCount, messages[k].width, messages[k].height); - k++; - } - } - - *numMessages = k; - - for (i = 0; i < *numMessages; i++) - { - PaddedMaxPlaneSize = messages[i].MaxPlaneSize + 32; - messages[i].PlaneBuffer = (BYTE*) BufferPool_Take(context->priv->PlanePool, - PaddedMaxPlaneSize * 5); - - if (!messages[i].PlaneBuffer) - goto fail; - - messages[i].PlaneBuffers[0] = (BYTE*) & - (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 0) + 16]); - messages[i].PlaneBuffers[1] = (BYTE*) & - (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 1) + 16]); - messages[i].PlaneBuffers[2] = (BYTE*) & - (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 2) + 16]); - messages[i].PlaneBuffers[3] = (BYTE*) & - (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 3) + 16]); - messages[i].PlaneBuffers[4] = (BYTE*) & - (messages[i].PlaneBuffer[(PaddedMaxPlaneSize * 4) + 16]); - } - - for (i = 0; i < *numMessages; i++) - { - context->width = messages[i].width; - context->height = messages[i].height; - context->OrgByteCount[0] = messages[i].OrgByteCount[0]; - context->OrgByteCount[1] = messages[i].OrgByteCount[1]; - context->OrgByteCount[2] = messages[i].OrgByteCount[2]; - context->OrgByteCount[3] = messages[i].OrgByteCount[3]; - context->priv->PlaneBuffersLength = messages[i].MaxPlaneSize; - context->priv->PlaneBuffers[0] = messages[i].PlaneBuffers[0]; - context->priv->PlaneBuffers[1] = messages[i].PlaneBuffers[1]; - context->priv->PlaneBuffers[2] = messages[i].PlaneBuffers[2]; - context->priv->PlaneBuffers[3] = messages[i].PlaneBuffers[3]; - context->priv->PlaneBuffers[4] = messages[i].PlaneBuffers[4]; - dataOffset = (messages[i].y * messages[i].scanline) + (messages[i].x * - BytesPerPixel); - PROFILER_ENTER(context->priv->prof_nsc_encode) - context->encode(context, &data[dataOffset], scanline); - PROFILER_EXIT(context->priv->prof_nsc_encode) - PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data) - nsc_rle_compress_data(context); - PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data) - messages[i].LumaPlaneByteCount = context->PlaneByteCount[0]; - messages[i].OrangeChromaPlaneByteCount = context->PlaneByteCount[1]; - messages[i].GreenChromaPlaneByteCount = context->PlaneByteCount[2]; - messages[i].AlphaPlaneByteCount = context->PlaneByteCount[3]; - messages[i].ColorLossLevel = context->ColorLossLevel; - messages[i].ChromaSubsamplingLevel = context->ChromaSubsamplingLevel; - } - - context->priv->PlaneBuffers[0] = NULL; - context->priv->PlaneBuffers[1] = NULL; - context->priv->PlaneBuffers[2] = NULL; - context->priv->PlaneBuffers[3] = NULL; - context->priv->PlaneBuffers[4] = NULL; - return messages; -fail: - - for (i = 0; i < *numMessages; i++) - BufferPool_Return(context->priv->PlanePool, messages[i].PlaneBuffer); - - free(messages); - return NULL; -} - -BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, NSC_MESSAGE* message) +BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, const NSC_MESSAGE* message) { UINT32 totalPlaneByteCount; totalPlaneByteCount = message->LumaPlaneByteCount + @@ -564,7 +462,7 @@ BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, NSC_MESSAGE* message) message->GreenChromaPlaneByteCount + message->AlphaPlaneByteCount; if (!Stream_EnsureRemainingCapacity(s, 20 + totalPlaneByteCount)) - return -1; + return FALSE; Stream_Write_UINT32(s, message->LumaPlaneByteCount); /* LumaPlaneByteCount (4 bytes) */ @@ -598,16 +496,15 @@ BOOL nsc_write_message(NSC_CONTEXT* context, wStream* s, NSC_MESSAGE* message) return TRUE; } -void nsc_message_free(NSC_CONTEXT* context, NSC_MESSAGE* message) -{ - BufferPool_Return(context->priv->PlanePool, message->PlaneBuffer); -} - BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, const BYTE* data, UINT32 width, UINT32 height, UINT32 scanline) { - NSC_MESSAGE s_message = { 0 }; - NSC_MESSAGE* message = &s_message; + BOOL rc; + NSC_MESSAGE message = { 0 }; + + if (!context || !s || !data) + return FALSE; + context->width = width; context->height = height; @@ -616,21 +513,41 @@ BOOL nsc_compose_message(NSC_CONTEXT* context, wStream* s, const BYTE* data, /* ARGB to AYCoCg conversion, chroma subsampling and colorloss reduction */ PROFILER_ENTER(context->priv->prof_nsc_encode) - context->encode(context, data, scanline); + rc = context->encode(context, data, scanline); PROFILER_EXIT(context->priv->prof_nsc_encode) + if (!rc) + return FALSE; + /* RLE encode */ PROFILER_ENTER(context->priv->prof_nsc_rle_compress_data) nsc_rle_compress_data(context); PROFILER_EXIT(context->priv->prof_nsc_rle_compress_data) - message->PlaneBuffers[0] = context->priv->PlaneBuffers[0]; - message->PlaneBuffers[1] = context->priv->PlaneBuffers[1]; - message->PlaneBuffers[2] = context->priv->PlaneBuffers[2]; - message->PlaneBuffers[3] = context->priv->PlaneBuffers[3]; - message->LumaPlaneByteCount = context->PlaneByteCount[0]; - message->OrangeChromaPlaneByteCount = context->PlaneByteCount[1]; - message->GreenChromaPlaneByteCount = context->PlaneByteCount[2]; - message->AlphaPlaneByteCount = context->PlaneByteCount[3]; - message->ColorLossLevel = context->ColorLossLevel; - message->ChromaSubsamplingLevel = context->ChromaSubsamplingLevel; - return nsc_write_message(context, s, message); + message.PlaneBuffers[0] = context->priv->PlaneBuffers[0]; + message.PlaneBuffers[1] = context->priv->PlaneBuffers[1]; + message.PlaneBuffers[2] = context->priv->PlaneBuffers[2]; + message.PlaneBuffers[3] = context->priv->PlaneBuffers[3]; + message.LumaPlaneByteCount = context->PlaneByteCount[0]; + message.OrangeChromaPlaneByteCount = context->PlaneByteCount[1]; + message.GreenChromaPlaneByteCount = context->PlaneByteCount[2]; + message.AlphaPlaneByteCount = context->PlaneByteCount[3]; + message.ColorLossLevel = context->ColorLossLevel; + message.ChromaSubsamplingLevel = context->ChromaSubsamplingLevel; + return nsc_write_message(context, s, &message); +} + +BOOL nsc_decompose_message(NSC_CONTEXT* context, wStream* s, BYTE* bmpdata, + UINT32 x, UINT32 y, UINT32 width, UINT32 height, + UINT32 rowstride, UINT32 format, UINT32 flip) +{ + size_t size = Stream_GetRemainingLength(s); + if (size > UINT32_MAX) + return FALSE; + + if (!nsc_process_message(context, (UINT16)GetBitsPerPixel(context->format), + width, height, Stream_Pointer(s), + (UINT32)size, bmpdata, format, + rowstride, x, y, width, height, flip)) + return FALSE; + Stream_Seek(s, size); + return TRUE; } diff --git a/libfreerdp/codec/nsc_types.h b/libfreerdp/codec/nsc_types.h index 37c0e2fe1..b9b997c55 100644 --- a/libfreerdp/codec/nsc_types.h +++ b/libfreerdp/codec/nsc_types.h @@ -29,8 +29,8 @@ #include #include - #include +#include #define ROUND_UP_TO(_b, _n) (_b + ((~(_b & (_n-1)) + 0x1) & (_n-1))) #define MINMAX(_v,_l,_h) ((_v) < (_l) ? (_l) : ((_v) > (_h) ? (_h) : (_v))) @@ -39,10 +39,8 @@ struct _NSC_CONTEXT_PRIV { wLog* log; - wBufferPool* PlanePool; - - BYTE* PlaneBuffers[5]; /* Decompressed Plane Buffers in the respective order */ - UINT32 PlaneBuffersLength; /* Lengths of each plane buffer */ + BYTE* PlaneBuffers[5]; /* Decompressed Plane Buffers in the respective order */ + UINT32 PlaneBuffersLength; /* Lengths of each plane buffer */ /* profilers */ PROFILER_DEFINE(prof_nsc_rle_decompress_data) @@ -51,4 +49,30 @@ struct _NSC_CONTEXT_PRIV PROFILER_DEFINE(prof_nsc_encode) }; + typedef struct _NSC_CONTEXT_PRIV NSC_CONTEXT_PRIV; + + struct _NSC_CONTEXT + { + UINT32 OrgByteCount[4]; + UINT32 format; + UINT16 width; + UINT16 height; + BYTE* BitmapData; + UINT32 BitmapDataLength; + + BYTE* Planes; + UINT32 PlaneByteCount[4]; + UINT32 ColorLossLevel; + UINT32 ChromaSubsamplingLevel; + BOOL DynamicColorFidelity; + + /* color palette allocated by the application */ + const BYTE* palette; + + BOOL (*decode)(NSC_CONTEXT* context); + BOOL (*encode)(NSC_CONTEXT* context, const BYTE* BitmapData, UINT32 rowstride); + + NSC_CONTEXT_PRIV* priv; + }; + #endif /* FREERDP_LIB_CODEC_NSC_TYPES_H */ diff --git a/server/shadow/shadow_encoder.c b/server/shadow/shadow_encoder.c index b7e422ea8..43d2709ca 100644 --- a/server/shadow/shadow_encoder.c +++ b/server/shadow/shadow_encoder.c @@ -161,11 +161,14 @@ static int shadow_encoder_init_nsc(rdpShadowEncoder* encoder) if (!nsc_context_reset(encoder->nsc, encoder->width, encoder->height)) goto fail; - encoder->nsc->ColorLossLevel = settings->NSCodecColorLossLevel; - encoder->nsc->ChromaSubsamplingLevel = settings->NSCodecAllowSubsampling ? 1 : - 0; - encoder->nsc->DynamicColorFidelity = settings->NSCodecAllowDynamicColorFidelity; - nsc_context_set_pixel_format(encoder->nsc, PIXEL_FORMAT_BGRX32); + if (!nsc_context_set_parameters(encoder->nsc, NSC_COLOR_LOSS_LEVEL, settings->NSCodecColorLossLevel)) + goto fail; + if (!nsc_context_set_parameters(encoder->nsc, NSC_ALLOW_SUBSAMPLING, settings->NSCodecAllowSubsampling)) + goto fail; + if (!nsc_context_set_parameters(encoder->nsc, NSC_DYNAMIC_COLOR_FIDELITY, settings->NSCodecAllowDynamicColorFidelity)) + goto fail; + if (!nsc_context_set_parameters(encoder->nsc, NSC_COLOR_FORMAT, PIXEL_FORMAT_BGRX32)) + goto fail; encoder->codecs |= FREERDP_CODEC_NSCODEC; return 1; fail: