mirrors/FreeRDP
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed H264 related warnings

Browse Source
This commit is contained in:
Armin Novak 2021-12-17 17:08:08 +01:00 committed by akallabeth
parent feb6d9ae5c
commit f3df9a520e
4 changed files with 63 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/freerdp/codec/h264.h
View File

@ -46,7 +46,7 @@ struct _H264_CONTEXT
H264_RATECONTROL_MODE RateControlMode;
UINT32 BitRate;
FLOAT FrameRate;
UINT32 FrameRate;
UINT32 QP;
UINT32 NumberOfThreads;

2
include/freerdp/server/shadow.h
View File

@ -143,7 +143,7 @@ struct rdp_shadow_server
RLGR_MODE rfxMode;
H264_RATECONTROL_MODE h264RateControlMode;
UINT32 h264BitRate;
FLOAT h264FrameRate;
UINT32 h264FrameRate;
UINT32 h264QP;
char* ipcSocket;

28
libfreerdp/codec/h264.c
View File

@ -120,23 +120,31 @@ static BOOL allocate_h264_metablock(UINT32 QP, RECTANGLE_16* rectangles,
RDPGFX_H264_METABLOCK* meta, size_t count)
{
size_t x;
if (!meta)
/* [MS-RDPEGFX] 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY */
if (!meta || (QP > UINT8_MAX))
return FALSE;
meta->regionRects = rectangles;
if (count == 0)
return TRUE;
if (count > UINT32_MAX)
return FALSE;
meta->quantQualityVals = calloc(count, sizeof(RDPGFX_H264_QUANT_QUALITY));
if (!meta->quantQualityVals || !meta->regionRects)
return FALSE;
meta->numRegionRects = count;
meta->numRegionRects = (UINT32)count;
for (x = 0; x < count; x++)
{
RDPGFX_H264_QUANT_QUALITY* cur = &meta->quantQualityVals[x];
cur->qp = QP;
cur->qualityVal = 100 - QP;
cur->qp = (UINT8)QP;
/* qpVal bit 6 and 7 are flags, so mask them out here.
* qualityVal is [0-100] so 100 - qpVal [0-64] is always in range */
cur->qualityVal = 100 - (QP & 0x3F);
}
return TRUE;
}
@ -202,10 +210,12 @@ static BOOL detect_changes(BOOL firstFrameDone, const UINT32 QP, const RECTANGLE
for (x = regionRect->left; x < regionRect->right; x += 64)
{
RECTANGLE_16 rect;
rect.left = regionRect->left + x;
rect.top = regionRect->top + y;
rect.right = MIN(regionRect->left + x + 64, regionRect->right);
rect.bottom = MIN(regionRect->top + y + 64, regionRect->bottom);
rect.left = (UINT16)MIN(UINT16_MAX, regionRect->left + x);
rect.top = (UINT16)MIN(UINT16_MAX, regionRect->top + y);
rect.right =
(UINT16)MIN(UINT16_MAX, MIN(regionRect->left + x + 64, regionRect->right));
rect.bottom =
(UINT16)MIN(UINT16_MAX, MIN(regionRect->top + y + 64, regionRect->bottom));
if (diff_tile(&rect, pYUVData, pOldYUVData, iStride))
rectangles[count++] = rect;
}
@ -450,7 +460,7 @@ static BOOL avc444_process_rects(H264_CONTEXT* h264, const BYTE* pSrcData, UINT3
pYUVDstData[0] = ppYUVDstData[0];
pYUVDstData[1] = ppYUVDstData[1];
pYUVDstData[2] = ppYUVDstData[2];
if (!yuv444_context_decode(h264->yuv, type, pYUVData, piStride, h264->height, pYUVDstData,
if (!yuv444_context_decode(h264->yuv, (BYTE)type, pYUVData, piStride, h264->height, pYUVDstData,
piDstStride, DstFormat, pDstData, nDstStep, rects, nrRects))
return FALSE;

69
libfreerdp/codec/h264_ffmpeg.c
View File

@ -79,7 +79,10 @@ struct _H264_CONTEXT_LIBAVCODEC
AVCodecContext* codecEncoderContext;
AVCodecParserContext* codecParser;
AVFrame* videoFrame;
AVPacket packet;
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
AVPacket bufferpacket;
#endif
AVPacket* packet;
#ifdef WITH_VAAPI
AVBufferRef* hwctx;
AVFrame* hwVideoFrame;
@ -163,8 +166,8 @@ static BOOL libavcodec_create_encoder(H264_CONTEXT* h264)
break;
}
sys->codecEncoderContext->width = h264->width;
sys->codecEncoderContext->height = h264->height;
sys->codecEncoderContext->width = (int)MIN(INT32_MAX, h264->width);
sys->codecEncoderContext->height = (int)MIN(INT32_MAX, h264->height);
sys->codecEncoderContext->delay = 0;
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(56, 13, 100)
sys->codecEncoderContext->framerate = (AVRational){ h264->FrameRate, 1 };
@ -191,12 +194,16 @@ static int libavcodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT3
H264_CONTEXT_LIBAVCODEC* sys = (H264_CONTEXT_LIBAVCODEC*)h264->pSystemData;
BYTE** pYUVData = h264->pYUVData;
UINT32* iStride = h264->iStride;
av_init_packet(&sys->packet);
sys->packet.data = (BYTE*)pSrcData;
sys->packet.size = SrcSize;
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
av_init_packet(sys->packet);
#else
sys->packet = av_packet_alloc();
#endif
sys->packet->data = (BYTE*)pSrcData;
sys->packet->size = (int)MIN(SrcSize, INT32_MAX);
/* avcodec_decode_video2 is deprecated with libavcodec 57.48.101 */
#if LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(57, 48, 101)
status = avcodec_send_packet(sys->codecDecoderContext, &sys->packet);
status = avcodec_send_packet(sys->codecDecoderContext, sys->packet);
if (status < 0)
{
@ -221,10 +228,10 @@ static int libavcodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT3
#ifdef WITH_VAAPI
status = avcodec_decode_video2(sys->codecDecoderContext,
sys->hwctx ? sys->hwVideoFrame : sys->videoFrame, &gotFrame,
&sys->packet);
sys->packet);
#else
status =
avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, &sys->packet);
avcodec_decode_video2(sys->codecDecoderContext, sys->videoFrame, &gotFrame, sys->packet);
#endif
#endif
@ -274,9 +281,9 @@ static int libavcodec_decompress(H264_CONTEXT* h264, const BYTE* pSrcData, UINT3
pYUVData[0] = sys->videoFrame->data[0];
pYUVData[1] = sys->videoFrame->data[1];
pYUVData[2] = sys->videoFrame->data[2];
iStride[0] = sys->videoFrame->linesize[0];
iStride[1] = sys->videoFrame->linesize[1];
iStride[2] = sys->videoFrame->linesize[2];
iStride[0] = (UINT32)MAX(0, sys->videoFrame->linesize[0]);
iStride[1] = (UINT32)MAX(0, sys->videoFrame->linesize[1]);
iStride[2] = (UINT32)MAX(0, sys->videoFrame->linesize[2]);
}
else
return -2;
@ -295,13 +302,17 @@ static int libavcodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const U
return -1;
#if LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(55, 39, 100)
av_packet_unref(&sys->packet);
av_packet_unref(sys->packet);
#else
av_free(sys->packet.data);
av_free(sys->packet->data);
#endif
av_init_packet(&sys->packet);
sys->packet.data = NULL;
sys->packet.size = 0;
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
av_init_packet(sys->packet);
#else
sys->packet = av_packet_alloc();
#endif
sys->packet->data = NULL;
sys->packet->size = 0;
sys->videoFrame->format = sys->codecEncoderContext->pix_fmt;
sys->videoFrame->width = sys->codecEncoderContext->width;
sys->videoFrame->height = sys->codecEncoderContext->height;
@ -329,7 +340,7 @@ static int libavcodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const U
return -1;
}
status = avcodec_receive_packet(sys->codecEncoderContext, &sys->packet);
status = avcodec_receive_packet(sys->codecEncoderContext, sys->packet);
if (status < 0)
{
@ -343,22 +354,22 @@ static int libavcodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const U
do
{
status = avcodec_encode_video2(sys->codecEncoderContext, &sys->packet, sys->videoFrame,
status = avcodec_encode_video2(sys->codecEncoderContext, sys->packet, sys->videoFrame,
&gotFrame);
} while ((status >= 0) && (gotFrame == 0));
#else
sys->packet.size =
sys->packet->size =
avpicture_get_size(sys->codecDecoderContext->pix_fmt, sys->codecDecoderContext->width,
sys->codecDecoderContext->height);
sys->packet.data = av_malloc(sys->packet.size);
sys->packet->data = av_malloc(sys->packet->size);
if (!sys->packet.data)
if (!sys->packet->data)
status = -1;
else
{
status = avcodec_encode_video(sys->codecDecoderContext, sys->packet.data, sys->packet.size,
sys->videoFrame);
status = avcodec_encode_video(sys->codecDecoderContext, sys->packet->data,
sys->packet->size, sys->videoFrame);
}
#endif
@ -370,8 +381,8 @@ static int libavcodec_compress(H264_CONTEXT* h264, const BYTE** pSrcYuv, const U
return -1;
}
*ppDstData = sys->packet.data;
*pDstSize = sys->packet.size;
*ppDstData = sys->packet->data;
*pDstSize = (UINT32)MAX(0, sys->packet->size);
if (!gotFrame)
{
@ -505,9 +516,13 @@ static BOOL libavcodec_init(H264_CONTEXT* h264)
goto EXCEPTION;
}
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 133, 100)
sys->packet = &sys->bufferpacket;
#endif
h264->pSystemData = (void*)sys;
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 137, 100)
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 10, 100)
avcodec_register_all();
#endif