Add openh264 encoder support.

include/freerdp/codec/h264.h

@@ -30,6 +30,7 @@ typedef BOOL (*pfnH264SubsystemInit)(H264_CONTEXT* h264);
 typedef void (*pfnH264SubsystemUninit)(H264_CONTEXT* h264);
 
 typedef int (*pfnH264SubsystemDecompress)(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize);
+typedef int (*pfnH264SubsystemCompress)(H264_CONTEXT* h264, UINT32 TargetFrameSizeInBits, BYTE** ppDstData, UINT32* pDstSize);
 
 struct _H264_CONTEXT_SUBSYSTEM
 {
@@ -37,6 +38,7 @@ struct _H264_CONTEXT_SUBSYSTEM
 	pfnH264SubsystemInit Init;
 	pfnH264SubsystemUninit Uninit;
 	pfnH264SubsystemDecompress Decompress;
+	pfnH264SubsystemCompress Compress;
 };
 typedef struct _H264_CONTEXT_SUBSYSTEM H264_CONTEXT_SUBSYSTEM;
 
@@ -58,7 +60,9 @@ struct _H264_CONTEXT
 extern "C" {
 #endif
 
-FREERDP_API int h264_compress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize, BYTE** ppDstData, UINT32* pDstSize);
+FREERDP_API int h264_compress(H264_CONTEXT* h264, BYTE* pSrcData, DWORD SrcFormat,
+		int nSrcStep, int nSrcWidth, int nSrcHeight, UINT32 TargetFrameSizeInBits,
+		BYTE** ppDstData, UINT32* pDstSize);
 
 FREERDP_API int h264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize,
 		BYTE** ppDstData, DWORD DstFormat, int nDstStep, int nDstWidth, int nDstHeight,

include/freerdp/primitives.h

@@ -172,6 +172,10 @@ typedef pstatus_t (*__YUV420ToRGB_8u_P3AC4R_t)(
 	const BYTE* pSrc[3], INT32 srcStep[3],
 	BYTE* pDst, INT32 dstStep,
 	const prim_size_t* roi);
+typedef pstatus_t (*__RGBToYUV420_8u_P3AC4R_t)(
+	const BYTE* pSrc, INT32 srcStep,
+	BYTE* pDst[3], INT32 dstStep[3],
+	const prim_size_t* roi);
 typedef pstatus_t (*__andC_32u_t)(
 	const UINT32 *pSrc,
 	UINT32 val,
@@ -219,6 +223,7 @@ typedef struct
 	__YCoCgToRGB_8u_AC4R_t YCoCgToRGB_8u_AC4R;
 	__RGB565ToARGB_16u32u_C3C4_t RGB565ToARGB_16u32u_C3C4;
 	__YUV420ToRGB_8u_P3AC4R_t YUV420ToRGB_8u_P3AC4R;
+	__RGBToYUV420_8u_P3AC4R_t RGBToYUV420_8u_P3AC4R;
 } primitives_t;
 
 #ifdef __cplusplus

libfreerdp/codec/h264.c

@@ -70,6 +70,8 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_dummy =
 struct _H264_CONTEXT_OPENH264
 {
 	ISVCDecoder* pDecoder;
+	ISVCEncoder* pEncoder;
+	SEncParamBase EncParamBase;
 };
 typedef struct _H264_CONTEXT_OPENH264 H264_CONTEXT_OPENH264;
 
@@ -148,6 +150,86 @@ static int openh264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSiz
 	return 1;
 }
 
+static int openh264_compress(H264_CONTEXT* h264, UINT32 TargetFrameSizeInBits, BYTE** ppDstData, UINT32* pDstSize)
+{
+	int i, j;
+	int status;
+	SFrameBSInfo info;
+	SSourcePicture pic;
+	SBitrateInfo bitrate;
+	H264_CONTEXT_OPENH264* sys = (H264_CONTEXT_OPENH264*) h264->pSystemData;
+
+	if (!sys->pEncoder)
+		return -1;
+
+	if (!h264->pYUVData[0] || !h264->pYUVData[1] || !h264->pYUVData[2])
+		return -1;
+
+	if (sys->EncParamBase.iPicWidth != h264->width ||
+		sys->EncParamBase.iPicHeight != h264->height)
+	{
+		sys->EncParamBase.iUsageType = SCREEN_CONTENT_REAL_TIME;
+		sys->EncParamBase.iPicWidth = h264->width;
+		sys->EncParamBase.iPicHeight = h264->height;
+		sys->EncParamBase.iTargetBitrate = TargetFrameSizeInBits * 30;
+		sys->EncParamBase.iRCMode = RC_BITRATE_MODE;
+		sys->EncParamBase.fMaxFrameRate = 30.0f;
+
+		status = (*sys->pEncoder)->Initialize(sys->pEncoder, &sys->EncParamBase);
+
+		if (status < 0)
+		{
+			WLog_ERR(TAG, "Failed to initialize OpenH264 encoder (status=%ld)", status);
+			return status;
+		}
+	}
+	else if (sys->EncParamBase.iTargetBitrate != TargetFrameSizeInBits * 30)
+	{
+		sys->EncParamBase.iTargetBitrate = TargetFrameSizeInBits * 30;
+		bitrate.iLayer = SPATIAL_LAYER_ALL;
+		bitrate.iBitrate = TargetFrameSizeInBits * 30;
+
+		status = (*sys->pEncoder)->SetOption(sys->pEncoder, ENCODER_OPTION_BITRATE, &bitrate);
+
+		if (status < 0)
+		{
+			WLog_ERR(TAG, "Failed to set encoder bitrate (status=%ld)", status);
+			return status;
+		}
+	}
+
+	memset(&info, 0, sizeof(SFrameBSInfo));
+	memset(&pic, 0, sizeof(SSourcePicture));
+	pic.iPicWidth = h264->width;
+	pic.iPicHeight = h264->height;
+	pic.iColorFormat = videoFormatI420;
+	pic.iStride[0] = h264->iStride[0];
+	pic.iStride[1] = h264->iStride[1];
+	pic.iStride[2] = h264->iStride[2];
+	pic.pData[0] = h264->pYUVData[0];
+	pic.pData[1] = h264->pYUVData[1];
+	pic.pData[2] = h264->pYUVData[2];
+
+	status = (*sys->pEncoder)->EncodeFrame(sys->pEncoder, &pic, &info);
+
+	if (status < 0)
+	{
+		WLog_ERR(TAG, "Failed to encode frame (status=%ld)", status);
+		return status;
+	}
+	*ppDstData = info.sLayerInfo[0].pBsBuf;
+	*pDstSize = 0;
+	for (i = 0; i < info.iLayerNum; i++)
+	{
+		for (j = 0; j < info.sLayerInfo[i].iNalCount; j++)
+		{
+			*pDstSize += info.sLayerInfo[i].pNalLengthInByte[j];
+		}
+	}
+
+	return 1;
+}
+
 static void openh264_uninit(H264_CONTEXT* h264)
 {
 	H264_CONTEXT_OPENH264* sys = (H264_CONTEXT_OPENH264*) h264->pSystemData;
@@ -161,6 +243,13 @@ static void openh264_uninit(H264_CONTEXT* h264)
 			sys->pDecoder = NULL;
 		}
 
+		if (sys->pEncoder)
+		{
+			(*sys->pEncoder)->Uninitialize(sys->pEncoder);
+			WelsDestroySVCEncoder(sys->pEncoder);
+			sys->pEncoder = NULL;
+		}
+
 		free(sys);
 		h264->pSystemData = NULL;
 	}
@@ -184,55 +273,68 @@ static BOOL openh264_init(H264_CONTEXT* h264)
 
 	h264->pSystemData = (void*) sys;
 
-	WelsCreateDecoder(&sys->pDecoder);
+	if (h264->Compressor)
-
-	if (!sys->pDecoder)
 	{
-		WLog_ERR(TAG, "Failed to create OpenH264 decoder");
+		WelsCreateSVCEncoder(&sys->pEncoder);
-		goto EXCEPTION;
-	}
 
-	ZeroMemory(&sDecParam, sizeof(sDecParam));
+		if (!sys->pEncoder)
-	sDecParam.eOutputColorFormat  = videoFormatI420;
-	sDecParam.eEcActiveIdc = ERROR_CON_FRAME_COPY;
-	sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT;
-
-	status = (*sys->pDecoder)->Initialize(sys->pDecoder, &sDecParam);
-
-	if (status != 0)
-	{
-		WLog_ERR(TAG, "Failed to initialize OpenH264 decoder (status=%ld)", status);
-		goto EXCEPTION;
-	}
-
-	status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_DATAFORMAT, &videoFormat);
-
-	if (status != 0)
-	{
-		WLog_ERR(TAG, "Failed to set data format option on OpenH264 decoder (status=%ld)", status);
-	}
-
-	if (g_openh264_trace_enabled)
-	{
-		status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_LEVEL, &traceLevel);
-
-		if (status != 0)
 		{
-			WLog_ERR(TAG, "Failed to set trace level option on OpenH264 decoder (status=%ld)", status);
+			WLog_ERR(TAG, "Failed to create OpenH264 encoder");
+			goto EXCEPTION;
+		}
+	}
+	else
+	{
+		WelsCreateDecoder(&sys->pDecoder);
+
+		if (!sys->pDecoder)
+		{
+			WLog_ERR(TAG, "Failed to create OpenH264 decoder");
+			goto EXCEPTION;
 		}
 
-		status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK, &traceCallback);
+		ZeroMemory(&sDecParam, sizeof(sDecParam));
+		sDecParam.eOutputColorFormat  = videoFormatI420;
+		sDecParam.eEcActiveIdc = ERROR_CON_FRAME_COPY;
+		sDecParam.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_DEFAULT;
+
+		status = (*sys->pDecoder)->Initialize(sys->pDecoder, &sDecParam);
 
 		if (status != 0)
 		{
-			WLog_ERR(TAG, "Failed to set trace callback option on OpenH264 decoder (status=%ld)", status);
+			WLog_ERR(TAG, "Failed to initialize OpenH264 decoder (status=%ld)", status);
+			goto EXCEPTION;
 		}
 
-		status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK_CONTEXT, &h264);
+		status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_DATAFORMAT, &videoFormat);
 
 		if (status != 0)
 		{
-			WLog_ERR(TAG, "Failed to set trace callback context option on OpenH264 decoder (status=%ld)", status);
+			WLog_ERR(TAG, "Failed to set data format option on OpenH264 decoder (status=%ld)", status);
+		}
+
+		if (g_openh264_trace_enabled)
+		{
+			status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_LEVEL, &traceLevel);
+
+			if (status != 0)
+			{
+				WLog_ERR(TAG, "Failed to set trace level option on OpenH264 decoder (status=%ld)", status);
+			}
+
+			status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK, &traceCallback);
+
+			if (status != 0)
+			{
+				WLog_ERR(TAG, "Failed to set trace callback option on OpenH264 decoder (status=%ld)", status);
+			}
+
+			status = (*sys->pDecoder)->SetOption(sys->pDecoder, DECODER_OPTION_TRACE_CALLBACK_CONTEXT, &h264);
+
+			if (status != 0)
+			{
+				WLog_ERR(TAG, "Failed to set trace callback context option on OpenH264 decoder (status=%ld)", status);
+			}
 		}
 	}
 
@@ -249,7 +351,8 @@ static H264_CONTEXT_SUBSYSTEM g_Subsystem_OpenH264 =
 	"OpenH264",
 	openh264_init,
 	openh264_uninit,
-	openh264_decompress
+	openh264_decompress,
+	openh264_compress
 };
 
 #endif
@@ -494,9 +597,45 @@ int h264_decompress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize,
 	return 1;
 }
 
-int h264_compress(H264_CONTEXT* h264, BYTE* pSrcData, UINT32 SrcSize, BYTE** ppDstData, UINT32* pDstSize)
+int h264_compress(H264_CONTEXT* h264, BYTE* pSrcData, DWORD SrcFormat,
+		int nSrcStep, int nSrcWidth, int nSrcHeight, UINT32 TargetFrameSizeInBits,
+		BYTE** ppDstData, UINT32* pDstSize)
 {
-	return 1;
+	int status;
+	prim_size_t roi;
+	int nWidth, nHeight;
+	primitives_t *prims = primitives_get();
+
+	if (!h264)
+		return -1;
+	if (!h264->subsystem->Compress)
+		return -1;
+
+	nWidth = (nSrcWidth + 1) & ~1;
+	nHeight = (nSrcHeight + 1) & ~1;
+	h264->pYUVData[0] = (BYTE*) malloc(nWidth * nHeight);
+	h264->iStride[0] = nWidth;
+	h264->pYUVData[1] = (BYTE*) malloc(nWidth * nHeight / 4);
+	h264->iStride[1] = nWidth / 2;
+	h264->pYUVData[2] = (BYTE*) malloc(nWidth * nHeight / 4);
+	h264->iStride[2] = nWidth / 2;
+	h264->width = nWidth;
+	h264->height = nHeight;
+	roi.width = nSrcWidth;
+	roi.height = nSrcHeight;
+
+	prims->RGBToYUV420_8u_P3AC4R(pSrcData, nSrcStep, h264->pYUVData, h264->iStride, &roi);
+
+	status = h264->subsystem->Compress(h264, TargetFrameSizeInBits, ppDstData, pDstSize);
+
+	free(h264->pYUVData[0]);
+	free(h264->pYUVData[1]);
+	free(h264->pYUVData[2]);
+	h264->pYUVData[0] = NULL;
+	h264->pYUVData[1] = NULL;
+	h264->pYUVData[2] = NULL;
+
+	return status;
 }
 
 BOOL h264_context_init(H264_CONTEXT* h264)

libfreerdp/primitives/prim_YUV.c

@@ -265,9 +265,111 @@ pstatus_t general_YUV420ToRGB_8u_P3AC4R(const BYTE* pSrc[3], int srcStep[3],
 	return PRIMITIVES_SUCCESS;
 }
 
+pstatus_t general_RGBToYUV420_8u_P3AC4R(const BYTE* pSrc, INT32 srcStep,
+		BYTE* pDst[3], INT32 dstStep[3], const prim_size_t* roi)
+{
+	int x, y;
+	int dstPad[3];
+	int halfWidth;
+	int halfHeight;
+	BYTE* pY;
+	BYTE* pU;
+	BYTE* pV;
+	int Y, U, V;
+	int R, G, B;
+	int Ra, Ga, Ba;
+	const BYTE* pRGB;
+	int nWidth, nHeight;
+
+	pU = pDst[1];
+	pV = pDst[2];
+
+	nWidth = (roi->width + 1) & ~0x0001;
+	nHeight = (roi->height + 1) & ~0x0001;
+
+	halfWidth = nWidth / 2;
+	halfHeight = nHeight / 2;
+
+	dstPad[0] = (dstStep[0] - nWidth);
+	dstPad[1] = (dstStep[1] - halfWidth);
+	dstPad[2] = (dstStep[2] - halfWidth);
+
+	for (y = 0; y < halfHeight; y++)
+	{
+		for (x = 0; x < halfWidth; x++)
+		{
+			/* 1st pixel */
+			pRGB = pSrc + y * 2 * srcStep + x * 2 * 4;
+			pY = pDst[0] + y * 2 * dstStep[0] + x * 2;
+			Ba = B = pRGB[0];
+			Ga = G = pRGB[1];
+			Ra = R = pRGB[2];
+			Y = (54 * R + 183 * G + 18 * B) >> 8;
+			pY[0] = (BYTE) Y;
+
+			if (x * 2 + 1 < roi->width)
+			{
+				/* 2nd pixel */
+				Ba += B = pRGB[4];
+				Ga += G = pRGB[5];
+				Ra += R = pRGB[6];
+				Y = (54 * R + 183 * G + 18 * B) >> 8;
+				pY[1] = (BYTE) Y;
+			}
+
+			if (y * 2 + 1 < roi->height)
+			{
+				/* 3rd pixel */
+				pRGB += srcStep;
+				pY += dstStep[0];
+				Ba += B = pRGB[0];
+				Ga += G = pRGB[1];
+				Ra += R = pRGB[2];
+				Y = (54 * R + 183 * G + 18 * B) >> 8;
+				pY[0] = (BYTE) Y;
+
+				if (x * 2 + 1 < roi->width)
+				{
+					/* 4th pixel */
+					Ba += B = pRGB[4];
+					Ga += G = pRGB[5];
+					Ra += R = pRGB[6];
+					Y = (54 * R + 183 * G + 18 * B) >> 8;
+					pY[1] = (BYTE) Y;
+				}
+			}
+
+			/* U */
+			Ba >>= 2;
+			Ga >>= 2;
+			Ra >>= 2;
+			U = ((-29 * Ra - 99 * Ga + 128 * Ba) >> 8) + 128;
+			if (U < 0)
+				U = 0;
+			else if (U > 255)
+				U = 255;
+			*pU++ = (BYTE) U;
+
+			/* V */
+			V = ((128 * Ra - 116 * Ga - 12 * Ba) >> 8) + 128;
+			if (V < 0)
+				V = 0;
+			else if (V > 255)
+				V = 255;
+			*pV++ = (BYTE) V;
+		}
+
+		pU += dstPad[1];
+		pV += dstPad[2];
+	}
+
+	return PRIMITIVES_SUCCESS;
+}
+
 void primitives_init_YUV(primitives_t* prims)
 {
 	prims->YUV420ToRGB_8u_P3AC4R = general_YUV420ToRGB_8u_P3AC4R;
+	prims->RGBToYUV420_8u_P3AC4R = general_RGBToYUV420_8u_P3AC4R;
 	
 	primitives_init_YUV_opt(prims);
 }