mirrors/xrdp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add openh264 support

Browse Source
This commit is contained in:
Jay Sorg 2024-11-09 23:17:29 -08:00
parent 9c0a0a8c16
commit 8743ac19fb
7 changed files with 447 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
configure.ac
View File

@ -175,6 +175,10 @@ AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
[Use x264 library (default: no)]),
[], [enable_x264=no])
AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
AC_ARG_ENABLE(openh264, AS_HELP_STRING([--enable-openh264],
[Use openh264 library (default: no)]),
[], [enable_openh264=no])
AM_CONDITIONAL(XRDP_OPENH264, [test x$enable_openh264 = xyes])
AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
[Do not use included painter library (default: no)]),
[], [enable_painter=yes])
@ -493,6 +497,8 @@ AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >
AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
AS_IF( [test "x$enable_openh264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_OPENH264, openh264 >= 2.0.0)] )
# checking for TurboJPEG
if test "x$enable_tjpeg" = "xyes"
then
@ -668,6 +674,7 @@ echo " jpeg $enable_jpeg"
echo " turbo jpeg $enable_tjpeg"
echo " rfxcodec $enable_rfxcodec"
echo " x264 $enable_x264"
echo " openh264 $enable_openh264"
echo " painter $enable_painter"
echo " pixman $enable_pixman"
echo " fuse $enable_fuse"

7
xrdp/Makefile.am
View File

@ -33,6 +33,13 @@ XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS)
XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h
endif
if XRDP_OPENH264
AM_CPPFLAGS += -DXRDP_OPENH264
AM_CPPFLAGS += $(XRDP_OPENH264_CFLAGS)
XRDP_EXTRA_LIBS += $(XRDP_OPENH264_LIBS)
XRDP_EXTRA_SOURCES += xrdp_encoder_openh264.c xrdp_encoder_openh264.h
endif
if XRDP_PIXMAN
AM_CPPFLAGS += -DXRDP_PIXMAN
AM_CPPFLAGS += $(PIXMAN_CFLAGS)

72
xrdp/xrdp_encoder.c
View File

@ -38,6 +38,10 @@
#include "xrdp_encoder_x264.h"
#endif
#ifdef XRDP_OPENH264
#include "xrdp_encoder_openh264.h"
#endif
#define DEFAULT_XRDP_GFX_FRAMES_IN_FLIGHT 2
/* limits used for validate env var XRDP_GFX_FRAMES_IN_FLIGHT */
#define MIN_XRDP_GFX_FRAMES_IN_FLIGHT 1
@ -94,7 +98,7 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
static int
process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
#endif
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
static int
process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
#endif
@ -171,7 +175,7 @@ xrdp_encoder_create(struct xrdp_mm *mm)
client_info->capture_format = XRDP_a8b8g8r8;
self->process_enc = process_enc_jpg;
}
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
else if (mm->egfx_flags & XRDP_EGFX_H264)
{
LOG(LOG_LEVEL_INFO,
@ -309,6 +313,56 @@ xrdp_encoder_create(struct xrdp_mm *mm)
/* make sure frames_in_flight is at least 1 */
self->frames_in_flight = MAX(self->frames_in_flight, 1);
#if defined(XRDP_X264) && defined(XRDP_OPENH264)
if (self->h264_flags == 0)
{
char *env = g_getenv("XRDP_PREFER_OPENH264");
if (env != NULL)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: found env var "
"XRDP_PREFER_OPENH264=%s", env);
if (g_text2bool(env))
{
ENC_SET_BIT(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT);
}
}
else
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: "
"XRDP_PREFER_OPENH264 not found");
}
}
if (ENC_IS_BIT_SET(self->h264_flags, ENC_FLAGS_PREFER_OPENH264_BIT))
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
}
else
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
}
#elif defined(XRDP_OPENH264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using openh264 for "
"software encoder");
self->h264_flags |= 1;
self->xrdp_encoder_h264_create = xrdp_encoder_openh264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_openh264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_openh264_encode;
#elif defined(XRDP_X264)
LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: using x264 for "
"software encoder");
self->xrdp_encoder_h264_create = xrdp_encoder_x264_create;
self->xrdp_encoder_h264_delete = xrdp_encoder_x264_delete;
self->xrdp_encoder_h264_encode = xrdp_encoder_x264_encode;
#endif
/* create thread to process messages */
tc_thread_create(proc_enc_msg, self);
@ -354,17 +408,17 @@ xrdp_encoder_delete(struct xrdp_encoder *self)
}
#endif
#if defined(XRDP_X264)
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
for (index = 0; index < 16; index++)
{
if (self->codec_handle_h264_gfx[index] != NULL)
{
xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]);
self->xrdp_encoder_h264_delete(self->codec_handle_h264_gfx[index]);
}
}
if (self->codec_handle_h264 != NULL)
{
xrdp_encoder_x264_delete(self->codec_handle_h264);
self->xrdp_encoder_h264_delete(self->codec_handle_h264);
}
#endif
@ -628,7 +682,7 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
}
#endif
#if defined(XRDP_X264)
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
/*****************************************************************************/
static int
@ -734,7 +788,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
struct xrdp_egfx_bulk *bulk, struct stream *in_s,
XRDP_ENC_DATA *enc)
{
#ifdef XRDP_X264
#if defined(XRDP_X264) || defined(XRDP_OPENH264)
int index;
int surface_id;
int codec_id;
@ -901,7 +955,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
if (self->codec_handle_h264_gfx[mon_index] == NULL)
{
self->codec_handle_h264_gfx[mon_index] =
xrdp_encoder_x264_create();
self->xrdp_encoder_h264_create();
if (self->codec_handle_h264_gfx[mon_index] == NULL)
{
g_free(s->data);
@ -909,7 +963,7 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
return NULL;
}
}
error = xrdp_encoder_x264_encode(
error = self->xrdp_encoder_h264_encode(
self->codec_handle_h264_gfx[mon_index], 0,
0, 0,
width, height, twidth, theight, 0,

18
xrdp/xrdp_encoder.h
View File

@ -14,6 +14,19 @@
struct xrdp_enc_data;
typedef void *(*xrdp_encoder_h264_create_proc)(void);
typedef int (*xrdp_encoder_h264_delete_proc)(void *handle);
typedef int (*xrdp_encoder_h264_encode_proc)(
void *handle, int session, int left, int top,
int width, int height, int twidth, int theight,
int format, const char *data,
short *crects, int num_crects,
char *cdata, int *cdata_bytes,
int connection_type, int *flags_ptr);
/* h264_flags */
#define ENC_FLAGS_PREFER_OPENH264_BIT 0
/* for codec mode operations */
struct xrdp_encoder
{
@ -46,6 +59,11 @@ struct xrdp_encoder
int quant_idx_y;
int quant_idx_u;
int quant_idx_v;
int h264_flags;
int pad0;
xrdp_encoder_h264_create_proc xrdp_encoder_h264_create;
xrdp_encoder_h264_delete_proc xrdp_encoder_h264_delete;
xrdp_encoder_h264_encode_proc xrdp_encoder_h264_encode;
};
/* cmd_id = 0 */

306
xrdp/xrdp_encoder_openh264.c Normal file
View File

@ -0,0 +1,306 @@
/**
* xrdp: A Remote Desktop Protocol server.
*
* Copyright (C) Jay Sorg 2016-2024
* Copyright (C) Christopher Pitstick 2023-2024
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* openh264 Encoder
*/
#if defined(HAVE_CONFIG_H)
#include <config_ac.h>
#endif
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <inttypes.h>
#include <wels/codec_api.h>
#include <wels/codec_def.h>
#include "xrdp.h"
#include "arch.h"
#include "os_calls.h"
#include "xrdp_encoder_openh264.h"
#include "xrdp_tconfig.h"
#define OPENH264_MAX_ENCODERS 16
struct openh264_encoder
{
ISVCEncoder *openh264_enc_han;
char *yuvdata;
int width;
int height;
};
struct openh264_global
{
struct openh264_encoder encoders[OPENH264_MAX_ENCODERS];
struct xrdp_tconfig_gfx_openh264_param
openh264_param[NUM_CONNECTION_TYPES];
};
/*****************************************************************************/
void *
xrdp_encoder_openh264_create(void)
{
struct openh264_global *og;
struct xrdp_tconfig_gfx gfxconfig;
LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_openh264_create:");
og = g_new0(struct openh264_global, 1);
tconfig_load_gfx(GFX_CONF, &gfxconfig);
memcpy(&og->openh264_param, &gfxconfig.openh264_param,
sizeof(struct xrdp_tconfig_gfx_openh264_param) *
NUM_CONNECTION_TYPES);
return og;
}
/*****************************************************************************/
int
xrdp_encoder_openh264_delete(void *handle)
{
struct openh264_global *og;
struct openh264_encoder *oe;
int index;
if (handle == NULL)
{
return 0;
}
og = (struct openh264_global *) handle;
for (index = 0; index < 16; index++)
{
oe = &(og->encoders[index]);
if (oe->openh264_enc_han != NULL)
{
WelsDestroySVCEncoder(oe->openh264_enc_han);
}
g_free(oe->yuvdata);
}
g_free(og);
return 0;
}
/*****************************************************************************/
int
xrdp_encoder_openh264_encode(void *handle, int session, int left, int top,
int width, int height, int twidth, int theight,
int format, const char *data,
short *crects, int num_crects,
char *cdata, int *cdata_bytes,
int connection_type, int *flags_ptr)
{
struct openh264_global *og;
struct openh264_encoder *oe;
const char *src8;
const char *src8a;
char *dst8;
char *dst8a;
char *dst8b;
char *dst8c;
int index;
int jndex;
int flags;
int x;
int y;
int cx;
int cy;
int ct; /* connection_type */
SSourcePicture pic1;
SFrameBSInfo info;
SLayerBSInfo *layer;
SEncParamExt encParamExt;
SSpatialLayerConfig *slc;
int status;
int layer_position;
char *write_location;
unsigned char *payload;
int size;
int lcdata_bytes;
LOG(LOG_LEVEL_TRACE, "xrdp_encoder_openh264_encode:");
flags = 0;
og = (struct openh264_global *) handle;
oe = &(og->encoders[session % OPENH264_MAX_ENCODERS]);
/* validate connection type */
ct = connection_type;
if (ct > CONNECTION_TYPE_LAN || ct < CONNECTION_TYPE_MODEM)
{
ct = CONNECTION_TYPE_LAN;
}
if ((oe->openh264_enc_han == NULL) ||
(oe->width != width) || (oe->height != height))
{
if (oe->openh264_enc_han != NULL)
{
LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: "
"WelsDestroySVCEncoder %p", oe->openh264_enc_han);
WelsDestroySVCEncoder(oe->openh264_enc_han);
oe->openh264_enc_han = NULL;
g_free(oe->yuvdata);
oe->yuvdata = NULL;
flags |= 2;
}
if ((width > 0) && (height > 0))
{
status = WelsCreateSVCEncoder(&oe->openh264_enc_han);
if ((status != 0) || (oe->openh264_enc_han == NULL))
{
LOG(LOG_LEVEL_ERROR, "Failed to create H.264 encoder");
return 1;
}
LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: "
"WelsCreateSVCEncoder rv %p for width %d height %d",
oe->openh264_enc_han, width, height);
status = (*oe->openh264_enc_han)->GetDefaultParams(
oe->openh264_enc_han, &encParamExt);
LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: "
"GetDefaultParams rv %d", status);
if (status == 0)
{
encParamExt.iUsageType = CAMERA_VIDEO_REAL_TIME;
encParamExt.iPicWidth = (width + 15) & ~15;
encParamExt.iPicHeight = (height + 15) & ~15;
encParamExt.iRCMode = RC_OFF_MODE;
encParamExt.bEnableFrameSkip = 0;
encParamExt.iSpatialLayerNum = 1;
slc = encParamExt.sSpatialLayers + 0;
slc->fFrameRate = encParamExt.fMaxFrameRate;
slc->iVideoWidth = encParamExt.iPicWidth;
slc->iVideoHeight = encParamExt.iPicHeight;
slc->iSpatialBitrate = encParamExt.iTargetBitrate;
slc->iMaxSpatialBitrate = encParamExt.iMaxBitrate;
status = (*oe->openh264_enc_han)->InitializeExt(
oe->openh264_enc_han, &encParamExt);
LOG(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: "
"InitializeExt rv %d", status);
}
oe->yuvdata = g_new(char, (width + 16) * (height + 16) * 2);
if (oe->yuvdata == NULL)
{
WelsDestroySVCEncoder(oe->openh264_enc_han);
oe->openh264_enc_han = NULL;
return 2;
}
flags |= 1;
}
oe->width = width;
oe->height = height;
}
if ((data != NULL) && (oe->openh264_enc_han != NULL))
{
g_memset(&pic1, 0, sizeof(pic1));
pic1.iPicWidth = (width + 15) & ~15;
pic1.iPicHeight = (height + 15) & ~15;
pic1.iColorFormat = videoFormatI420;
pic1.iStride[0] = pic1.iPicWidth;
pic1.iStride[1] = pic1.iPicWidth / 2;
pic1.iStride[2] = pic1.iPicWidth / 2;
pic1.pData[0] = (unsigned char *) (oe->yuvdata);
pic1.pData[1] = pic1.pData[0] + pic1.iPicWidth * pic1.iPicHeight;
pic1.pData[2] = pic1.pData[1] + (pic1.iPicWidth / 2) *
(pic1.iPicHeight / 2);
for (index = 0; index < num_crects; index++)
{
src8 = data;
dst8 = (char *) (pic1.pData[0]);
x = crects[index * 4 + 0];
y = crects[index * 4 + 1];
cx = crects[index * 4 + 2];
cy = crects[index * 4 + 3];
LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_openh264_encode: "
"x %d y %d cx %d cy %d", x, y, cx, cy);
src8 += twidth * y + x;
dst8 += pic1.iStride[0] * (y - top) + (x - left);
for (; cy > 0; cy -= 1)
{
g_memcpy(dst8, src8, cx);
src8 += twidth;
dst8 += pic1.iStride[0];
}
}
for (index = 0; index < num_crects; index++)
{
src8 = data; /* uv */
src8 += twidth * theight;
dst8 = (char *) (pic1.pData[1]); /* u */
dst8a = (char *) (pic1.pData[2]); /* v */
x = crects[index * 4 + 0];
y = crects[index * 4 + 1];
cx = crects[index * 4 + 2];
cy = crects[index * 4 + 3];
src8 += twidth * (y / 2) + x;
dst8 += pic1.iStride[1] * ((y - top) / 2) + ((x - left) / 2);
dst8a += pic1.iStride[2] * ((y - top) / 2) + ((x - left) / 2);
for (; cy > 0; cy -= 2)
{
src8a = src8; /* uv */
dst8b = dst8; /* u */
dst8c = dst8a; /* v */
for (jndex = 0; jndex < cx; jndex += 2)
{
*(dst8b++) = *(src8a++); /* u */
*(dst8c++) = *(src8a++); /* v */
}
src8 += twidth; /* uv */
dst8 += pic1.iStride[1]; /* u */
dst8a += pic1.iStride[2]; /* v */
}
}
g_memset(&info, 0, sizeof(info));
status = (*oe->openh264_enc_han)->EncodeFrame(oe->openh264_enc_han,
&pic1, &info);
if (status != 0)
{
LOG(LOG_LEVEL_INFO, "Failed to encode frame");
return 3;
}
if (info.eFrameType == videoFrameTypeSkip)
{
LOG(LOG_LEVEL_INFO, "frame was skipped!");
return 4;
}
lcdata_bytes = 0;
for (index = 0; index < info.iLayerNum; index++)
{
layer_position = 0;
layer = info.sLayerInfo + index;
for (jndex = 0; jndex < layer->iNalCount; jndex++)
{
write_location = cdata + lcdata_bytes;
payload = layer->pBsBuf + layer_position;
size = layer->pNalLengthInByte[jndex];
if (lcdata_bytes + size > *cdata_bytes)
{
LOG(LOG_LEVEL_INFO, "out of room");
return 5;
}
g_memcpy(write_location, payload, size);
layer_position += size;
lcdata_bytes += size;
}
}
*cdata_bytes = lcdata_bytes;
}
if (flags_ptr != NULL)
{
*flags_ptr = flags;
}
return 0;
}

39
xrdp/xrdp_encoder_openh264.h Normal file
View File

@ -0,0 +1,39 @@
/**
* xrdp: A Remote Desktop Protocol server.
*
* Copyright (C) Jay Sorg 2016-2024
* Copyright (C) Christopher Pitstick 2023-2024
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* openh264 Encoder
*/
#ifndef _XRDP_ENCODER_OPENH264_H
#define _XRDP_ENCODER_OPENH264_H
#include "arch.h"
void *
xrdp_encoder_openh264_create(void);
int
xrdp_encoder_openh264_delete(void *handle);
int
xrdp_encoder_openh264_encode(void *handle, int session, int left, int top,
int width, int height, int twidth, int theight,
int format, const char *data,
short *crects, int num_crects,
char *cdata, int *cdata_bytes,
int connection_type, int *flags_ptr);
#endif

7
xrdp/xrdp_tconfig.h
View File

@ -42,6 +42,11 @@ struct xrdp_tconfig_gfx_x264_param
int fps_den;
};
struct xrdp_tconfig_gfx_openh264_param
{
int pad0;
};
enum xrdp_tconfig_codecs
{
XTC_H264,
@ -59,6 +64,8 @@ struct xrdp_tconfig_gfx
struct xrdp_tconfig_gfx_codec_order codec;
/* store x264 parameters for each connection type */
struct xrdp_tconfig_gfx_x264_param x264_param[NUM_CONNECTION_TYPES];
struct xrdp_tconfig_gfx_openh264_param
openh264_param[NUM_CONNECTION_TYPES];
};
static const char *const rdpbcgr_connection_type_names[] =