Merge pull request #3073 from jsorg71/x264

add GFX h264 software via --enable-x264

common/xrdp_client_info.h

@@ -74,6 +74,16 @@ enum client_resize_mode
     CRMODE_MULTI_SCREEN
 };
 
+enum xrdp_capture_code
+{
+    CC_SIMPLE       = 0,
+    CC_SUF_A16      = 1,
+    CC_SUF_RFX      = 2,
+    CC_SUF_A2       = 3,
+    CC_GFX_PRO      = 4,
+    CC_GFX_A2       = 5
+};
+
 /**
  * Type describing Unicode input state
  */
@@ -179,7 +189,7 @@ struct xrdp_client_info
     int mcs_early_capability_flags;
 
     int max_fastpath_frag_bytes;
-    int capture_code;
+    int pad0; /* unused */
     int capture_format;
 
     char certificate[1024];
@@ -239,6 +249,7 @@ struct xrdp_client_info
     enum client_resize_mode client_resize_mode;
 
     enum unicode_input_state unicode_input_support;
+    enum xrdp_capture_code capture_code;
 };
 
 enum xrdp_encoder_flags
@@ -258,6 +269,6 @@ enum xrdp_encoder_flags
 
 /* yyyymmdd of last incompatible change to xrdp_client_info */
 /* also used for changes to all the xrdp installed headers */
-#define CLIENT_INFO_CURRENT_VERSION 20230425
+#define CLIENT_INFO_CURRENT_VERSION 20240514
 
 #endif

configure.ac

@@ -172,7 +172,10 @@ AC_ARG_ENABLE(pixman, AS_HELP_STRING([--enable-pixman],
               [Use pixman library (default: no)]),
               [], [enable_pixman=no])
 AM_CONDITIONAL(XRDP_PIXMAN, [test x$enable_pixman = xyes])
-
+AC_ARG_ENABLE(x264, AS_HELP_STRING([--enable-x264],
+              [Use x264 library (default: no)]),
+              [], [enable_x264=no])
+AM_CONDITIONAL(XRDP_X264, [test x$enable_x264 = xyes])
 AC_ARG_ENABLE(painter, AS_HELP_STRING([--disable-painter],
               [Do not use included painter library (default: no)]),
               [], [enable_painter=yes])
@@ -489,6 +492,8 @@ fi
 
 AS_IF( [test "x$enable_pixman" = "xyes"] , [PKG_CHECK_MODULES(PIXMAN, pixman-1 >= 0.1.0)] )
 
+AS_IF( [test "x$enable_x264" = "xyes"] , [PKG_CHECK_MODULES(XRDP_X264, x264 >= 0.3.0)] )
+
 # checking for TurboJPEG
 if test "x$enable_tjpeg" = "xyes"
 then
@@ -662,6 +667,7 @@ echo "  fdkaac                  $enable_fdkaac"
 echo "  jpeg                    $enable_jpeg"
 echo "  turbo jpeg              $enable_tjpeg"
 echo "  rfxcodec                $enable_rfxcodec"
+echo "  x264                    $enable_x264"
 echo "  painter                 $enable_painter"
 echo "  pixman                  $enable_pixman"
 echo "  fuse                    $enable_fuse"

xrdp/Makefile.am

@@ -18,6 +18,7 @@ AM_CPPFLAGS = \
   -I$(top_srcdir)/third_party/tomlc99
 
 XRDP_EXTRA_LIBS =
+XRDP_EXTRA_SOURCES =
 
 if XRDP_RFXCODEC
 AM_CPPFLAGS += -DXRDP_RFXCODEC
@@ -25,6 +26,13 @@ AM_CPPFLAGS += -I$(top_srcdir)/librfxcodec/include
 XRDP_EXTRA_LIBS += $(top_builddir)/librfxcodec/src/.libs/librfxencode.a
 endif
 
+if XRDP_X264
+AM_CPPFLAGS += -DXRDP_X264
+AM_CPPFLAGS += $(XRDP_X264_CFLAGS)
+XRDP_EXTRA_LIBS += $(XRDP_X264_LIBS)
+XRDP_EXTRA_SOURCES += xrdp_encoder_x264.c xrdp_encoder_x264.h
+endif
+
 if XRDP_PIXMAN
 AM_CPPFLAGS += -DXRDP_PIXMAN
 AM_CPPFLAGS += $(PIXMAN_CFLAGS)
@@ -63,7 +71,8 @@ xrdp_SOURCES = \
   xrdp_egfx.c \
   xrdp_egfx.h \
   xrdp_wm.c \
-  xrdp_main_utils.c
+  xrdp_main_utils.c \
+  $(XRDP_EXTRA_SOURCES)
 
 xrdp_LDADD = \
   $(top_builddir)/common/libcommon.la \

xrdp/xrdp_encoder.c

@@ -34,6 +34,10 @@
 #include "rfxcodec_encode.h"
 #endif
 
+#ifdef XRDP_X264
+#include "xrdp_encoder_x264.h"
+#endif
+
 #define DEFAULT_XRDP_GFX_FRAMES_IN_FLIGHT 2
 /* limits used for validate env var XRDP_GFX_FRAMES_IN_FLIGHT */
 #define MIN_XRDP_GFX_FRAMES_IN_FLIGHT 1
@@ -70,11 +74,13 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 #ifdef XRDP_RFXCODEC
 static int
 process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
-static int
-process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 #endif
+#ifdef XRDP_X264
 static int
 process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
+#endif
+static int
+process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc);
 
 /*****************************************************************************/
 /* Item destructor for self->fifo_to_proc */
@@ -128,25 +134,44 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         return NULL;
     }
     self->mm = mm;
-
+    self->process_enc = process_enc_egfx;
     if (client_info->jpeg_codec_id != 0)
     {
         LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting jpeg codec session");
         self->codec_id = client_info->jpeg_codec_id;
         self->in_codec_mode = 1;
         self->codec_quality = client_info->jpeg_prop[0];
-        client_info->capture_code = 0;
+        client_info->capture_code = CC_SIMPLE;
         client_info->capture_format = XRDP_a8b8g8r8;
         self->process_enc = process_enc_jpg;
     }
+#ifdef XRDP_X264
+    else if (mm->egfx_flags & XRDP_EGFX_H264)
+    {
+        LOG(LOG_LEVEL_INFO,
+            "xrdp_encoder_create: starting h264 codec session gfx");
+        self->in_codec_mode = 1;
+        client_info->capture_code = CC_GFX_A2;
+        client_info->capture_format = XRDP_nv12_709fr;
+        self->gfx = 1;
+    }
+    else if (client_info->h264_codec_id != 0)
+    {
+        LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session");
+        self->codec_id = client_info->h264_codec_id;
+        self->in_codec_mode = 1;
+        client_info->capture_code = CC_SUF_A2;
+        client_info->capture_format = XRDP_nv12;
+        self->process_enc = process_enc_h264;
+    }
+#endif
 #ifdef XRDP_RFXCODEC
     else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO)
     {
         LOG(LOG_LEVEL_INFO,
             "xrdp_encoder_create: starting gfx rfx pro codec session");
         self->in_codec_mode = 1;
-        client_info->capture_code = 4;
+        client_info->capture_code = CC_GFX_PRO;
-        self->process_enc = process_enc_egfx;
         self->gfx = 1;
         self->quants = (const char *) g_rfx_quantization_values;
         self->num_quants = 2;
@@ -159,22 +184,13 @@ xrdp_encoder_create(struct xrdp_mm *mm)
         LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session");
         self->codec_id = client_info->rfx_codec_id;
         self->in_codec_mode = 1;
-        client_info->capture_code = 2;
+        client_info->capture_code = CC_SUF_RFX;
         self->process_enc = process_enc_rfx;
         self->codec_handle_rfx = rfxcodec_encode_create(mm->wm->screen->width,
                                  mm->wm->screen->height,
                                  RFX_FORMAT_YUV, 0);
     }
 #endif
-    else if (client_info->h264_codec_id != 0)
-    {
-        LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session");
-        self->codec_id = client_info->h264_codec_id;
-        self->in_codec_mode = 1;
-        client_info->capture_code = 3;
-        client_info->capture_format = XRDP_nv12;
-        self->process_enc = process_enc_h264;
-    }
     else
     {
         g_free(self);
@@ -295,7 +311,7 @@ xrdp_encoder_delete(struct xrdp_encoder *self)
     {
         if (self->codec_handle_h264_gfx[index] != NULL)
         {
-            rfxcodec_encode_destroy(self->codec_handle_h264_gfx[index]);
+            xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]);
         }
     }
     if (self->codec_handle_h264 != NULL)
@@ -563,6 +579,61 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
 }
 #endif
 
+#if defined(XRDP_X264)
+
+/*****************************************************************************/
+static int
+out_RFX_AVC420_METABLOCK(struct xrdp_egfx_rect *dst_rect,
+                         struct stream *s,
+                         struct xrdp_egfx_rect *rects,
+                         int num_rects)
+{
+    struct xrdp_region *reg;
+    struct xrdp_rect rect;
+    int index;
+    int count;
+
+    /* RFX_AVC420_METABLOCK */
+    s_push_layer(s, iso_hdr, 4); /* numRegionRects, set later */
+    reg = xrdp_region_create(NULL);
+    if (reg == NULL)
+    {
+        return 1;
+    }
+    for (index = 0; index < num_rects; index++)
+    {
+        rect.left = MAX(0, rects[index].x1 - dst_rect->x1 - 1);
+        rect.top = MAX(0, rects[index].y1 - dst_rect->y1 - 1);
+        rect.right = MIN(dst_rect->x2 - dst_rect->x1,
+                         rects[index].x2 - dst_rect->x1 + 1);
+        rect.bottom = MIN(dst_rect->y2 - dst_rect->y1,
+                          rects[index].y2 - dst_rect->y1 + 1);
+        xrdp_region_add_rect(reg, &rect);
+    }
+    index = 0;
+    while (xrdp_region_get_rect(reg, index, &rect) == 0)
+    {
+        out_uint16_le(s, rect.left);
+        out_uint16_le(s, rect.top);
+        out_uint16_le(s, rect.right);
+        out_uint16_le(s, rect.bottom);
+        index++;
+    }
+    xrdp_region_delete(reg);
+    count = index;
+    while (index > 0)
+    {
+        out_uint8(s, 23); /* qp */
+        out_uint8(s, 100); /* quality level 0..100 */
+        index--;
+    }
+    s_push_layer(s, mcs_hdr, 0);
+    s_pop_layer(s, iso_hdr);
+    out_uint32_le(s, count); /* numRegionRects */
+    s_pop_layer(s, mcs_hdr);
+    return 0;
+}
+
 /*****************************************************************************/
 /* called from encoder thread */
 static int
@@ -572,7 +643,7 @@ process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
     return 0;
 }
 
-#ifdef XRDP_RFXCODEC
+#endif
 
 /*****************************************************************************/
 static int
@@ -614,11 +685,212 @@ gfx_wiretosurface1(struct xrdp_encoder *self,
                    struct xrdp_egfx_bulk *bulk, struct stream *in_s,
                    XRDP_ENC_DATA *enc)
 {
+#ifdef XRDP_X264
+    int index;
+    int surface_id;
+    int codec_id;
+    int pixel_format;
+    int num_rects_d;
+    int num_rects_c;
+    struct stream *rv;
+    short left;
+    short top;
+    short width;
+    short height;
+    short twidth;
+    short theight;
+    int bitmap_data_length;
+    int flags;
+    struct xrdp_egfx_rect *d_rects;
+    struct xrdp_egfx_rect *c_rects;
+    struct xrdp_egfx_rect dst_rect;
+    int error;
+    struct stream ls;
+    struct stream *s;
+    short *crects;
+    struct xrdp_enc_gfx_cmd *enc_gfx_cmd = &(enc->u.gfx);
+    int mon_index;
+
+    s = &ls;
+    g_memset(s, 0, sizeof(struct stream));
+    s->size = self->max_compressed_bytes;
+    s->data = g_new(char, s->size);
+    if (s->data == NULL)
+    {
+        return NULL;
+    }
+    s->p = s->data;
+    if (!s_check_rem(in_s, 11))
+    {
+        g_free(s->data);
+        return NULL;
+    }
+    in_uint16_le(in_s, surface_id);
+    in_uint16_le(in_s, codec_id);
+    in_uint8(in_s, pixel_format);
+    in_uint32_le(in_s, flags);
+    mon_index = (flags >> 28) & 0xF;
+    in_uint16_le(in_s, num_rects_d);
+    if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
+            (!s_check_rem(in_s, num_rects_d * 8)))
+    {
+        g_free(s->data);
+        return NULL;
+    }
+    d_rects = g_new0(struct xrdp_egfx_rect, num_rects_d);
+    if (d_rects == NULL)
+    {
+        g_free(s->data);
+        return NULL;
+    }
+    for (index = 0; index < num_rects_d; index++)
+    {
+        in_uint16_le(in_s, left);
+        in_uint16_le(in_s, top);
+        in_uint16_le(in_s, width);
+        in_uint16_le(in_s, height);
+        d_rects[index].x1 = left;
+        d_rects[index].y1 = top;
+        d_rects[index].x2 = left + width;
+        d_rects[index].y2 = top + height;
+
+    }
+    if (!s_check_rem(in_s, 2))
+    {
+        g_free(s->data);
+        g_free(d_rects);
+        return NULL;
+    }
+    in_uint16_le(in_s, num_rects_c);
+    if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
+            (!s_check_rem(in_s, num_rects_c * 8)))
+    {
+        g_free(s->data);
+        g_free(d_rects);
+        return NULL;
+    }
+    c_rects = g_new0(struct xrdp_egfx_rect, num_rects_c);
+    if (c_rects == NULL)
+    {
+        g_free(s->data);
+        g_free(d_rects);
+        return NULL;
+    }
+    crects = g_new(short, num_rects_c * 4);
+    if (crects == NULL)
+    {
+        g_free(s->data);
+        g_free(c_rects);
+        g_free(d_rects);
+        return NULL;
+    }
+    g_memcpy(crects, in_s->p, num_rects_c * 2 * 4);
+    for (index = 0; index < num_rects_c; index++)
+    {
+        in_uint16_le(in_s, left);
+        in_uint16_le(in_s, top);
+        in_uint16_le(in_s, width);
+        in_uint16_le(in_s, height);
+        c_rects[index].x1 = left;
+        c_rects[index].y1 = top;
+        c_rects[index].x2 = left + width;
+        c_rects[index].y2 = top + height;
+    }
+    if (!s_check_rem(in_s, 8))
+    {
+        g_free(s->data);
+        g_free(c_rects);
+        g_free(d_rects);
+        g_free(crects);
+        return NULL;
+    }
+    in_uint16_le(in_s, left);
+    in_uint16_le(in_s, top);
+    in_uint16_le(in_s, width);
+    in_uint16_le(in_s, height);
+    twidth = width;
+    theight = height;
+    dst_rect.x1 = 0;
+    dst_rect.y1 = 0;
+    dst_rect.x2 = width;
+    dst_rect.y2 = height;
+    LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface1: left %d top "
+              "%d width %d height %d mon_index %d",
+              left, top, width, height, mon_index);
+    /* RFX_AVC420_METABLOCK */
+    if (out_RFX_AVC420_METABLOCK(&dst_rect, s, d_rects, num_rects_d) != 0)
+    {
+        g_free(s->data);
+        g_free(c_rects);
+        g_free(d_rects);
+        g_free(crects);
+        LOG(LOG_LEVEL_INFO, "10");
+        return NULL;
+    }
+
+    g_free(c_rects);
+    g_free(d_rects);
+
+    if (ENC_IS_BIT_SET(flags, 0))
+    {
+        /* already compressed */
+        out_uint8a(s, enc_gfx_cmd->data, enc_gfx_cmd->data_bytes);
+    }
+    else
+    {
+        /* assume NV12 format */
+        if (twidth * theight * 3 / 2 > enc_gfx_cmd->data_bytes)
+        {
+            g_free(s->data);
+            g_free(crects);
+            return NULL;
+        }
+        bitmap_data_length = s_rem_out(s);
+        if (self->codec_handle_h264_gfx[mon_index] == NULL)
+        {
+            self->codec_handle_h264_gfx[mon_index] =
+                xrdp_encoder_x264_create();
+            if (self->codec_handle_h264_gfx[mon_index] == NULL)
+            {
+                g_free(s->data);
+                g_free(crects);
+                return NULL;
+            }
+        }
+        error = xrdp_encoder_x264_encode(
+                    self->codec_handle_h264_gfx[mon_index], 0,
+                    0, 0,
+                    width, height, twidth, theight, 0,
+                    enc_gfx_cmd->data,
+                    crects, num_rects_c,
+                    s->p, &bitmap_data_length, NULL);
+        if (error == 0)
+        {
+            xstream_seek(s, bitmap_data_length);
+        }
+        else
+        {
+            g_free(s->data);
+            g_free(crects);
+            return NULL;
+        }
+    }
+    s_mark_end(s);
+    bitmap_data_length = (int) (s->end - s->data);
+    rv = xrdp_egfx_wire_to_surface1(bulk, surface_id,
+                                    codec_id,
+                                    pixel_format, &dst_rect,
+                                    s->data, bitmap_data_length);
+    g_free(s->data);
+    g_free(crects);
+    return rv;
+#else
     (void)self;
     (void)bulk;
     (void)in_s;
     (void)enc;
     return NULL;
+#endif
 }
 
 /*****************************************************************************/
@@ -627,6 +899,7 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
                    struct xrdp_egfx_bulk *bulk, struct stream *in_s,
                    XRDP_ENC_DATA *enc)
 {
+#ifdef XRDP_RFXCODEC
     int index;
     int surface_id;
     int codec_id;
@@ -723,6 +996,9 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
     in_uint16_le(in_s, top);
     in_uint16_le(in_s, width);
     in_uint16_le(in_s, height);
+    LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface2: left %d top "
+              "%d width %d height %d mon_index %d",
+              left, top, width, height, mon_index);
     if (self->codec_handle_prfx_gfx[mon_index] == NULL)
     {
         self->codec_handle_prfx_gfx[mon_index] = rfxcodec_encode_create(
@@ -798,6 +1074,13 @@ gfx_wiretosurface2(struct xrdp_encoder *self,
     g_free(rfxrects);
     g_free(bitmap_data);
     return rv;
+#else
+    (void)self;
+    (void)bulk;
+    (void)in_s;
+    (void)enc;
+    return NULL;
+#endif
 }
 
 /*****************************************************************************/
@@ -1026,6 +1309,7 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
         }
         holdend = in_s.end;
         in_s.end = holdp + cmd_bytes;
+        LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_egfx: cmd_id %d", cmd_id);
         switch (cmd_id)
         {
             case XR_RDPGFX_CMDID_WIRETOSURFACE_1:       /* 0x0001 */
@@ -1080,12 +1364,14 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
             }
             g_free(s); /* don't call free_stream() here so s->data is valid */
         }
+        else
+        {
+            LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_egfx: nil");
+        }
     }
     return 0;
 }
 
-#endif
-
 /**
  * Encoder thread main loop
  *****************************************************************************/

xrdp/xrdp_encoder_x264.c

@@ -0,0 +1,235 @@
+/**
+ * xrdp: A Remote Desktop Protocol server.
+ *
+ * Copyright (C) Jay Sorg 2016-2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * x264 Encoder
+ */
+
+#if defined(HAVE_CONFIG_H)
+#include <config_ac.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <inttypes.h>
+#include <x264.h>
+
+#include "xrdp.h"
+#include "arch.h"
+#include "os_calls.h"
+#include "xrdp_encoder_x264.h"
+
+#define X264_MAX_ENCODERS 16
+
+struct x264_encoder
+{
+    x264_t *x264_enc_han;
+    char *yuvdata;
+    x264_param_t x264_params;
+    int width;
+    int height;
+};
+
+struct x264_global
+{
+    struct x264_encoder encoders[X264_MAX_ENCODERS];
+};
+
+/*****************************************************************************/
+void *
+xrdp_encoder_x264_create(void)
+{
+    LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_encoder_x264_create:");
+    return g_new0(struct x264_global, 1);
+}
+
+/*****************************************************************************/
+int
+xrdp_encoder_x264_delete(void *handle)
+{
+    struct x264_global *xg;
+    struct x264_encoder *xe;
+    int index;
+
+    if (handle == NULL)
+    {
+        return 0;
+    }
+    xg = (struct x264_global *) handle;
+    for (index = 0; index < 16; index++)
+    {
+        xe = &(xg->encoders[index]);
+        if (xe->x264_enc_han != NULL)
+        {
+            x264_encoder_close(xe->x264_enc_han);
+        }
+        g_free(xe->yuvdata);
+    }
+    g_free(xg);
+    return 0;
+}
+
+/*****************************************************************************/
+int
+xrdp_encoder_x264_encode(void *handle, int session, int left, int top,
+                         int width, int height, int twidth, int theight,
+                         int format, const char *data,
+                         short *crects, int num_crects,
+                         char *cdata, int *cdata_bytes, int *flags_ptr)
+{
+    struct x264_global *xg;
+    struct x264_encoder *xe;
+    const char *src8;
+    char *dst8;
+    int index;
+    x264_nal_t *nals;
+    int num_nals;
+    int frame_size;
+    int x264_width_height;
+    int flags;
+    int x;
+    int y;
+    int cx;
+    int cy;
+
+    x264_picture_t pic_in;
+    x264_picture_t pic_out;
+
+    LOG(LOG_LEVEL_TRACE, "xrdp_encoder_x264_encode:");
+    flags = 0;
+    xg = (struct x264_global *) handle;
+    xe = &(xg->encoders[session % X264_MAX_ENCODERS]);
+    if ((xe->x264_enc_han == NULL) ||
+            (xe->width != width) || (xe->height != height))
+    {
+        if (xe->x264_enc_han != NULL)
+        {
+            LOG(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: "
+                "x264_encoder_close %p", xe->x264_enc_han);
+            x264_encoder_close(xe->x264_enc_han);
+            xe->x264_enc_han = NULL;
+            g_free(xe->yuvdata);
+            xe->yuvdata = NULL;
+            flags |= 2;
+        }
+        if ((width > 0) && (height > 0))
+        {
+            //x264_param_default_preset(&(xe->x264_params), "superfast", "zerolatency");
+            x264_param_default_preset(&(xe->x264_params), "ultrafast", "zerolatency");
+            xe->x264_params.i_threads = 1;
+            xe->x264_params.i_width = (width + 15) & ~15;
+            xe->x264_params.i_height = (height + 15) & ~15;
+            xe->x264_params.i_fps_num = 24;
+            xe->x264_params.i_fps_den = 1;
+            //xe->x264_params.b_cabac = 1;
+            //xe->x264_params.i_bframe = 0;
+            //xe->x264_params.rc.i_rc_method = X264_RC_CQP;
+            //xe->x264_params.rc.i_qp_constant = 23;
+            //x264_param_apply_profile(&(xe->x264_params), "high");
+            x264_param_apply_profile(&(xe->x264_params), "main");
+            //x264_param_apply_profile(&(xe->x264_params), "baseline");
+            xe->x264_enc_han = x264_encoder_open(&(xe->x264_params));
+            LOG(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: "
+                "x264_encoder_open rv %p for width %d height %d",
+                xe->x264_enc_han, width, height);
+            if (xe->x264_enc_han == NULL)
+            {
+                return 1;
+            }
+            xe->yuvdata = g_new(char, width * height * 2);
+            if (xe->yuvdata == NULL)
+            {
+                x264_encoder_close(xe->x264_enc_han);
+                xe->x264_enc_han = NULL;
+                return 2;
+            }
+            flags |= 1;
+        }
+        xe->width = width;
+        xe->height = height;
+    }
+
+    if ((data != NULL) && (xe->x264_enc_han != NULL))
+    {
+        x264_width_height = xe->x264_params.i_width * xe->x264_params.i_height;
+        for (index = 0; index < num_crects; index++)
+        {
+            src8 = data;
+            dst8 = xe->yuvdata;
+            x = crects[index * 4 + 0];
+            y = crects[index * 4 + 1];
+            cx = crects[index * 4 + 2];
+            cy = crects[index * 4 + 3];
+            LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_x264_encode: x %d y %d "
+                      "cx %d cy %d", x, y, cx, cy);
+            src8 += twidth * y + x;
+            dst8 += xe->x264_params.i_width * (y - top) + (x - left);
+            for (; cy > 0; cy -= 1)
+            {
+                g_memcpy(dst8, src8, cx);
+                src8 += twidth;
+                dst8 += xe->x264_params.i_width;
+            }
+        }
+        for (index = 0; index < num_crects; index++)
+        {
+            src8 = data;
+            src8 += twidth * theight;
+            dst8 = xe->yuvdata;
+            dst8 += x264_width_height;
+            x = crects[index * 4 + 0];
+            y = crects[index * 4 + 1];
+            cx = crects[index * 4 + 2];
+            cy = crects[index * 4 + 3];
+            src8 += twidth * (y / 2) + x;
+            dst8 += xe->x264_params.i_width * ((y - top) / 2) + (x - left);
+            for (; cy > 0; cy -= 2)
+            {
+                g_memcpy(dst8, src8, cx);
+                src8 += twidth;
+                dst8 += xe->x264_params.i_width;
+            }
+        }
+        g_memset(&pic_in, 0, sizeof(pic_in));
+        pic_in.img.i_csp = X264_CSP_NV12;
+        pic_in.img.i_plane = 2;
+        pic_in.img.plane[0] = (unsigned char *) (xe->yuvdata);
+        pic_in.img.plane[1] = (unsigned char *)
+                              (xe->yuvdata + x264_width_height);
+        pic_in.img.i_stride[0] = xe->x264_params.i_width;
+        pic_in.img.i_stride[1] = xe->x264_params.i_width;
+        num_nals = 0;
+        frame_size = x264_encoder_encode(xe->x264_enc_han, &nals, &num_nals,
+                                         &pic_in, &pic_out);
+        LOG(LOG_LEVEL_TRACE, "i_type %d", pic_out.i_type);
+        if (frame_size < 1)
+        {
+            return 3;
+        }
+        if (*cdata_bytes < frame_size)
+        {
+            return 4;
+        }
+        g_memcpy(cdata, nals[0].p_payload, frame_size);
+        *cdata_bytes = frame_size;
+    }
+    if (flags_ptr != NULL)
+    {
+        *flags_ptr = flags;
+    }
+    return 0;
+}

xrdp/xrdp_encoder_x264.h

@@ -0,0 +1,38 @@
+/**
+ * xrdp: A Remote Desktop Protocol server.
+ *
+ * Copyright (C) Jay Sorg 2016-2024
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * libx264 Encoder
+ */
+
+#ifndef _XRDP_ENCODER_X264_H
+#define _XRDP_ENCODER_X264_H
+
+#include "arch.h"
+
+void *
+xrdp_encoder_x264_create(void);
+int
+xrdp_encoder_x264_delete(void *handle);
+int
+xrdp_encoder_x264_encode(void *handle, int session, int left, int top,
+                         int width, int height, int twidth, int theight,
+                         int format, const char *data,
+                         short *crects, int num_crects,
+                         char *cdata, int *cdata_bytes, int *flags_ptr);
+
+#endif
+