Further changes to selectable H.264 support

- Fix CI errors
- tconfig_load_gfx() removes H.264 from the supported codec list
  if significant errors are found loading the H.264 configuration
- tconfig_load_gfx() always produces a usable config, even if the
  specified file can't be loaded

(cherry picked from commit 5351511272)

tests/xrdp/gfx/gfx_missing_h264.toml

@@ -0,0 +1,9 @@
+[codec]
+order = [ "H.264", "RFX" ]
+
+[x264.lan]
+[x264.wan]
+[x264.broadband_high]
+[x264.satellite]
+[x264.broadband_low]
+[x264.modem]

tests/xrdp/test_tconfig.c

@@ -73,6 +73,28 @@ START_TEST(test_tconfig_gfx_codec_order)
 }
 END_TEST
 
+START_TEST(test_tconfig_gfx_missing_file)
+{
+    struct xrdp_tconfig_gfx gfxconfig;
+
+    /* Check RFX config is returned if the file doesn't exist */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/no_such_file.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+}
+END_TEST
+
+START_TEST(test_tconfig_gfx_missing_h264)
+{
+    struct xrdp_tconfig_gfx gfxconfig;
+
+    /* Check RFX config only is returned if H.264 parameters are missing */
+    tconfig_load_gfx(GFXCONF_STUBDIR "/gfx_missing_h264.toml", &gfxconfig);
+    ck_assert_int_eq(gfxconfig.codec.codec_count, 1);
+    ck_assert_int_eq(gfxconfig.codec.codecs[0], XTC_RFX);
+}
+END_TEST
+
 /******************************************************************************/
 Suite *
 make_suite_tconfig_load_gfx(void)
@@ -86,6 +108,8 @@ make_suite_tconfig_load_gfx(void)
     tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_always_success);
     tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_x264_load_basic);
     tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_codec_order);
+    tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_missing_file);
+    tcase_add_test(tc_tconfig_load_gfx, test_tconfig_gfx_missing_h264);
 
     suite_add_tcase(s, tc_tconfig_load_gfx);
 

xrdp/xrdp_mm.c

@@ -1277,6 +1277,10 @@ xrdp_mm_egfx_caps_advertise(void *user, int caps_count,
     int flags;
     struct ver_flags_t *ver_flags;
 
+#if !defined(XRDP_H264)
+    UNUSED_VAR(best_h264_index);
+#endif
+
     LOG(LOG_LEVEL_INFO, "xrdp_mm_egfx_caps_advertise:");
     self = (struct xrdp_mm *) user;
     screen = self->wm->screen;
@@ -1350,7 +1354,7 @@ xrdp_mm_egfx_caps_advertise(void *user, int caps_count,
 
     LOG(LOG_LEVEL_INFO, "Codec search order is %s",
         tconfig_codec_order_to_str(co, cobuff, sizeof(cobuff)));
-    for (index = 0 ; index < (unsigned int)co->codec_count ; ++index)
+    for (index = 0 ; index < co->codec_count ; ++index)
     {
 #if defined(XRDP_H264)
         if (co->codecs[index] == XTC_H264 && best_h264_index >= 0)

xrdp/xrdp_tconfig.c

@@ -318,14 +318,61 @@ static int tconfig_load_gfx_order(toml_table_t *tfile, struct xrdp_tconfig_gfx *
     return 0;
 }
 
+/**
+ * Determines whether a codec is enabled
+ * @param co Ordered codec list
+ * @param code Code of codec to look for
+ * @return boolean
+ */
+static int
+codec_enabled(const struct xrdp_tconfig_gfx_codec_order *co,
+              enum xrdp_tconfig_codecs code)
+{
+    for (unsigned short i = 0; i < co->codec_count; ++i)
+    {
+        if (co->codecs[i] == code)
+        {
+            return 1;
+        }
+    }
+
+    return 0;
+}
+
+/**
+ * Disables a Codec by removing it from the codec list
+ * @param co Ordered codec list
+ * @param code Code of codec to remove from list
+ *
+ * The order of the passed-in codec list is preserved.
+ */
+static void
+disable_codec(struct xrdp_tconfig_gfx_codec_order *co,
+              enum xrdp_tconfig_codecs code)
+{
+    unsigned short j = 0;
+    for (unsigned short i = 0; i < co->codec_count; ++i)
+    {
+        if (co->codecs[i] != code)
+        {
+            co->codecs[j++] = co->codecs[i];
+        }
+    }
+    co->codec_count = j;
+}
+
 int
 tconfig_load_gfx(const char *filename, struct xrdp_tconfig_gfx *config)
 {
     FILE *fp;
     char errbuf[200];
     toml_table_t *tfile;
+    int rv = 0;
 
-    memset(config, 0, sizeof(struct xrdp_tconfig_gfx));
+    /* Default to just RFX support. in case we can't load anything */
+    config->codec.codec_count = 1;
+    config->codec.codecs[0] = XTC_RFX;
+    memset(config->x264_param, 0, sizeof(config->x264_param));
 
     if ((fp = fopen(filename, "r")) == NULL)
     {
@@ -344,20 +391,34 @@ tconfig_load_gfx(const char *filename, struct xrdp_tconfig_gfx *config)
     TCLOG(LOG_LEVEL_INFO, "Loading GFX config file %s", filename);
     fclose(fp);
 
-    /* Load GFX order */
+    /* Load GFX codec order */
     tconfig_load_gfx_order(tfile, config);
 
-    /* First of all, read the default params and override later */
-    tconfig_load_gfx_x264_ct(tfile, 0, config->x264_param);
-
-    for (int ct = CONNECTION_TYPE_MODEM; ct < NUM_CONNECTION_TYPES; ct++)
+    /* H.264 configuration */
+    if (codec_enabled(&config->codec, XTC_H264))
     {
-        config->x264_param[ct] = config->x264_param[0];
-        tconfig_load_gfx_x264_ct(tfile, ct, config->x264_param);
+        /* First of all, read the default params */
+        if (tconfig_load_gfx_x264_ct(tfile, 0, config->x264_param) != 0)
+        {
+            /* We can't read the H.264 defaults. Disable H.264 */
+            LOG(LOG_LEVEL_WARNING, "H.264 support will be disabled");
+            disable_codec(&config->codec, XTC_H264);
+            rv = 1;
+        }
+        else
+        {
+            /* Copy default params to other connection types, and
+             * then override them */
+            for (int ct = CONNECTION_TYPE_MODEM; ct < NUM_CONNECTION_TYPES;
+                    ct++)
+            {
+                config->x264_param[ct] = config->x264_param[0];
+                tconfig_load_gfx_x264_ct(tfile, ct, config->x264_param);
+            }
+        }
     }
-
     toml_free(tfile);
 
-    return 0;
+    return rv;
 }
 

xrdp/xrdp_tconfig.h

@@ -50,9 +50,8 @@ enum xrdp_tconfig_codecs
 
 struct xrdp_tconfig_gfx_codec_order
 {
-
     enum xrdp_tconfig_codecs codecs[2];
-    unsigned int codec_count;
+    unsigned short codec_count;
 };
 
 struct xrdp_tconfig_gfx
@@ -89,6 +88,17 @@ tconfig_codec_order_to_str(
     char *buff,
     unsigned int bufflen);
 
-int tconfig_load_gfx(const char *filename, struct xrdp_tconfig_gfx *config);
+/**
+ * Loads the GFX config from the specified file
+ *
+ * @param filename Name of file to load
+ * @param config Struct to receive result
+ * @return 0 for success
+ *
+ * In the event of failure, an error is logged. A minimal
+ * useable configuration is always returned
+ */
+int
+tconfig_load_gfx(const char *filename, struct xrdp_tconfig_gfx *config);
 
 #endif