diff --git a/librfxcodec b/librfxcodec index 64df536c..d3543f26 160000 --- a/librfxcodec +++ b/librfxcodec @@ -1 +1 @@ -Subproject commit 64df536cba3d64d63f7eaa6bac1bfea77eeb478d +Subproject commit d3543f26637abae85a384983c33d7e32cddbb9d8 diff --git a/libxrdp/xrdp_caps.c b/libxrdp/xrdp_caps.c index 2ef2543c..5c5e74a5 100644 --- a/libxrdp/xrdp_caps.c +++ b/libxrdp/xrdp_caps.c @@ -23,10 +23,23 @@ #include #endif +#include + #include "libxrdp.h" #include "ms-rdpbcgr.h" #include "ms-rdperp.h" +/** + * The largest supported size for a fastpath update + * (TS_MULTIFRAGMENTUPDATE_CAPABILITYSET) we advertise to the client. This + * size is big enough for the tiles required for two 3840x2160 monitors + * without using multiple update PDUS. + * + * Consult calculate_multifragmentupdate_len() below before changing this + * value. + */ +#define MAX_MULTIFRAGMENTUPDATE_SIZE (2U * (3840 * 2160) * 16384 + 16384) + /*****************************************************************************/ static int xrdp_caps_send_monitorlayout(struct xrdp_rdp *self) @@ -844,6 +857,46 @@ xrdp_caps_process_confirm_active(struct xrdp_rdp *self, struct stream *s) LOG_DEVEL(LOG_LEVEL_TRACE, "Completed processing received [MS-RDPBCGR] TS_CONFIRM_ACTIVE_PDU"); return 0; } + +/**************************************************************************//** + * Calculate the multifragmentupdate len we advertised to the client + * for fastpath updates + * + * See [MS-RDPBCGR] 2.2.7.2.6 + * + * The basic logic is taken from freerdp 2.4. We try to use the highest + * useful request size that will allow us to pack a complete screen + * update into a single fast path PDU using any of the supported codecs. + * For RemoteFX, the client MUST use at least this value + * + * A backstop on the maximum advertised size is implemented to prevent + * extreme memory usage for large screen configurations. RDP supports a + * maximum desktop size of 32768x32768, which would cause overflow for + * 32-bit integers using a simple calculation. + * + * The codecs have to deal with the value returned by the client after + * we advertise our own value, and must not assume a complete update + * will fit in a single PDU + */ +static +unsigned int calculate_multifragmentupdate_len(const struct xrdp_rdp *self) +{ + unsigned int result = MAX_MULTIFRAGMENTUPDATE_SIZE; + + unsigned int x_tiles = (self->client_info.width + 63) / 64; + unsigned int y_tiles = (self->client_info.height + 63) / 64; + + /* Check for overflow on calculation if bad parameters are supplied */ + if ((x_tiles * y_tiles + 1) < (UINT_MAX / 16384)) + { + result = x_tiles * y_tiles * 16384; + /* and add room for headers, regions, frame markers, etc. */ + result += 16384; + } + + return result; +} + /*****************************************************************************/ int xrdp_caps_send_demand_active(struct xrdp_rdp *self) @@ -1137,13 +1190,13 @@ xrdp_caps_send_demand_active(struct xrdp_rdp *self) if (self->client_info.use_fast_path & FASTPATH_OUTPUT_SUPPORTED) /* fastpath output on */ { - /* multifragment update */ + unsigned int max_request_size = calculate_multifragmentupdate_len(self); caps_count++; out_uint16_le(s, CAPSSETTYPE_MULTIFRAGMENTUPDATE); out_uint16_le(s, CAPSSETTYPE_MULTIFRAGMENTUPDATE_LEN); - out_uint32_le(s, 3 * 1024 * 1024); /* 3MB */ + out_uint32_le(s, max_request_size); LOG_DEVEL(LOG_LEVEL_TRACE, "xrdp_caps_send_demand_active: Server Capability " - "CAPSSETTYPE_MULTIFRAGMENTUPDATE = 3MB"); + "CAPSSETTYPE_MULTIFRAGMENTUPDATE = %d", max_request_size); /* frame acks */ caps_count++; diff --git a/xrdp/xrdp_encoder.c b/xrdp/xrdp_encoder.c index bbd64cc8..3ccc9603 100644 --- a/xrdp/xrdp_encoder.c +++ b/xrdp/xrdp_encoder.c @@ -316,7 +316,10 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) int cy; int out_data_bytes; int count; - int error; + int tiles_written; + int all_tiles_written; + int tiles_left; + int finished; char *out_data; XRDP_ENC_DATA_DONE *enc_done; FIFO *fifo_processed; @@ -333,87 +336,104 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc) mutex = self->mutex; event_processed = self->xrdp_encoder_event_processed; - error = 1; - out_data = NULL; - out_data_bytes = 0; - - if ((enc->num_crects > 0) && (enc->num_drects > 0)) + all_tiles_written = 0; + do { - alloc_bytes = XRDP_SURCMD_PREFIX_BYTES; - alloc_bytes += self->max_compressed_bytes; - alloc_bytes += sizeof(struct rfx_tile) * enc->num_crects + - sizeof(struct rfx_rect) * enc->num_drects; - out_data = g_new(char, alloc_bytes); - if (out_data != NULL) + tiles_written = 0; + tiles_left = enc->num_crects - all_tiles_written; + out_data = NULL; + out_data_bytes = 0; + + if ((tiles_left > 0) && (enc->num_drects > 0)) { - tiles = (struct rfx_tile *) - (out_data + XRDP_SURCMD_PREFIX_BYTES + - self->max_compressed_bytes); - rfxrects = (struct rfx_rect *) (tiles + enc->num_crects); - - count = enc->num_crects; - for (index = 0; index < count; index++) + alloc_bytes = XRDP_SURCMD_PREFIX_BYTES; + alloc_bytes += self->max_compressed_bytes; + alloc_bytes += sizeof(struct rfx_tile) * tiles_left + + sizeof(struct rfx_rect) * enc->num_drects; + out_data = g_new(char, alloc_bytes); + if (out_data != NULL) { - x = enc->crects[index * 4 + 0]; - y = enc->crects[index * 4 + 1]; - cx = enc->crects[index * 4 + 2]; - cy = enc->crects[index * 4 + 3]; - tiles[index].x = x; - tiles[index].y = y; - tiles[index].cx = cx; - tiles[index].cy = cy; - tiles[index].quant_y = 0; - tiles[index].quant_cb = 0; - tiles[index].quant_cr = 0; - } + tiles = (struct rfx_tile *) + (out_data + XRDP_SURCMD_PREFIX_BYTES + + self->max_compressed_bytes); + rfxrects = (struct rfx_rect *) (tiles + tiles_left); - count = enc->num_drects; - for (index = 0; index < count; index++) - { - x = enc->drects[index * 4 + 0]; - y = enc->drects[index * 4 + 1]; - cx = enc->drects[index * 4 + 2]; - cy = enc->drects[index * 4 + 3]; - rfxrects[index].x = x; - rfxrects[index].y = y; - rfxrects[index].cx = cx; - rfxrects[index].cy = cy; - } + count = tiles_left; + for (index = 0; index < count; index++) + { + x = enc->crects[(index + all_tiles_written) * 4 + 0]; + y = enc->crects[(index + all_tiles_written) * 4 + 1]; + cx = enc->crects[(index + all_tiles_written) * 4 + 2]; + cy = enc->crects[(index + all_tiles_written) * 4 + 3]; + tiles[index].x = x; + tiles[index].y = y; + tiles[index].cx = cx; + tiles[index].cy = cy; + tiles[index].quant_y = 0; + tiles[index].quant_cb = 0; + tiles[index].quant_cr = 0; + } - out_data_bytes = self->max_compressed_bytes; - error = rfxcodec_encode(self->codec_handle, - out_data + XRDP_SURCMD_PREFIX_BYTES, - &out_data_bytes, enc->data, - enc->width, enc->height, enc->width * 4, - rfxrects, enc->num_drects, - tiles, enc->num_crects, 0, 0); + count = enc->num_drects; + for (index = 0; index < count; index++) + { + x = enc->drects[index * 4 + 0]; + y = enc->drects[index * 4 + 1]; + cx = enc->drects[index * 4 + 2]; + cy = enc->drects[index * 4 + 3]; + rfxrects[index].x = x; + rfxrects[index].y = y; + rfxrects[index].cx = cx; + rfxrects[index].cy = cy; + } + + out_data_bytes = self->max_compressed_bytes; + tiles_written = rfxcodec_encode(self->codec_handle, + out_data + XRDP_SURCMD_PREFIX_BYTES, + &out_data_bytes, enc->data, + enc->width, enc->height, enc->width * 4, + rfxrects, enc->num_drects, + tiles, tiles_left, 0, 0); + } } - } - LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_rfx: rfxcodec_encode rv %d", error); - /* only if enc_done->comp_bytes is not zero is something sent - to the client but you must always send something back even - on error so Xorg can get ack */ - enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); - if (enc_done == NULL) - { - return 1; - } - enc_done->comp_bytes = error == 0 ? out_data_bytes : 0; - enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; - enc_done->comp_pad_data = out_data; - enc_done->enc = enc; - enc_done->last = 1; - enc_done->cx = self->mm->wm->screen->width; - enc_done->cy = self->mm->wm->screen->height; + LOG_DEVEL(LOG_LEVEL_DEBUG, + "process_enc_rfx: rfxcodec_encode tiles_written %d", + tiles_written); + /* only if enc_done->comp_bytes is not zero is something sent + to the client but you must always send something back even + on error so Xorg can get ack */ + enc_done = g_new0(XRDP_ENC_DATA_DONE, 1); + if (enc_done == NULL) + { + return 1; + } + enc_done->comp_bytes = tiles_written > 0 ? out_data_bytes : 0; + enc_done->pad_bytes = XRDP_SURCMD_PREFIX_BYTES; + enc_done->comp_pad_data = out_data; + enc_done->enc = enc; + enc_done->cx = self->mm->wm->screen->width; + enc_done->cy = self->mm->wm->screen->height; - /* done with msg */ - /* inform main thread done */ - tc_mutex_lock(mutex); - fifo_add_item(fifo_processed, enc_done); - tc_mutex_unlock(mutex); - /* signal completion for main thread */ - g_set_wait_obj(event_processed); + enc_done->continuation = all_tiles_written > 0; + if (tiles_written > 0) + { + all_tiles_written += tiles_written; + } + finished = + (all_tiles_written == enc->num_crects) || (tiles_written < 0); + enc_done->last = finished; + + /* done with msg */ + /* inform main thread done */ + tc_mutex_lock(mutex); + fifo_add_item(fifo_processed, enc_done); + tc_mutex_unlock(mutex); + /* signal completion for main thread */ + g_set_wait_obj(event_processed); + + } + while (!finished); return 0; } diff --git a/xrdp/xrdp_encoder.h b/xrdp/xrdp_encoder.h index a84a48e1..31260b34 100644 --- a/xrdp/xrdp_encoder.h +++ b/xrdp/xrdp_encoder.h @@ -54,6 +54,7 @@ struct xrdp_enc_data_done char *comp_pad_data; struct xrdp_enc_data *enc; int last; /* true is this is last message for enc */ + int continuation; /* true if this isn't the start of a frame */ int x; int y; int cx; diff --git a/xrdp/xrdp_mm.c b/xrdp/xrdp_mm.c index 13a165f0..2df0dfed 100644 --- a/xrdp/xrdp_mm.c +++ b/xrdp/xrdp_mm.c @@ -2820,8 +2820,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) cy = enc_done->cy; if (enc_done->comp_bytes > 0) { - libxrdp_fastpath_send_frame_marker(self->wm->session, 0, - enc_done->enc->frame_id); + if (!enc_done->continuation) + { + libxrdp_fastpath_send_frame_marker(self->wm->session, 0, + enc_done->enc->frame_id); + } libxrdp_fastpath_send_surface(self->wm->session, enc_done->comp_pad_data, enc_done->pad_bytes, @@ -2829,8 +2832,11 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self) x, y, x + cx, y + cy, 32, self->encoder->codec_id, cx, cy); - libxrdp_fastpath_send_frame_marker(self->wm->session, 1, - enc_done->enc->frame_id); + if (enc_done->last) + { + libxrdp_fastpath_send_frame_marker(self->wm->session, 1, + enc_done->enc->frame_id); + } } /* free enc_done */ if (enc_done->last)