mirrors/xrdp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

posix shm, can not unmap shmem_ptr until encoder is done with it

Browse Source
This commit is contained in:
Jay Sorg 2023-09-04 14:50:59 -07:00 committed by jsorg71
parent def567c2e0
commit bfdcdb0082
6 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
xrdp/xrdp.h
View File

@ -528,7 +528,8 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
int num_crects, short *crects, int num_crects, short *crects,
char *data, int left, int top, char *data, int left, int top,
int width, int height, int width, int height,
int flags, int frame_id); int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
int int
server_palette(struct xrdp_mod *mod, int *palette); server_palette(struct xrdp_mod *mod, int *palette);
int int

2
xrdp/xrdp_encoder.h
View File

@ -44,6 +44,8 @@ struct xrdp_enc_data
int height; int height;
int flags; int flags;
int frame_id; int frame_id;
void *shmem_ptr;
int shmem_bytes;
}; };
typedef struct xrdp_enc_data XRDP_ENC_DATA; typedef struct xrdp_enc_data XRDP_ENC_DATA;

42
xrdp/xrdp_mm.c
View File

@ -2928,6 +2928,7 @@ xrdp_mm_update_module_frame_ack(struct xrdp_mm *self)
static int static int
xrdp_mm_process_enc_done(struct xrdp_mm *self) xrdp_mm_process_enc_done(struct xrdp_mm *self)
{ {
XRDP_ENC_DATA *enc;
XRDP_ENC_DATA_DONE *enc_done; XRDP_ENC_DATA_DONE *enc_done;
int x; int x;
int y; int y;
@ -2976,21 +2977,26 @@ xrdp_mm_process_enc_done(struct xrdp_mm *self)
/* free enc_done */ /* free enc_done */
if (enc_done->last) if (enc_done->last)
{ {
enc = enc_done->enc;
LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: last set"); LOG_DEVEL(LOG_LEVEL_DEBUG, "xrdp_mm_process_enc_done: last set");
if (self->wm->client_info->use_frame_acks == 0) if (self->wm->client_info->use_frame_acks == 0)
{ {
self->mod->mod_frame_ack(self->mod, self->mod->mod_frame_ack(self->mod,
enc_done->enc->flags, enc->flags,
enc_done->enc->frame_id); enc->frame_id);
} }
else else
{ {
self->encoder->frame_id_server = enc_done->enc->frame_id; self->encoder->frame_id_server = enc->frame_id;
xrdp_mm_update_module_frame_ack(self); xrdp_mm_update_module_frame_ack(self);
} }
g_free(enc_done->enc->drects); g_free(enc->drects);
g_free(enc_done->enc->crects); g_free(enc->crects);
g_free(enc_done->enc); if (enc->shmem_ptr != NULL)
{
g_munmap(enc->shmem_ptr, enc->shmem_bytes);
}
g_free(enc);
} }
g_free(enc_done->comp_pad_data); g_free(enc_done->comp_pad_data);
g_free(enc_done); g_free(enc_done);
@ -3360,7 +3366,8 @@ server_paint_rects(struct xrdp_mod *mod, int num_drects, short *drects,
int height, int flags, int frame_id) int height, int flags, int frame_id)
{ {
return server_paint_rects_ex(mod, num_drects, drects, num_crects, crects, return server_paint_rects_ex(mod, num_drects, drects, num_crects, crects,
data, 0, 0, width, height, flags, frame_id); data, 0, 0, width, height, flags, frame_id,
NULL, 0);
} }
/*****************************************************************************/ /*****************************************************************************/
@ -3368,7 +3375,8 @@ int
server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects, server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
int num_crects, short *crects, char *data, int num_crects, short *crects, char *data,
int left, int top, int width, int height, int left, int top, int width, int height,
int flags, int frame_id) int flags, int frame_id,
void *shmem_ptr, int shmem_bytes)
{ {
struct xrdp_wm *wm; struct xrdp_wm *wm;
struct xrdp_mm *mm; struct xrdp_mm *mm;
@ -3389,6 +3397,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 1); enc_data = (XRDP_ENC_DATA *) g_malloc(sizeof(XRDP_ENC_DATA), 1);
if (enc_data == 0) if (enc_data == 0)
{ {
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
return 1; return 1;
} }
@ -3396,6 +3408,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
g_malloc(sizeof(short) * num_drects * 4, 0); g_malloc(sizeof(short) * num_drects * 4, 0);
if (enc_data->drects == 0) if (enc_data->drects == 0)
{ {
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
g_free(enc_data); g_free(enc_data);
return 1; return 1;
} }
@ -3404,6 +3420,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
g_malloc(sizeof(short) * num_crects * 4, 0); g_malloc(sizeof(short) * num_crects * 4, 0);
if (enc_data->crects == 0) if (enc_data->crects == 0)
{ {
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
g_free(enc_data->drects); g_free(enc_data->drects);
g_free(enc_data); g_free(enc_data);
return 1; return 1;
@ -3422,6 +3442,8 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
enc_data->height = height; enc_data->height = height;
enc_data->flags = flags; enc_data->flags = flags;
enc_data->frame_id = frame_id; enc_data->frame_id = frame_id;
enc_data->shmem_ptr = shmem_ptr;
enc_data->shmem_bytes = shmem_bytes;
if (width == 0 || height == 0) if (width == 0 || height == 0)
{ {
LOG_DEVEL(LOG_LEVEL_WARNING, "server_paint_rects: error"); LOG_DEVEL(LOG_LEVEL_WARNING, "server_paint_rects: error");
@ -3456,6 +3478,10 @@ server_paint_rects_ex(struct xrdp_mod *mod, int num_drects, short *drects,
} }
xrdp_bitmap_delete(b); xrdp_bitmap_delete(b);
mm->mod->mod_frame_ack(mm->mod, flags, frame_id); mm->mod->mod_frame_ack(mm->mod, flags, frame_id);
if (shmem_ptr != NULL)
{
g_munmap(shmem_ptr, shmem_bytes);
}
return 0; return 0;
} }

3
xrdp/xrdp_types.h
View File

@ -179,7 +179,8 @@ struct xrdp_mod
int num_crects, short *crects, int num_crects, short *crects,
char *data, int left, int top, char *data, int left, int top,
int width, int height, int width, int height,
int flags, int frame_id); int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
functions above */ functions above */
/* common */ /* common */

7
xup/xup.c
View File

@ -1386,11 +1386,14 @@ process_server_paint_rect_shmfd(struct mod *amod, struct stream *s)
{ {
bmpdata = (char *)shmem_ptr; bmpdata = (char *)shmem_ptr;
bmpdata += shmem_offset; bmpdata += shmem_offset;
/* we give up ownership of shmem_ptr
will get cleaned up in server_paint_rects_ex or
xrdp_mm_process_enc_done(rfx, gfx) */
rv = amod->server_paint_rects_ex(amod, num_drects, ldrects, rv = amod->server_paint_rects_ex(amod, num_drects, ldrects,
num_crects, lcrects, bmpdata, num_crects, lcrects, bmpdata,
left, top, width, height, left, top, width, height,
flags, frame_id); flags, frame_id,
g_munmap(shmem_ptr, shmem_bytes); shmem_ptr, shmem_bytes);
} }
g_file_close(fd); g_file_close(fd);
} }

3
xup/xup.h
View File

@ -165,7 +165,8 @@ struct mod
int num_crects, short *crects, int num_crects, short *crects,
char *data, int left, int top, char *data, int left, int top,
int width, int height, int width, int height,
int flags, int frame_id); int flags, int frame_id,
void *shmem_ptr, int shmem_bytes);
tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server tintptr server_dumby[100 - 48]; /* align, 100 minus the number of server
functions above */ functions above */
/* common */ /* common */