mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

vnc-tight: fix regression with libxenstore

Browse Source 
commit 095497ff added thread local storage for the color counting
palette. Unfortunately, a VncPalette is about 7kB on a x86_64 system.
This memory is reserved from the stack of every thread and it
exhausted the stack space of a libxenstore thread.

Fix this by allocating memory only for the VNC encoding thread.

Fixes: 095497ffc6
Reported-by: Juergen Gross <jgross@suse.com>
Tested-by: Juergen Gross <jgross@suse.com>
Signed-off-by: Peter Lieven <pl@kamp.de>
Message-id: 1468575911-20656-1-git-send-email-pl@kamp.de
Signed-off-by: Gerd Hoffmann <kraxel@redhat.com>
This commit is contained in:
Peter Lieven 2016-07-15 11:45:11 +02:00 committed by Gerd Hoffmann
parent 3f7e51bca3
commit 66668d197f
1 changed files with 21 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
ui/vnc-enc-tight.c
View File

@ -1458,11 +1458,17 @@ static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h,
} }
#endif #endif
static __thread VncPalette color_count_palette; static __thread VncPalette *color_count_palette;
static __thread Notifier vnc_tight_cleanup_notifier;
static void vnc_tight_cleanup(Notifier *n, void *value)
{
g_free(color_count_palette);
color_count_palette = NULL;
}
static int send_sub_rect(VncState *vs, int x, int y, int w, int h) static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
{ {
VncPalette *palette = &color_count_palette;
uint32_t bg = 0, fg = 0; uint32_t bg = 0, fg = 0;
int colors; int colors;
int ret = 0; int ret = 0;
@ -1471,6 +1477,12 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
bool allow_jpeg = true; bool allow_jpeg = true;
#endif #endif
if (!color_count_palette) {
color_count_palette = g_malloc(sizeof(VncPalette));
vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup;
qemu_thread_atexit_add(&vnc_tight_cleanup_notifier);
}
vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type); vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type);
vnc_tight_start(vs); vnc_tight_start(vs);
@ -1491,17 +1503,19 @@ static int send_sub_rect(VncState *vs, int x, int y, int w, int h)
} }
#endif #endif
colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, palette); colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette);
#ifdef CONFIG_VNC_JPEG #ifdef CONFIG_VNC_JPEG
if (allow_jpeg && vs->tight.quality != (uint8_t)-1) { if (allow_jpeg && vs->tight.quality != (uint8_t)-1) {
ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette, ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors,
force_jpeg); color_count_palette, force_jpeg);
} else { } else {
ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
color_count_palette);
} }
#else #else
ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors,
color_count_palette);
#endif #endif
return ret; return ret;