shadow: reduce resource usage in encoder

server/shadow/X11/x11_shadow.c

@@ -153,7 +153,7 @@ void x11_shadow_validate_region(x11ShadowSubsystem* subsystem, int x, int y, int
 {
 	XRectangle region;
 
-	if (!subsystem->use_xfixes)
+	if (!subsystem->use_xfixes || !subsystem->use_xdamage)
 		return;
 
 	region.x = x;
@@ -260,9 +260,14 @@ int x11_shadow_surface_copy(x11ShadowSubsystem* subsystem)
 
 void* x11_shadow_subsystem_thread(x11ShadowSubsystem* subsystem)
 {
+	int fps;
 	DWORD status;
 	DWORD nCount;
 	XEvent xevent;
+	UINT64 cTime;
+	DWORD dwTimeout;
+	DWORD dwInterval;
+	UINT64 frameTime;
 	HANDLE events[32];
 	HANDLE StopEvent;
 	int x, y, width, height;
@@ -274,9 +279,21 @@ void* x11_shadow_subsystem_thread(x11ShadowSubsystem* subsystem)
 	events[nCount++] = StopEvent;
 	events[nCount++] = subsystem->event;
 
+	fps = 16;
+	dwInterval = 1000 / fps;
+	frameTime = GetTickCount64() + dwInterval;
+
 	while (1)
 	{
-		status = WaitForMultipleObjects(nCount, events, FALSE, INFINITE);
+		dwTimeout = INFINITE;
+
+		if (!subsystem->use_xdamage)
+		{
+			cTime = GetTickCount64();
+			dwTimeout = (cTime > frameTime) ? 0 : frameTime - cTime;
+		}
+
+		status = WaitForMultipleObjects(nCount, events, FALSE, dwTimeout);
 
 		if (WaitForSingleObject(StopEvent, 0) == WAIT_OBJECT_0)
 		{
@@ -300,6 +317,17 @@ void* x11_shadow_subsystem_thread(x11ShadowSubsystem* subsystem)
 				x11_shadow_invalidate_region(subsystem, x, y, width, height);
 			}
 		}
+
+		if (!subsystem->use_xdamage)
+		{
+			if ((status == WAIT_TIMEOUT) || (GetTickCount64() > frameTime))
+			{
+				x11_shadow_invalidate_region(subsystem, 0, 0, subsystem->width, subsystem->height);
+
+				dwInterval = 1000 / fps;
+				frameTime += dwInterval;
+			}
+		}
 	}
 
 	ExitThread(0);
@@ -495,6 +523,8 @@ int x11_shadow_subsystem_init(x11ShadowSubsystem* subsystem)
 	int i;
 	int pf_count;
 	int vi_count;
+	int nextensions;
+	char** extensions;
 	XVisualInfo* vi;
 	XVisualInfo* vis;
 	XVisualInfo template;
@@ -524,6 +554,25 @@ int x11_shadow_subsystem_init(x11ShadowSubsystem* subsystem)
 		return -1;
 	}
 
+	extensions = XListExtensions(subsystem->display, &nextensions);
+
+	if (!extensions || (nextensions < 0))
+		return -1;
+
+	for (i = 0; i < nextensions; i++)
+	{
+		if (strcmp(extensions[i], "Composite") == 0)
+			subsystem->composite = TRUE;
+	}
+
+	XFreeExtensionList(extensions);
+
+	if (subsystem->composite)
+		subsystem->use_xdamage = FALSE;
+
+	if (!subsystem->use_xdamage)
+		subsystem->use_xfixes = FALSE;
+
 	subsystem->xfds = ConnectionNumber(subsystem->display);
 	subsystem->number = DefaultScreen(subsystem->display);
 	subsystem->screen = ScreenOfDisplay(subsystem->display, subsystem->number);
@@ -706,9 +755,10 @@ x11ShadowSubsystem* x11_shadow_subsystem_new(rdpShadowServer* server)
 	subsystem->MouseEvent = (pfnShadowMouseEvent) x11_shadow_input_mouse_event;
 	subsystem->ExtendedMouseEvent = (pfnShadowExtendedMouseEvent) x11_shadow_input_extended_mouse_event;
 
+	subsystem->composite = FALSE;
 	subsystem->use_xshm = TRUE;
 	subsystem->use_xfixes = TRUE;
-	subsystem->use_xdamage = TRUE;
+	subsystem->use_xdamage = FALSE;
 	subsystem->use_xinerama = TRUE;
 
 	return subsystem;

server/shadow/X11/x11_shadow.h

@@ -68,6 +68,7 @@ struct x11_shadow_subsystem
 	Visual* visual;
 	Display* display;
 	int scanline_pad;
+	BOOL composite;
 
 	BOOL use_xshm;
 	BOOL use_xfixes;

server/shadow/shadow_client.c

@@ -248,7 +248,7 @@ int shadow_client_send_surface_bits(rdpShadowClient* client, rdpShadowSurface* s
 		RFX_RECT rect;
 		RFX_MESSAGE* messages;
 
-		s = encoder->rfx_s;
+		s = encoder->bs;
 
 		rect.x = nXSrc;
 		rect.y = nYSrc;
@@ -288,7 +288,7 @@ int shadow_client_send_surface_bits(rdpShadowClient* client, rdpShadowSurface* s
 	{
 		NSC_MESSAGE* messages;
 
-		s = encoder->nsc_s;
+		s = encoder->bs;
 
 		messages = nsc_encode_messages(encoder->nsc, pSrcData,
 				nXSrc, nYSrc, nWidth, nHeight, nSrcStep,
@@ -550,12 +550,14 @@ int shadow_client_send_surface_update(rdpShadowClient* client)
 	rdpSettings* settings;
 	rdpShadowServer* server;
 	rdpShadowSurface* surface;
+	rdpShadowEncoder* encoder;
 	RECTANGLE_16 surfaceRect;
 	const RECTANGLE_16* extents;
 
 	context = (rdpContext*) client;
 	settings = context->settings;
 	server = client->server;
+	encoder = server->encoder;
 
 	surface = client->inLobby ? client->lobby : server->surface;
 
@@ -578,10 +580,17 @@ int shadow_client_send_surface_update(rdpShadowClient* client)
 
 	if (settings->RemoteFxCodec || settings->NSCodec)
 	{
+		if (settings->RemoteFxCodec)
+			shadow_encoder_prepare(encoder, SHADOW_CODEC_REMOTEFX);
+		else if (settings->NSCodec)
+			shadow_encoder_prepare(encoder, SHADOW_CODEC_NSCODEC);
+
 		status = shadow_client_send_surface_bits(client, surface, nXSrc, nYSrc, nWidth, nHeight);
 	}
 	else
 	{
+		shadow_encoder_prepare(encoder, SHADOW_CODEC_BITMAP);
+
 		status = shadow_client_send_bitmap_update(client, surface, nXSrc, nYSrc, nWidth, nHeight);
 	}
 

server/shadow/shadow_encoder.c

@@ -58,7 +58,7 @@ int shadow_encoder_create_frame_id(rdpShadowEncoder* encoder)
 	return (int) frame->frameId;
 }
 
-int shadow_encoder_grid_init(rdpShadowEncoder* encoder)
+int shadow_encoder_init_grid(rdpShadowEncoder* encoder)
 {
 	int i, j, k;
 	int tileSize;
@@ -92,7 +92,7 @@ int shadow_encoder_grid_init(rdpShadowEncoder* encoder)
 	return 0;
 }
 
-int shadow_encoder_grid_uninit(rdpShadowEncoder* encoder)
+int shadow_encoder_uninit_grid(rdpShadowEncoder* encoder)
 {
 	if (encoder->gridBuffer)
 	{
@@ -112,59 +112,145 @@ int shadow_encoder_grid_uninit(rdpShadowEncoder* encoder)
 	return 0;
 }
 
-int shadow_encoder_init(rdpShadowEncoder* encoder)
+int shadow_encoder_init_rfx(rdpShadowEncoder* encoder)
 {
-	DWORD planarFlags;
+	if (!encoder->rfx)
+		encoder->rfx = rfx_context_new(TRUE);
 
-	encoder->scanline = (encoder->width + (encoder->width % 4)) * 4;
+	if (!encoder->rfx)
-
-	encoder->data = (BYTE*) malloc(encoder->scanline * encoder->height);
-
-	if (!encoder->data)
 		return -1;
 
-	encoder->maxTileWidth = 64;
-	encoder->maxTileHeight = 64;
-
-	encoder->bs = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
-	encoder->bts = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
-
-	planarFlags = PLANAR_FORMAT_HEADER_NA;
-	planarFlags |= PLANAR_FORMAT_HEADER_RLE;
-
-	encoder->planar = freerdp_bitmap_planar_context_new(planarFlags,
-			encoder->maxTileWidth, encoder->maxTileHeight);
-
-	encoder->rfx = rfx_context_new(TRUE);
-	encoder->rfx_s = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
-
 	encoder->rfx->mode = RLGR3;
 	encoder->rfx->width = encoder->width;
 	encoder->rfx->height = encoder->height;
 
-	encoder->nsc = nsc_context_new();
-	encoder->nsc_s = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
-
 	rfx_context_set_pixel_format(encoder->rfx, RDP_PIXEL_FORMAT_B8G8R8A8);
-	nsc_context_set_pixel_format(encoder->nsc, RDP_PIXEL_FORMAT_B8G8R8A8);
 
-	shadow_encoder_grid_init(encoder);
+	if (!encoder->frameList)
+	{
+		encoder->fps = 16;
+		encoder->maxFps = 32;
+		encoder->frameId = 0;
+		encoder->frameAck = TRUE;
+		encoder->frameList = ListDictionary_New(TRUE);
+	}
 
-	encoder->fps = 10;
+	encoder->codecs |= SHADOW_CODEC_REMOTEFX;
-	encoder->maxFps = 32;
-	encoder->frameId = 0;
-	encoder->frameAck = TRUE;
-	encoder->frameList = ListDictionary_New(TRUE);
 
 	return 1;
 }
 
-int shadow_encoder_uninit(rdpShadowEncoder* encoder)
+int shadow_encoder_init_nsc(rdpShadowEncoder* encoder)
 {
-	if (encoder->bs)
+	if (!encoder->nsc)
+		encoder->nsc = nsc_context_new();
+
+	if (!encoder->nsc)
+		return -1;
+
+	nsc_context_set_pixel_format(encoder->nsc, RDP_PIXEL_FORMAT_B8G8R8A8);
+
+	if (!encoder->frameList)
 	{
-		Stream_Free(encoder->bs, TRUE);
+		encoder->fps = 16;
-		encoder->bs = NULL;
+		encoder->maxFps = 32;
+		encoder->frameId = 0;
+		encoder->frameAck = TRUE;
+		encoder->frameList = ListDictionary_New(TRUE);
+	}
+
+	encoder->codecs |= SHADOW_CODEC_NSCODEC;
+
+	return 1;
+}
+
+int shadow_encoder_init_bitmap(rdpShadowEncoder* encoder)
+{
+	DWORD planarFlags;
+
+	planarFlags = PLANAR_FORMAT_HEADER_NA;
+	planarFlags |= PLANAR_FORMAT_HEADER_RLE;
+
+	if (!encoder->planar)
+	{
+		encoder->planar = freerdp_bitmap_planar_context_new(planarFlags,
+				encoder->maxTileWidth, encoder->maxTileHeight);
+	}
+
+	if (!encoder->planar)
+		return -1;
+
+	if (!encoder->bts)
+		encoder->bts = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
+
+	if (!encoder->bts)
+		return -1;
+
+	encoder->codecs |= SHADOW_CODEC_BITMAP;
+
+	return 1;
+}
+
+int shadow_encoder_init(rdpShadowEncoder* encoder)
+{
+	encoder->maxTileWidth = 64;
+	encoder->maxTileHeight = 64;
+
+	shadow_encoder_init_grid(encoder);
+
+	if (!encoder->bs)
+		encoder->bs = Stream_New(NULL, encoder->maxTileWidth * encoder->maxTileHeight * 4);
+
+	if (!encoder->bs)
+		return -1;
+
+	return 1;
+}
+
+int shadow_encoder_uninit_rfx(rdpShadowEncoder* encoder)
+{
+	if (encoder->rfx)
+	{
+		rfx_context_free(encoder->rfx);
+		encoder->rfx = NULL;
+	}
+
+	if (encoder->frameList)
+	{
+		ListDictionary_Free(encoder->frameList);
+		encoder->frameList = NULL;
+	}
+
+	encoder->codecs &= ~SHADOW_CODEC_REMOTEFX;
+
+	return 1;
+}
+
+int shadow_encoder_uninit_nsc(rdpShadowEncoder* encoder)
+{
+	if (encoder->nsc)
+	{
+		nsc_context_free(encoder->nsc);
+		encoder->nsc = NULL;
+	}
+
+	if (encoder->frameList)
+	{
+		ListDictionary_Free(encoder->frameList);
+		encoder->frameList = NULL;
+	}
+
+	encoder->codecs &= ~SHADOW_CODEC_NSCODEC;
+
+	return 1;
+}
+
+int shadow_encoder_uninit_bitmap(rdpShadowEncoder* encoder)
+{
+	if (encoder->planar)
+	{
+		freerdp_bitmap_planar_context_free(encoder->planar);
+		encoder->planar = NULL;
 	}
 
 	if (encoder->bts)
@@ -173,42 +259,34 @@ int shadow_encoder_uninit(rdpShadowEncoder* encoder)
 		encoder->bts = NULL;
 	}
 
-	if (encoder->rfx_s)
+	encoder->codecs &= ~SHADOW_CODEC_BITMAP;
+
+	return 1;
+}
+
+int shadow_encoder_uninit(rdpShadowEncoder* encoder)
+{
+	shadow_encoder_uninit_grid(encoder);
+
+	if (encoder->bs)
 	{
-		Stream_Free(encoder->rfx_s, TRUE);
+		Stream_Free(encoder->bs, TRUE);
-		encoder->rfx_s = NULL;
+		encoder->bs = NULL;
 	}
 
-	if (encoder->rfx)
+	if (encoder->codecs & SHADOW_CODEC_REMOTEFX)
 	{
-		rfx_context_free(encoder->rfx);
+		shadow_encoder_uninit_rfx(encoder);
-		encoder->rfx = NULL;
 	}
 
-	if (encoder->nsc_s)
+	if (encoder->codecs & SHADOW_CODEC_NSCODEC)
 	{
-		Stream_Free(encoder->nsc_s, TRUE);
+		shadow_encoder_uninit_nsc(encoder);
-		encoder->nsc_s = NULL;
 	}
 
-	if (encoder->nsc)
+	if (encoder->codecs & SHADOW_CODEC_BITMAP)
 	{
-		nsc_context_free(encoder->nsc);
+		shadow_encoder_uninit_bitmap(encoder);
-		encoder->nsc = NULL;
-	}
-
-	if (encoder->planar)
-	{
-		freerdp_bitmap_planar_context_free(encoder->planar);
-		encoder->planar = NULL;
-	}
-
-	shadow_encoder_grid_uninit(encoder);
-
-	if (encoder->frameList)
-	{
-		ListDictionary_Free(encoder->frameList);
-		encoder->frameList = NULL;
 	}
 
 	return 1;
@@ -216,10 +294,56 @@ int shadow_encoder_uninit(rdpShadowEncoder* encoder)
 
 int shadow_encoder_reset(rdpShadowEncoder* encoder)
 {
-	if (shadow_encoder_uninit(encoder) < 0)
+	int status;
+	UINT32 codecs = encoder->codecs;
+
+	status = shadow_encoder_uninit(encoder);
+
+	if (status < 0)
 		return -1;
 
-	return shadow_encoder_init(encoder);
+	status = shadow_encoder_init(encoder);
+
+	if (status < 0)
+		return -1;
+
+	status = shadow_encoder_prepare(encoder, codecs);
+
+	if (status < 0)
+		return -1;
+
+	return 1;
+}
+
+int shadow_encoder_prepare(rdpShadowEncoder* encoder, UINT32 codecs)
+{
+	int status;
+
+	if ((codecs & SHADOW_CODEC_REMOTEFX) && !(encoder->codecs & SHADOW_CODEC_REMOTEFX))
+	{
+		status = shadow_encoder_init_rfx(encoder);
+
+		if (status < 0)
+			return -1;
+	}
+
+	if ((codecs & SHADOW_CODEC_NSCODEC) && !(encoder->codecs & SHADOW_CODEC_NSCODEC))
+	{
+		status = shadow_encoder_init_nsc(encoder);
+
+		if (status < 0)
+			return -1;
+	}
+
+	if ((codecs & SHADOW_CODEC_BITMAP) && !(encoder->codecs & SHADOW_CODEC_BITMAP))
+	{
+		status = shadow_encoder_init_bitmap(encoder);
+
+		if (status < 0)
+			return -1;
+	}
+
+	return 1;
 }
 
 rdpShadowEncoder* shadow_encoder_new(rdpShadowServer* server)
@@ -236,9 +360,6 @@ rdpShadowEncoder* shadow_encoder_new(rdpShadowServer* server)
 	encoder->width = server->screen->width;
 	encoder->height = server->screen->height;
 
-	encoder->bitsPerPixel = 32;
-	encoder->bytesPerPixel = 4;
-
 	if (shadow_encoder_init(encoder) < 0)
 		return NULL;
 

server/shadow/shadow_encoder.h

@@ -29,17 +29,17 @@
 
 #include <freerdp/server/shadow.h>
 
+#define SHADOW_CODEC_REMOTEFX		1
+#define SHADOW_CODEC_NSCODEC		2
+#define SHADOW_CODEC_BITMAP		4
+
 struct rdp_shadow_encoder
 {
 	rdpShadowServer* server;
 
-	BYTE* data;
 	int width;
 	int height;
-	int scanline;
+	UINT32 codecs;
-
-	UINT32 bitsPerPixel;
-	UINT32 bytesPerPixel;
 
 	BYTE** grid;
 	int gridWidth;
@@ -48,14 +48,11 @@ struct rdp_shadow_encoder
 	int maxTileWidth;
 	int maxTileHeight;
 
-	wStream* rfx_s;
-	RFX_CONTEXT* rfx;
-
-	wStream* nsc_s;
-	NSC_CONTEXT* nsc;
-
 	wStream* bs;
 	wStream* bts;
+
+	RFX_CONTEXT* rfx;
+	NSC_CONTEXT* nsc;
 	BITMAP_PLANAR_CONTEXT* planar;
 
 	int fps;
@@ -70,6 +67,7 @@ extern "C" {
 #endif
 
 int shadow_encoder_reset(rdpShadowEncoder* encoder);
+int shadow_encoder_prepare(rdpShadowEncoder* encoder, UINT32 codecs);
 int shadow_encoder_create_frame_id(rdpShadowEncoder* encoder);
 
 rdpShadowEncoder* shadow_encoder_new(rdpShadowServer* server);