libfreerdp-core: improve bitmap codec negotiation

include/freerdp/settings.h

@@ -759,6 +759,9 @@ typedef struct _RDPDR_PARALLEL RDPDR_PARALLEL;
 #define FreeRDP_NSCodec						3712
 #define FreeRDP_NSCodecId					3713
 #define FreeRDP_FrameAcknowledge				3714
+#define FreeRDP_NSCodecColorLossLevel				3715
+#define FreeRDP_NSCodecAllowSubsampling				3716
+#define FreeRDP_NSCodecAllowDynamicColorFidelity		3717
 #define FreeRDP_JpegCodec					3776
 #define FreeRDP_JpegCodecId					3777
 #define FreeRDP_JpegQuality					3778
@@ -1271,7 +1274,10 @@ struct rdp_settings
 	ALIGN64 BOOL NSCodec; /* 3712 */
 	ALIGN64 UINT32 NSCodecId; /* 3713 */
 	ALIGN64 UINT32 FrameAcknowledge; /* 3714 */
-	UINT64 padding3776[3776 - 3715]; /* 3715 */
+	ALIGN64 UINT32 NSCodecColorLossLevel; /* 3715 */
+	ALIGN64 BOOL NSCodecAllowSubsampling; /* 3716 */
+	ALIGN64 BOOL NSCodecAllowDynamicColorFidelity; /* 3717 */
+	UINT64 padding3776[3776 - 3718]; /* 3718 */
 
 	/* JPEG */
 	ALIGN64 BOOL JpegCodec; /* 3776 */

libfreerdp/common/settings.c

@@ -976,8 +976,11 @@ BOOL freerdp_get_param_bool(rdpSettings* settings, int id)
 		case FreeRDP_NSCodec:
 			return settings->NSCodec;
 
-		case FreeRDP_FrameAcknowledge:
+		case FreeRDP_NSCodecAllowSubsampling:
-			return settings->FrameAcknowledge;
+			return settings->NSCodecAllowSubsampling;
+
+		case FreeRDP_NSCodecAllowDynamicColorFidelity:
+			return settings->NSCodecAllowDynamicColorFidelity;
 
 		case FreeRDP_JpegCodec:
 			return settings->JpegCodec;
@@ -1464,8 +1467,12 @@ int freerdp_set_param_bool(rdpSettings* settings, int id, BOOL param)
 			settings->NSCodec = param;
 			break;
 
-		case FreeRDP_FrameAcknowledge:
+		case FreeRDP_NSCodecAllowSubsampling:
-			settings->FrameAcknowledge = param;
+			settings->NSCodecAllowSubsampling = param;
+			break;
+
+		case FreeRDP_NSCodecAllowDynamicColorFidelity:
+			settings->NSCodecAllowDynamicColorFidelity = param;
 			break;
 
 		case FreeRDP_JpegCodec:
@@ -1788,6 +1795,12 @@ UINT32 freerdp_get_param_uint32(rdpSettings* settings, int id)
 		case FreeRDP_NSCodecId:
 			return settings->NSCodecId;
 
+		case FreeRDP_FrameAcknowledge:
+			return settings->FrameAcknowledge;
+
+		case FreeRDP_NSCodecColorLossLevel:
+			return settings->NSCodecColorLossLevel;
+
 		case FreeRDP_JpegCodecId:
 			return settings->JpegCodecId;
 
@@ -2095,6 +2108,14 @@ int freerdp_set_param_uint32(rdpSettings* settings, int id, UINT32 param)
 			settings->NSCodecId = param;
 			break;
 
+		case FreeRDP_FrameAcknowledge:
+			settings->FrameAcknowledge = param;
+			break;
+
+		case FreeRDP_NSCodecColorLossLevel:
+			settings->NSCodecColorLossLevel = param;
+			break;
+
 		case FreeRDP_JpegCodecId:
 			settings->JpegCodecId = param;
 			break;

libfreerdp/core/capabilities.c

@@ -2475,6 +2475,33 @@ BOOL rdp_print_surface_commands_capability_set(wStream* s, UINT16 length)
 	return TRUE;
 }
 
+void rdp_print_bitmap_codec_guid(GUID* guid)
+{
+	WLog_INFO(TAG,  "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X",
+			  guid->Data1, guid->Data2, guid->Data3,
+			  guid->Data4[0], guid->Data4[1], guid->Data4[2], guid->Data4[3],
+			  guid->Data4[4], guid->Data4[5], guid->Data4[6], guid->Data4[7]);
+}
+
+char* rdp_get_bitmap_codec_guid_name(GUID* guid)
+{
+	RPC_STATUS rpc_status;
+
+	if (UuidEqual(guid, &CODEC_GUID_REMOTEFX, &rpc_status))
+		return "CODEC_GUID_REMOTEFX";
+	else if (UuidEqual(guid, &CODEC_GUID_NSCODEC, &rpc_status))
+		return "CODEC_GUID_NSCODEC";
+	else if (UuidEqual(guid, &CODEC_GUID_IGNORE, &rpc_status))
+		return "CODEC_GUID_IGNORE";
+	else if (UuidEqual(guid, &CODEC_GUID_IMAGE_REMOTEFX, &rpc_status))
+		return "CODEC_GUID_IMAGE_REMOTEFX";
+	else if (UuidEqual(guid, &CODEC_GUID_JPEG, &rpc_status))
+		return "CODEC_GUID_JPEG";
+
+	return "CODEC_GUID_UNKNOWN";
+}
+
+
 void rdp_read_bitmap_codec_guid(wStream* s, GUID* guid)
 {
 	BYTE g[16];
@@ -2518,32 +2545,6 @@ void rdp_write_bitmap_codec_guid(wStream* s, GUID* guid)
 	Stream_Write(s, g, 16);
 }
 
-void rdp_print_bitmap_codec_guid(GUID* guid)
-{
-	WLog_INFO(TAG,  "%08X%04X%04X%02X%02X%02X%02X%02X%02X%02X%02X",
-			  guid->Data1, guid->Data2, guid->Data3,
-			  guid->Data4[0], guid->Data4[1], guid->Data4[2], guid->Data4[3],
-			  guid->Data4[4], guid->Data4[5], guid->Data4[6], guid->Data4[7]);
-}
-
-char* rdp_get_bitmap_codec_guid_name(GUID* guid)
-{
-	RPC_STATUS rpc_status;
-
-	if (UuidEqual(guid, &CODEC_GUID_REMOTEFX, &rpc_status))
-		return "CODEC_GUID_REMOTEFX";
-	else if (UuidEqual(guid, &CODEC_GUID_NSCODEC, &rpc_status))
-		return "CODEC_GUID_NSCODEC";
-	else if (UuidEqual(guid, &CODEC_GUID_IGNORE, &rpc_status))
-		return "CODEC_GUID_IGNORE";
-	else if (UuidEqual(guid, &CODEC_GUID_IMAGE_REMOTEFX, &rpc_status))
-		return "CODEC_GUID_IMAGE_REMOTEFX";
-	else if (UuidEqual(guid, &CODEC_GUID_JPEG, &rpc_status))
-		return "CODEC_GUID_JPEG";
-
-	return "CODEC_GUID_UNKNOWN";
-}
-
 /**
  * Read bitmap codecs capability set.\n
  * @msdn{dd891377}
@@ -2554,13 +2555,15 @@ char* rdp_get_bitmap_codec_guid_name(GUID* guid)
 
 BOOL rdp_read_bitmap_codecs_capability_set(wStream* s, UINT16 length, rdpSettings* settings)
 {
+	BYTE codecId;
 	GUID codecGuid;
 	RPC_STATUS rpc_status;
 	BYTE bitmapCodecCount;
 	UINT16 codecPropertiesLength;
 	UINT16 remainingLength;
-	BOOL receivedRemoteFxCodec = FALSE;
+	BOOL guidNSCodec = FALSE;
-	BOOL receivedNSCodec = FALSE;
+	BOOL guidRemoteFx = FALSE;
+	BOOL guidRemoteFxImage = FALSE;
 
 	if (length < 5)
 		return FALSE;
@@ -2575,28 +2578,7 @@ BOOL rdp_read_bitmap_codecs_capability_set(wStream* s, UINT16 length, rdpSetting
 
 		rdp_read_bitmap_codec_guid(s, &codecGuid); /* codecGuid (16 bytes) */
 
-		if (settings->ServerMode)
+		Stream_Read_UINT8(s, codecId); /* codecId (1 byte) */
-		{
-			if (UuidEqual(&codecGuid, &CODEC_GUID_REMOTEFX, &rpc_status))
-			{
-				Stream_Read_UINT8(s, settings->RemoteFxCodecId);
-				receivedRemoteFxCodec = TRUE;
-			}
-			else if (UuidEqual(&codecGuid, &CODEC_GUID_NSCODEC, &rpc_status))
-			{
-				Stream_Read_UINT8(s, settings->NSCodecId);
-				receivedNSCodec = TRUE;
-			}
-			else
-			{
-				Stream_Seek_UINT8(s); /* codecID (1 byte) */
-			}
-		}
-		else
-		{
-			Stream_Seek_UINT8(s); /* codecID (1 byte) */
-		}
-
 		Stream_Read_UINT16(s, codecPropertiesLength); /* codecPropertiesLength (2 bytes) */
 		remainingLength -= 19;
 
@@ -2605,21 +2587,154 @@ BOOL rdp_read_bitmap_codecs_capability_set(wStream* s, UINT16 length, rdpSetting
 
 		if (settings->ServerMode)
 		{
+			UINT32 beg;
+			UINT32 end;
+
+			beg = (UINT32) Stream_GetPosition(s);
+			end = beg + codecPropertiesLength;
+
 			if (UuidEqual(&codecGuid, &CODEC_GUID_REMOTEFX, &rpc_status))
 			{
-				Stream_Seek_UINT32(s); /* length */
+				UINT32 rfxCapsLength;
-				Stream_Read_UINT32(s, settings->RemoteFxCaptureFlags); /* captureFlags */
+				UINT32 rfxPropsLength;
-				Stream_Rewind(s, 8);
+				UINT32 captureFlags;
 
-				if (settings->RemoteFxCaptureFlags & CARDP_CAPS_CAPTURE_NON_CAC)
+				guidRemoteFx = TRUE;
+				settings->RemoteFxCodecId = codecId;
+
+				Stream_Read_UINT32(s, rfxPropsLength); /* length (4 bytes) */
+				Stream_Read_UINT32(s, captureFlags); /* captureFlags (4 bytes) */
+				Stream_Read_UINT32(s, rfxCapsLength); /* capsLength (4 bytes) */
+
+				settings->RemoteFxCaptureFlags = captureFlags;
+				settings->RemoteFxOnly = (captureFlags & CARDP_CAPS_CAPTURE_NON_CAC) ? TRUE : FALSE;
+
+				if (rfxCapsLength)
 				{
-					settings->RemoteFxOnly = TRUE;
+					UINT16 blockType;
+					UINT32 blockLen;
+					UINT16 numCapsets;
+					BYTE rfxCodecId;
+					UINT16 capsetType;
+					UINT16 numIcaps;
+					UINT16 icapLen;
+
+					/* TS_RFX_CAPS */
+
+					Stream_Read_UINT16(s, blockType); /* blockType (2 bytes) */
+					Stream_Read_UINT32(s, blockLen); /* blockLen (4 bytes) */
+					Stream_Read_UINT16(s, numCapsets); /* numCapsets (2 bytes) */
+
+					if (blockType != 0xCBC0)
+						return FALSE;
+
+					if (blockLen != 8)
+						return FALSE;
+
+					if (numCapsets != 1)
+						return FALSE;
+
+					/* TS_RFX_CAPSET */
+
+					Stream_Read_UINT16(s, blockType); /* blockType (2 bytes) */
+					Stream_Read_UINT32(s, blockLen); /* blockLen (4 bytes) */
+					Stream_Read_UINT8(s, rfxCodecId); /* codecId (1 byte) */
+					Stream_Read_UINT16(s, capsetType); /* capsetType (2 bytes) */
+					Stream_Read_UINT16(s, numIcaps); /* numIcaps (2 bytes) */
+					Stream_Read_UINT16(s, icapLen); /* icapLen (2 bytes) */
+
+					if (blockType != 0xCBC1)
+						return FALSE;
+
+					if (rfxCodecId != 1)
+						return FALSE;
+
+					if (capsetType != 0xCFC0)
+						return FALSE;
+
+					while (numIcaps--)
+					{
+						UINT16 version;
+						UINT16 tileSize;
+						BYTE codecFlags;
+						BYTE colConvBits;
+						BYTE transformBits;
+						BYTE entropyBits;
+
+						/* TS_RFX_ICAP */
+
+						Stream_Read_UINT16(s, version); /* version (2 bytes) */
+						Stream_Read_UINT16(s, tileSize); /* tileSize (2 bytes) */
+						Stream_Read_UINT8(s, codecFlags); /* flags (1 byte) */
+						Stream_Read_UINT8(s, colConvBits); /* colConvBits (1 byte) */
+						Stream_Read_UINT8(s, transformBits); /* transformBits (1 byte) */
+						Stream_Read_UINT8(s, entropyBits); /* entropyBits (1 byte) */
+
+						if (version != 0x0100)
+							return FALSE;
+
+						if (tileSize != 0x0040)
+							return FALSE;
+
+						if (colConvBits != 1)
+							return FALSE;
+
+						if (transformBits != 1)
+							return FALSE;
+					}
 				}
 			}
-		}
+			else if (UuidEqual(&codecGuid, &CODEC_GUID_IMAGE_REMOTEFX, &rpc_status))
+			{
+				guidRemoteFxImage = TRUE;
+				Stream_Seek(s, codecPropertiesLength); /* codecProperties */
+			}
+			else if (UuidEqual(&codecGuid, &CODEC_GUID_NSCODEC, &rpc_status))
+			{
+				BYTE colorLossLevel;
+				BYTE fAllowSubsampling;
+				BYTE fAllowDynamicFidelity;
 
-		Stream_Seek(s, codecPropertiesLength); /* codecProperties */
+				guidNSCodec = TRUE;
-		remainingLength -= codecPropertiesLength;
+				settings->NSCodecId = codecId;
+
+				Stream_Read_UINT8(s, fAllowDynamicFidelity); /* fAllowDynamicFidelity (1 byte) */
+				Stream_Read_UINT8(s, fAllowSubsampling); /* fAllowSubsampling (1 byte) */
+				Stream_Read_UINT8(s, colorLossLevel); /* colorLossLevel (1 byte) */
+
+				if (colorLossLevel < 1)
+					colorLossLevel = 1;
+
+				if (colorLossLevel > 7)
+					colorLossLevel = 7;
+
+				settings->NSCodecAllowDynamicColorFidelity = fAllowDynamicFidelity;
+				settings->NSCodecAllowSubsampling = fAllowSubsampling;
+				settings->NSCodecColorLossLevel = colorLossLevel;
+			}
+			else if (UuidEqual(&codecGuid, &CODEC_GUID_IGNORE, &rpc_status))
+			{
+				Stream_Seek(s, codecPropertiesLength); /* codecProperties */
+			}
+			else
+			{
+				Stream_Seek(s, codecPropertiesLength); /* codecProperties */
+			}
+
+			if (Stream_GetPosition(s) != end)
+			{
+				fprintf(stderr, "error while reading codec properties: actual offset: %d expected offset: %d\n",
+						Stream_GetPosition(s), end);
+				Stream_SetPosition(s, end);
+			}
+
+			remainingLength -= codecPropertiesLength;
+		}
+		else
+		{
+			Stream_Seek(s, codecPropertiesLength); /* codecProperties */
+			remainingLength -= codecPropertiesLength;
+		}
 
 		bitmapCodecCount--;
 	}
@@ -2627,8 +2742,9 @@ BOOL rdp_read_bitmap_codecs_capability_set(wStream* s, UINT16 length, rdpSetting
 	if (settings->ServerMode)
 	{
 		/* only enable a codec if we've announced/enabled it before */
-		settings->RemoteFxCodec = settings->RemoteFxCodec && receivedRemoteFxCodec;
+		settings->RemoteFxCodec = settings->RemoteFxCodec && guidRemoteFx;
-		settings->NSCodec = settings->NSCodec && receivedNSCodec;
+		settings->RemoteFxImageCodec = settings->RemoteFxImageCodec && guidRemoteFxImage;
+		settings->NSCodec = settings->NSCodec && guidNSCodec;
 		settings->JpegCodec = FALSE;
 	}
 
@@ -2699,9 +2815,9 @@ void rdp_write_nsc_client_capability_container(wStream* s, rdpSettings* settings
 	Stream_Write_UINT16(s, 3); /* codecPropertiesLength */
 
 	/* TS_NSCODEC_CAPABILITYSET */
-	Stream_Write_UINT8(s, 1);  /* fAllowDynamicFidelity */
+	Stream_Write_UINT8(s, settings->NSCodecAllowDynamicColorFidelity);  /* fAllowDynamicFidelity */
-	Stream_Write_UINT8(s, 1);  /* fAllowSubsampling */
+	Stream_Write_UINT8(s, settings->NSCodecAllowSubsampling);  /* fAllowSubsampling */
-	Stream_Write_UINT8(s, 3);  /* colorLossLevel */
+	Stream_Write_UINT8(s, settings->NSCodecColorLossLevel);  /* colorLossLevel */
 }
 
 void rdp_write_jpeg_client_capability_container(wStream* s, rdpSettings* settings)
@@ -2764,9 +2880,6 @@ void rdp_write_bitmap_codecs_capability_set(wStream* s, rdpSettings* settings)
 
 	bitmapCodecCount = 0;
 
-	if (settings->RemoteFxCodec)
-		settings->RemoteFxImageCodec = TRUE;
-
 	if (settings->RemoteFxCodec)
 		bitmapCodecCount++;
 	if (settings->NSCodec)

libfreerdp/core/settings.c

@@ -402,6 +402,10 @@ rdpSettings* freerdp_settings_new(DWORD flags)
 		settings->FrameAcknowledge = 2;
 		settings->MouseMotion = TRUE;
 
+		settings->NSCodecColorLossLevel = 3;
+		settings->NSCodecAllowSubsampling = TRUE;
+		settings->NSCodecAllowDynamicColorFidelity = TRUE;
+
 		settings->AutoReconnectionEnabled = FALSE;
 		settings->AutoReconnectMaxRetries = 20;
 
@@ -584,6 +588,7 @@ rdpSettings* freerdp_settings_clone(rdpSettings* settings)
 		_settings->RemoteFxCaptureFlags = settings->RemoteFxCaptureFlags; /* 3653 */
 		_settings->NSCodecId = settings->NSCodecId; /* 3713 */
 		_settings->FrameAcknowledge = settings->FrameAcknowledge; /* 3714 */
+		_settings->NSCodecColorLossLevel = settings->NSCodecColorLossLevel; /* 3715 */
 		_settings->JpegCodecId = settings->JpegCodecId; /* 3777 */
 		_settings->JpegQuality = settings->JpegQuality; /* 3778 */
 		_settings->BitmapCacheV3CodecId = settings->BitmapCacheV3CodecId; /* 3904 */
@@ -708,6 +713,8 @@ rdpSettings* freerdp_settings_clone(rdpSettings* settings)
 		_settings->RemoteFxCodec = settings->RemoteFxCodec; /* 3649 */
 		_settings->RemoteFxImageCodec = settings->RemoteFxImageCodec; /* 3652 */
 		_settings->NSCodec = settings->NSCodec; /* 3712 */
+		_settings->NSCodecAllowSubsampling = settings->NSCodecAllowSubsampling; /* 3716 */
+		_settings->NSCodecAllowDynamicColorFidelity = settings->NSCodecAllowDynamicColorFidelity; /* 3717 */
 		_settings->JpegCodec = settings->JpegCodec; /* 3776 */
 		_settings->GfxThinClient = settings->GfxThinClient; /* 3840 */
 		_settings->GfxSmallCache = settings->GfxSmallCache; /* 3841 */

server/shadow/shadow_client.c

@@ -159,6 +159,12 @@ BOOL shadow_client_post_connect(freerdp_peer* peer)
 	if (settings->ColorDepth == 24)
 		settings->ColorDepth = 16; /* disable 24bpp */
 
+	if (settings->ColorDepth < 32)
+	{
+		settings->NSCodec = FALSE;
+		settings->RemoteFxCodec = FALSE;
+	}
+
 	WLog_ERR(TAG, "Client from %s is activated (%dx%d@%d)",
 			peer->hostname, settings->DesktopWidth, settings->DesktopHeight, settings->ColorDepth);