diff --git a/libfreerdp/codec/rfx.c b/libfreerdp/codec/rfx.c
index f105e648b..0df8d9967 100644
--- a/libfreerdp/codec/rfx.c
+++ b/libfreerdp/codec/rfx.c
@@ -161,9 +161,14 @@ RFX_CONTEXT* rfx_context_new(void)
 	 *
 	 * y_r_buffer, cb_g_buffer, cr_b_buffer: 64 * 64 * 4 = 16384 (0x4000)
 	 * dwt_buffer: 32 * 32 * 2 * 2 * 4 = 16384, maximum sub-band width is 32
+	 *
+	 * Additionally we add 32 bytes (16 in front and 16 at the back of the buffer)
+	 * in order to allow optimized functions (SEE, NEON) to read from positions 
+	 * that are actually in front/beyond the buffer. Offset calculations are
+	 * performed at the BufferPool_Take function calls in rfx_encode/decode.c.
 	 */
 
-	context->priv->BufferPool = BufferPool_New(TRUE, 16384, 16);
+	context->priv->BufferPool = BufferPool_New(TRUE, 16384 + 32, 16);
 
 #ifdef _WIN32
 	{
diff --git a/libfreerdp/codec/rfx_decode.c b/libfreerdp/codec/rfx_decode.c
index 2d9cf3481..a7c0b08a9 100644
--- a/libfreerdp/codec/rfx_decode.c
+++ b/libfreerdp/codec/rfx_decode.c
@@ -153,9 +153,9 @@ BOOL rfx_decode_rgb(RFX_CONTEXT* context, STREAM* data_in,
 
 	PROFILER_ENTER(context->priv->prof_rfx_decode_rgb);
 
-	pSrcDst[0] = BufferPool_Take(context->priv->BufferPool, -1); /* y_r_buffer */
-	pSrcDst[1] = BufferPool_Take(context->priv->BufferPool, -1); /* cb_g_buffer */
-	pSrcDst[2] = BufferPool_Take(context->priv->BufferPool, -1); /* cr_b_buffer */
+	pSrcDst[0] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* y_r_buffer */
+	pSrcDst[1] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cb_g_buffer */
+	pSrcDst[2] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cr_b_buffer */
 
 #if 0
 	if (context->priv->UseThreads)
@@ -227,8 +227,8 @@ BOOL rfx_decode_rgb(RFX_CONTEXT* context, STREAM* data_in,
 	
 	PROFILER_EXIT(context->priv->prof_rfx_decode_rgb);
 
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[0]);
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[1]);
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[2]);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[0] - 16);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[1] - 16);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[2] - 16);
 	return TRUE;
 }
diff --git a/libfreerdp/codec/rfx_encode.c b/libfreerdp/codec/rfx_encode.c
index b7ebe9d9d..e6cf10a0a 100644
--- a/libfreerdp/codec/rfx_encode.c
+++ b/libfreerdp/codec/rfx_encode.c
@@ -225,9 +225,9 @@ void rfx_encode_rgb(RFX_CONTEXT* context, const BYTE* rgb_data, int width, int h
 	primitives_t* prims = primitives_get();
 	static const prim_size_t roi_64x64 = { 64, 64 };
 
-	pSrcDst[0] = BufferPool_Take(context->priv->BufferPool, -1); /* y_r_buffer */
-	pSrcDst[1] = BufferPool_Take(context->priv->BufferPool, -1); /* cb_g_buffer */
-	pSrcDst[2] = BufferPool_Take(context->priv->BufferPool, -1); /* cr_b_buffer */
+	pSrcDst[0] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* y_r_buffer */
+	pSrcDst[1] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cb_g_buffer */
+	pSrcDst[2] = (INT16*)((BYTE*)BufferPool_Take(context->priv->BufferPool, -1) + 16); /* cr_b_buffer */
 
 	PROFILER_ENTER(context->priv->prof_rfx_encode_rgb);
 
@@ -260,7 +260,7 @@ void rfx_encode_rgb(RFX_CONTEXT* context, const BYTE* rgb_data, int width, int h
 
 	PROFILER_EXIT(context->priv->prof_rfx_encode_rgb);
 
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[0]);
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[1]);
-	BufferPool_Return(context->priv->BufferPool, pSrcDst[2]);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[0] - 16);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[1] - 16);
+	BufferPool_Return(context->priv->BufferPool, (BYTE*)pSrcDst[2] - 16);
 }