From 8a6b6133ec8d491b5b7c6424c654a422ec68ba73 Mon Sep 17 00:00:00 2001
From: bkaradzic <branimirkaradzic@gmail.com>
Date: Sun, 14 Apr 2013 18:18:11 -0700
Subject: [PATCH] Fixed stencil clear when using clear quad.

---
 premake/bgfx.lua       |  2 +-
 src/bgfx.cpp           |  2 ++
 src/renderer_d3d11.cpp | 15 +++++++++++++--
 src/renderer_gl.cpp    | 32 +++++++++++++++++++++++++++-----
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/premake/bgfx.lua b/premake/bgfx.lua
index 843156aec..3888447ce 100755
--- a/premake/bgfx.lua
+++ b/premake/bgfx.lua
@@ -30,7 +30,7 @@ project "bgfx"
 			BGFX_DIR .. "src/**.mm",
 		}
 
-	configuration { "(not nacl*) and (not qnx*)" }
+	configuration { "vs*" }
 		includedirs {
 			--nacl has GLES2 headers modified...
 			BGFX_DIR .. "3rdparty/glext",
diff --git a/src/bgfx.cpp b/src/bgfx.cpp
index efa0afe3d..2293c55c5 100755
--- a/src/bgfx.cpp
+++ b/src/bgfx.cpp
@@ -1252,12 +1252,14 @@ namespace bgfx
 	void setViewClear(uint8_t _id, uint8_t _flags, uint32_t _rgba, float _depth, uint8_t _stencil)
 	{
 		BGFX_CHECK_MAIN_THREAD();
+		BX_CHECK(_flags != BGFX_CLEAR_NONE, "At least one clear bit flag must be used.");
 		s_ctx.setViewClear(_id, _flags, _rgba, _depth, _stencil);
 	}
 
 	void setViewClearMask(uint32_t _viewMask, uint8_t _flags, uint32_t _rgba, float _depth, uint8_t _stencil)
 	{
 		BGFX_CHECK_MAIN_THREAD();
+		BX_CHECK(_flags != BGFX_CLEAR_NONE, "At least one clear bit flag must be used.");
 		s_ctx.setViewClearMask(_viewMask, _flags, _rgba, _depth, _stencil);
 	}
 
diff --git a/src/renderer_d3d11.cpp b/src/renderer_d3d11.cpp
index dba76d96f..7e812a891 100644
--- a/src/renderer_d3d11.cpp
+++ b/src/renderer_d3d11.cpp
@@ -768,7 +768,7 @@ namespace bgfx
 
 			uint32_t fstencil = unpackStencil(0, _stencil);
 			uint32_t ref = (fstencil&BGFX_STENCIL_FUNC_REF_MASK)>>BGFX_STENCIL_FUNC_REF_SHIFT;
-			_stencil &= packStencil(BGFX_STENCIL_FUNC_REF_MASK, BGFX_STENCIL_MASK);
+			_stencil &= packStencil(~BGFX_STENCIL_FUNC_REF_MASK, BGFX_STENCIL_MASK);
 
 			HashMurmur2A murmur;
 			murmur.begin();
@@ -1339,8 +1339,19 @@ namespace bgfx
 			state |= _clear.m_flags & BGFX_CLEAR_COLOR_BIT ? BGFX_STATE_RGB_WRITE|BGFX_STATE_ALPHA_WRITE : 0;
 			state |= _clear.m_flags & BGFX_CLEAR_DEPTH_BIT ? BGFX_STATE_DEPTH_TEST_ALWAYS|BGFX_STATE_DEPTH_WRITE : 0;
 
+			uint64_t stencil = 0;
+			stencil |= _clear.m_flags & BGFX_CLEAR_STENCIL_BIT ? 0
+				| BGFX_STENCIL_TEST_ALWAYS
+				| BGFX_STENCIL_FUNC_REF(_clear.m_stencil)
+				| BGFX_STENCIL_FUNC_RMASK(0xff)
+				| BGFX_STENCIL_OP_FAIL_S_REPLACE
+				| BGFX_STENCIL_OP_FAIL_Z_REPLACE
+				| BGFX_STENCIL_OP_PASS_Z_REPLACE
+				: 0
+				;
+
 			s_renderCtx.setBlendState(state);
-			s_renderCtx.setDepthStencilState(state);
+			s_renderCtx.setDepthStencilState(state, stencil);
 			s_renderCtx.setRasterizerState(state, false);
 
 			Program& program = s_renderCtx.m_program[m_program.idx];
diff --git a/src/renderer_gl.cpp b/src/renderer_gl.cpp
index 7a855fe32..dbb3257d6 100644
--- a/src/renderer_gl.cpp
+++ b/src/renderer_gl.cpp
@@ -1938,13 +1938,35 @@ namespace bgfx
 #if BGFX_CONFIG_CLEAR_QUAD
 		if (s_renderCtx.m_useClearQuad)
 		{
-			GL_CHECK(glDisable(GL_STENCIL_TEST) );
-			GL_CHECK(glEnable(GL_DEPTH_TEST) );
-			GL_CHECK(glDepthFunc(GL_ALWAYS) );
 			GL_CHECK(glDisable(GL_CULL_FACE) );
 			GL_CHECK(glDisable(GL_BLEND) );
-			GL_CHECK(glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE) );
-			GL_CHECK(glDepthMask(GL_TRUE) );
+
+			GLboolean colorMask = !!(BGFX_CLEAR_COLOR_BIT & _clear.m_flags);
+			GL_CHECK(glColorMask(colorMask, colorMask, colorMask, colorMask) );
+
+			GLboolean depthMask = !!(BGFX_CLEAR_DEPTH_BIT & _clear.m_flags);
+			if (depthMask)
+			{
+				GL_CHECK(glEnable(GL_DEPTH_TEST) );
+				GL_CHECK(glDepthFunc(GL_ALWAYS) );
+				GL_CHECK(glDepthMask(depthMask) );
+			}
+			else
+			{
+				GL_CHECK(glDisable(GL_DEPTH_TEST) );
+			}
+
+			GLboolean stencilMask = !!(BGFX_CLEAR_STENCIL_BIT & _clear.m_flags);
+			if (stencilMask)
+			{
+				GL_CHECK(glEnable(GL_STENCIL_TEST) );
+				GL_CHECK(glStencilFuncSeparate(GL_FRONT_AND_BACK, GL_ALWAYS, _clear.m_stencil,  0xff) );
+				GL_CHECK(glStencilOpSeparate(GL_FRONT_AND_BACK, GL_REPLACE, GL_REPLACE, GL_REPLACE) );
+			}
+			else
+			{
+				GL_CHECK(glDisable(GL_STENCIL_TEST) );
+			}
 
 			VertexBuffer& vb = s_renderCtx.m_vertexBuffers[m_vb->handle.idx];
 			VertexDecl& vertexDecl = s_renderCtx.m_vertexDecls[m_vb->decl.idx];