From 9543f51ccea565701bcc574590195f935109adcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sami=20Ky=C3=B6stil=C3=A4?= Date: Wed, 30 Sep 2020 23:26:58 +0100 Subject: [PATCH] Correctly decode invalid program handles When an invalid program handle is encoded (e.g., with handle 0xffff), it will overflow the space reserved for program handles in the sorting key. This patch fixes the issue by turning any out-of-range program handles back to invalid handles when decoding sorting keys. --- src/bgfx_p.h | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/bgfx_p.h b/src/bgfx_p.h index 17f0a1aa6..1e8595503 100644 --- a/src/bgfx_p.h +++ b/src/bgfx_p.h @@ -1161,23 +1161,33 @@ namespace bgfx uint64_t type = _key & kSortKeyDrawTypeMask; if (type == kSortKeyDrawTypeDepth) { - m_program.idx = uint16_t( (_key & kSortKeyDraw1ProgramMask) >> kSortKeyDraw1ProgramShift); + m_program = decodeProgram( _key, kSortKeyDraw1ProgramMask, kSortKeyDraw1ProgramShift); return false; } else if (type == kSortKeyDrawTypeSequence) { - m_program.idx = uint16_t( (_key & kSortKeyDraw2ProgramMask) >> kSortKeyDraw2ProgramShift); + m_program = decodeProgram( _key, kSortKeyDraw2ProgramMask, kSortKeyDraw2ProgramShift); return false; } - m_program.idx = uint16_t( (_key & kSortKeyDraw0ProgramMask) >> kSortKeyDraw0ProgramShift); + m_program = decodeProgram( _key, kSortKeyDraw0ProgramMask, kSortKeyDraw0ProgramShift); return false; // draw } - m_program.idx = uint16_t( (_key & kSortKeyComputeProgramMask) >> kSortKeyComputeProgramShift); + m_program = decodeProgram( _key, kSortKeyComputeProgramMask, kSortKeyComputeProgramShift); return true; // compute } + static ProgramHandle decodeProgram(uint64_t _key, uint64_t _mask, uint8_t _shift) + { + uint16_t idx = uint16_t( (_key & _mask) >> _shift); + if (idx == (_mask >> _shift)) + { + return ProgramHandle{kInvalidHandle}; + } + return ProgramHandle{idx}; + } + static ViewId decodeView(uint64_t _key) { return ViewId( (_key & kSortKeyViewMask) >> kSortKeyViewBitShift);