[winpr,asn1] fix sign extraction for ASN1 integer

ASN1 integer might not be full 32bit integers, so extract the sign from
the first byte and append 0x80000000 after the rest of the integer was
successfully read.
This commit is contained in:
akallabeth 2024-09-16 17:17:14 +02:00
parent 6c88d89566
commit ca260e13bb
No known key found for this signature in database
GPG Key ID: A49454A3FC909FD5
1 changed files with 13 additions and 7 deletions

View File

@ -1023,22 +1023,28 @@ static size_t WinPrAsn1DecReadIntegerLike(WinPrAsn1Decoder* dec, WinPrAsn1_tag e
if (len == 0 || Stream_GetRemainingLength(&dec->source) < len || (len > 4))
return 0;
WinPrAsn1_INTEGER val = 0;
UINT32 uval = 0;
UINT8 v = 0;
Stream_Read_UINT8(&dec->source, v);
if (v & 0x80)
val = 0xFFFFFFFF;
val |= v;
/* extract sign from first byte.
* the ASN integer might be smaller than 32bit so we need to set the initial
* value to FF for all unused bytes (e.g. all except the lowest one we just read)
*/
BOOL isNegative = (v & 0x80);
if (isNegative)
uval = 0xFFFFFF00;
uval |= v;
for (size_t x = 1; x < len; x++)
{
Stream_Read_UINT8(&dec->source, v);
val = (WinPrAsn1_INTEGER)(((UINT32)val) << 8);
val |= v;
uval <<= 8;
uval |= v;
}
*target = val;
*target = (WinPrAsn1_INTEGER)uval;
ret += len;
/* TODO: check ber/der rules */