block/ssh.c: Don't double-check that characters are hex digits

In compare_fingerprint() we effectively check whether the characters
in the fingerprint are valid hex digits twice: first we do so with
qemu_isxdigit(), but then the hex2decimal() function also has a code
path where it effectively detects an invalid digit and returns -1.
This causes Coverity to complain because it thinks that we might use
that -1 value in an expression where it would be an integer overflow.

Avoid the double-check of hex digit validity by testing the return
values from hex2decimal() rather than doing separate calls to
qemu_isxdigit().

Since this means we now use the illegal-character return value
from hex2decimal(), rewrite it from "-1" to "UINT_MAX", which
has the same effect since the return type is "unsigned" but
looks less confusing at the callsites when we detect it with
"c0 > 0xf".

Resolves: Coverity CID 1547813
Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
Reviewed-by: Kevin Wolf <kwolf@redhat.com>
Message-ID: <20241008164708.2966400-3-peter.maydell@linaro.org>
Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Signed-off-by: Kevin Wolf <kwolf@redhat.com>
This commit is contained in:
Peter Maydell 2024-10-08 17:47:06 +01:00 committed by Kevin Wolf
parent 75200708ce
commit 19c1e44123

View File

@ -364,7 +364,7 @@ static unsigned hex2decimal(char ch)
return 10 + (ch - 'A'); return 10 + (ch - 'A');
} }
return -1; return UINT_MAX;
} }
/* Compare the binary fingerprint (hash of host key) with the /* Compare the binary fingerprint (hash of host key) with the
@ -376,13 +376,15 @@ static int compare_fingerprint(const unsigned char *fingerprint, size_t len,
unsigned c; unsigned c;
while (len > 0) { while (len > 0) {
unsigned c0, c1;
while (*host_key_check == ':') while (*host_key_check == ':')
host_key_check++; host_key_check++;
if (!qemu_isxdigit(host_key_check[0]) || c0 = hex2decimal(host_key_check[0]);
!qemu_isxdigit(host_key_check[1])) c1 = hex2decimal(host_key_check[1]);
if (c0 > 0xf || c1 > 0xf) {
return 1; return 1;
c = hex2decimal(host_key_check[0]) * 16 + }
hex2decimal(host_key_check[1]); c = c0 * 16 + c1;
if (c - *fingerprint != 0) if (c - *fingerprint != 0)
return c - *fingerprint; return c - *fingerprint;
fingerprint++; fingerprint++;