Add UTF-16 LE I/O routines
These are intended to replace UTF-16 uses of mbstowcs() / wcstombs()
This commit is contained in:
matt335672
parent
0463e552dc
commit
0758fe03a6
258
common/parse.c
258
common/parse.c
@ -27,7 +27,47 @@
|
||||
#include "arch.h"
|
||||
#include "parse.h"
|
||||
#include "log.h"
|
||||
#include "string_calls.h"
|
||||
#include "unicode_defines.h"
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
#if defined(B_ENDIAN) || defined(NEED_ALIGN)
|
||||
#define out_uint16_le_unchecked(s, v) do \
|
||||
{ \
|
||||
*((s)->p) = (unsigned char)((v) >> 0); \
|
||||
(s)->p++; \
|
||||
*((s)->p) = (unsigned char)((v) >> 8); \
|
||||
(s)->p++; \
|
||||
} while (0)
|
||||
#else
|
||||
#define out_uint16_le_unchecked(s, v) do \
|
||||
{ \
|
||||
*((unsigned short*)((s)->p)) = (unsigned short)(v); \
|
||||
(s)->p += 2; \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
#if defined(B_ENDIAN) || defined(NEED_ALIGN)
|
||||
#define in_uint16_le_unchecked(s, v) do \
|
||||
{ \
|
||||
(v) = (unsigned short) \
|
||||
( \
|
||||
(*((unsigned char*)((s)->p + 0)) << 0) | \
|
||||
(*((unsigned char*)((s)->p + 1)) << 8) \
|
||||
); \
|
||||
(s)->p += 2; \
|
||||
} while (0)
|
||||
#else
|
||||
#define in_uint16_le_unchecked(s, v) do \
|
||||
{ \
|
||||
(v) = *((unsigned short*)((s)->p)); \
|
||||
(s)->p += 2; \
|
||||
} while (0)
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
void
|
||||
parser_stream_overflow_check(const struct stream *s, int n, int is_out,
|
||||
const char *file, int line)
|
||||
@ -64,3 +104,221 @@ parser_stream_overflow_check(const struct stream *s, int n, int is_out,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
void
|
||||
out_utf8_as_utf16_le_proc(struct stream *s, const char *v,
|
||||
unsigned int vn,
|
||||
const char *file, int line)
|
||||
{
|
||||
// Expansion of S_CHECK_REM_OUT(s, <octet_count>) using passed-in
|
||||
// file and line
|
||||
#ifdef USE_DEVEL_STREAMCHECK
|
||||
int octet_cnt = utf8_as_utf16_word_count(v, vn) * 2;
|
||||
parser_stream_overflow_check(s, octet_cnt, 1, file, line);
|
||||
#endif
|
||||
|
||||
while (vn > 0)
|
||||
{
|
||||
char32_t c32 = utf8_get_next_char(&v, &vn);
|
||||
char16_t low;
|
||||
if (c32 < 0x10000)
|
||||
{
|
||||
low = (char16_t)c32;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Need a surrogate pair */
|
||||
low = LOW_SURROGATE_FROM_C32(c32);
|
||||
char16_t high = HIGH_SURROGATE_FROM_C32(c32);
|
||||
out_uint16_le_unchecked(s, high);
|
||||
}
|
||||
out_uint16_le_unchecked(s, low);
|
||||
}
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Gets the next Unicode character from a code stream
|
||||
* @param s Stream
|
||||
* @return Unicode character
|
||||
*
|
||||
* Non-characters and illegally coded characters are mapped to
|
||||
* UCS_REPLACEMENT_CHARACTER
|
||||
*
|
||||
* @pre Two bytes are assumed to be available on the stram on entry
|
||||
*/
|
||||
static char32_t
|
||||
get_c32_from_stream(struct stream *s)
|
||||
{
|
||||
char32_t c32 = UCS_REPLACEMENT_CHARACTER; // Assume failure
|
||||
char16_t w;
|
||||
|
||||
in_uint16_le_unchecked(s, w);
|
||||
|
||||
if (IS_HIGH_SURROGATE(w))
|
||||
{
|
||||
if (s_check_rem(s, 2))
|
||||
{
|
||||
char16_t low;
|
||||
in_uint16_le_unchecked(s, low);
|
||||
if (IS_LOW_SURROGATE(low))
|
||||
{
|
||||
/* Valid surrogate pair */
|
||||
char32_t v = C32_FROM_SURROGATE_PAIR(low, w);
|
||||
|
||||
/* Ignore some values which can be successfully encoded
|
||||
* in this way */
|
||||
if (!IS_PLANE_END_NON_CHARACTER(c32))
|
||||
{
|
||||
c32 = v;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
/* Invalid low surrogate - pop character back */
|
||||
s->p -= 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (!IS_LOW_SURROGATE(w) &&
|
||||
!IS_PLANE_END_NON_CHARACTER(w) &&
|
||||
!IS_ARABIC_NON_CHARACTER(w))
|
||||
{
|
||||
/* Character from the Basic Multilingual Plane */
|
||||
c32 = (char32_t)w;
|
||||
}
|
||||
|
||||
return c32;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
unsigned int
|
||||
in_utf16_le_fixed_as_utf8_proc(struct stream *s, unsigned int n,
|
||||
char *v, unsigned int vn,
|
||||
const char *file, int line)
|
||||
{
|
||||
unsigned int rv = 0;
|
||||
char32_t c32;
|
||||
char u8str[MAXLEN_UTF8_CHAR];
|
||||
unsigned int u8len;
|
||||
char *saved_s_end = s->end;
|
||||
|
||||
// Expansion of S_CHECK_REM(s, n*2) using passed-in file and line
|
||||
#ifdef USE_DEVEL_STREAMCHECK
|
||||
parser_stream_overflow_check(s, n * 2, 0, file, line);
|
||||
#endif
|
||||
// Temporarily set the stream end pointer to allow us to use
|
||||
// s_check_rem() when reading in UTF-16 words
|
||||
if (s->end - s->p > (int)(n * 2))
|
||||
{
|
||||
s->end = s->p + (int)(n * 2);
|
||||
}
|
||||
|
||||
while (s_check_rem(s, 2))
|
||||
{
|
||||
c32 = get_c32_from_stream(s);
|
||||
|
||||
u8len = utf_char32_to_utf8(c32, u8str);
|
||||
if (u8len + 1 <= vn)
|
||||
{
|
||||
/* Room for this character and a terminator. Add the character */
|
||||
unsigned int i;
|
||||
for (i = 0 ; i < u8len ; ++i)
|
||||
{
|
||||
v[i] = u8str[i];
|
||||
}
|
||||
vn -= u8len;
|
||||
v += u8len;
|
||||
}
|
||||
else if (vn > 1)
|
||||
{
|
||||
/* We've skipped a character, but there's more than one byte
|
||||
* remaining in the output buffer. Mark the output buffer as
|
||||
* full so we don't get a smaller character being squeezed into
|
||||
* the remaining space */
|
||||
vn = 1;
|
||||
}
|
||||
|
||||
rv += u8len;
|
||||
}
|
||||
|
||||
// Restore stream to full length
|
||||
s->end = saved_s_end;
|
||||
|
||||
if (vn > 0)
|
||||
{
|
||||
*v = '\0';
|
||||
}
|
||||
++rv;
|
||||
return rv;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
unsigned int
|
||||
in_utf16_le_fixed_as_utf8_length(struct stream *s, unsigned int n)
|
||||
{
|
||||
char *saved_s_p = s->p;
|
||||
unsigned int rv = in_utf16_le_fixed_as_utf8(s, n, NULL, 0);
|
||||
s->p = saved_s_p;
|
||||
return rv;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
unsigned int
|
||||
in_utf16_le_terminated_as_utf8(struct stream *s,
|
||||
char *v, unsigned int vn)
|
||||
{
|
||||
unsigned int rv = 0;
|
||||
char32_t c32;
|
||||
char u8str[MAXLEN_UTF8_CHAR];
|
||||
unsigned int u8len;
|
||||
while (s_check_rem(s, 2))
|
||||
{
|
||||
c32 = get_c32_from_stream(s);
|
||||
if (c32 == 0)
|
||||
{
|
||||
break; // Terminator encountered
|
||||
}
|
||||
|
||||
u8len = utf_char32_to_utf8(c32, u8str);
|
||||
if (u8len + 1 <= vn)
|
||||
{
|
||||
/* Room for this character and a terminator. Add the character */
|
||||
unsigned int i;
|
||||
for (i = 0 ; i < u8len ; ++i)
|
||||
{
|
||||
v[i] = u8str[i];
|
||||
}
|
||||
vn -= u8len;
|
||||
v += u8len;
|
||||
}
|
||||
else if (vn > 1)
|
||||
{
|
||||
/* We've skipped a character, but there's more than one byte
|
||||
* remaining in the output buffer. Mark the output buffer as
|
||||
* full so we don't get a smaller character being squeezed into
|
||||
* the remaining space */
|
||||
vn = 1;
|
||||
}
|
||||
rv += u8len;
|
||||
}
|
||||
|
||||
if (vn > 0)
|
||||
{
|
||||
*v = '\0';
|
||||
}
|
||||
++rv;
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
/******************************************************************************/
|
||||
unsigned int
|
||||
in_utf16_le_terminated_as_utf8_length(struct stream *s)
|
||||
{
|
||||
char *saved_s_p = s->p;
|
||||
unsigned int rv = in_utf16_le_terminated_as_utf8(s, NULL, 0);
|
||||
s->p = saved_s_p;
|
||||
return rv;
|
||||
}
|
||||
|
||||
@ -89,6 +89,102 @@ parser_stream_overflow_check(const struct stream *s, int n, int is_out,
|
||||
# define S_CHECK_REM_OUT(s,n)
|
||||
#endif
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Copies a UTF-8 string to a stream as little-endian UTF-16
|
||||
*
|
||||
* @param s Stream
|
||||
* @param v UTF-8 string
|
||||
* @param vn Length of UTF-8 string.
|
||||
* @param file Caller location (from __FILE__)
|
||||
* @param line Caller location (from __LINE__)
|
||||
*
|
||||
* Caller is expected to check there is room for the result in s
|
||||
*/
|
||||
void
|
||||
out_utf8_as_utf16_le_proc(struct stream *s, const char *v,
|
||||
unsigned int vn,
|
||||
const char *file, int line);
|
||||
|
||||
#define out_utf8_as_utf16_le(s,v,vn) \
|
||||
out_utf8_as_utf16_le_proc((s), (v), (vn), __FILE__, __LINE__)
|
||||
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Copies a fixed-size little-endian UTF-16 string from a stream as UTF-8
|
||||
*
|
||||
* @param s Stream
|
||||
* @param n Number of 16-bit words to copy
|
||||
* @param v Pointer to result buffer
|
||||
* @param vn Max size of result buffer
|
||||
*
|
||||
* @return number of characters which would be written to v, INCLUDING
|
||||
* an additional terminator. This can be used to check for a buffer
|
||||
* overflow. A terminator is added whether or not the input
|
||||
* includes one.
|
||||
*
|
||||
* Output is unconditionally NULL-terminated.
|
||||
* Input is not checked for NULLs - these are copied verbatim
|
||||
*/
|
||||
unsigned int
|
||||
in_utf16_le_fixed_as_utf8_proc(struct stream *s, unsigned int n,
|
||||
char *v, unsigned int vn,
|
||||
const char *file, int line);
|
||||
|
||||
#define in_utf16_le_fixed_as_utf8(s,n,v,vn) \
|
||||
in_utf16_le_fixed_as_utf8_proc((s), (n), (v), (vn), __FILE__, __LINE__)
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Returns the size of the buffer needed to store a fixed-size
|
||||
* little-endian UTF-16 string in a stream as a UTF-8 string
|
||||
*
|
||||
* @param s Stream
|
||||
* @param n Number of 16-bit words to consider
|
||||
* @return number of characters needed to store the UTF-8 string. This
|
||||
* includes a terminator, which is written whether the parsed
|
||||
* string includes one or not.
|
||||
* @post Stream position is not moved between start and end of this call
|
||||
*/
|
||||
unsigned int
|
||||
in_utf16_le_fixed_as_utf8_length(struct stream *s, unsigned int n);
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Copies a terminated little-endian UTF-16 string from a stream as UTF-8
|
||||
*
|
||||
* @param s Stream
|
||||
* @param v Pointer to result buffer
|
||||
* @param vn Max size of result buffer
|
||||
*
|
||||
* @return number of characters which would be written to v, INCLUDING
|
||||
* the terminator. This can be used to check for a buffer overflow.
|
||||
*
|
||||
* Output is unconditionally NULL-terminated.
|
||||
* Input processing stops when a NULL is encountered, or the end of the buffer
|
||||
* is reached.
|
||||
*/
|
||||
unsigned int
|
||||
in_utf16_le_terminated_as_utf8(struct stream *s,
|
||||
char *v, unsigned int vn);
|
||||
|
||||
/******************************************************************************/
|
||||
/**
|
||||
* Returns the size of the buffer needed to store a terminated
|
||||
* little-endian UTF-16 string in a stream as a terminated UTF-8 string
|
||||
*
|
||||
* @param s Stream
|
||||
* @return number of characters needed to store the UTF-8 string,
|
||||
* including the terminator
|
||||
* @post Stream position is not moved between start and end of this call
|
||||
*
|
||||
* Input processing stops when a NULL is encountered, or the end of the buffer
|
||||
* is reached.
|
||||
*/
|
||||
unsigned int
|
||||
in_utf16_le_terminated_as_utf8_length(struct stream *s);
|
||||
|
||||
/******************************************************************************/
|
||||
#define s_check_rem(s, n) ((s)->p + (n) <= (s)->end)
|
||||
|
||||
|
||||
@ -68,4 +68,33 @@
|
||||
INVALID_UNICODE_10000_TO_1FFFFF(c) \
|
||||
)
|
||||
|
||||
/**
|
||||
* Is this character a UTF-16 high surrogate?
|
||||
*/
|
||||
#define IS_HIGH_SURROGATE(u16) (((u16) & 0xfc00) == 0xd800)
|
||||
|
||||
/**
|
||||
* Is this character a UTF-16 low surrogate?
|
||||
*/
|
||||
#define IS_LOW_SURROGATE(u16) (((u16) & 0xfc00) == 0xdc00)
|
||||
|
||||
/**
|
||||
* Extract the UTF-16 high surrogate from a character
|
||||
*/
|
||||
#define HIGH_SURROGATE_FROM_C32(c32) \
|
||||
(((((c32) - 0x10000) >> 10) & 0x3ff) | 0xd800)
|
||||
|
||||
/**
|
||||
* Extract the UTF-16 low surrogate from a character
|
||||
*/
|
||||
#define LOW_SURROGATE_FROM_C32(c32) (((c32) & 0x3ff) | 0xdc00)
|
||||
|
||||
/**
|
||||
* Reconstruct a character from a UTF-16 surrogate pair
|
||||
*
|
||||
* This macro cannot return values higher than 0x10ffff
|
||||
*/
|
||||
#define C32_FROM_SURROGATE_PAIR(low,high) \
|
||||
((char32_t)(((high) & 0x3ff) << 10) + ((low) & 0x3ff) + 0x10000)
|
||||
|
||||
#endif // UNICODE_DEFINES_H
|
||||
|
||||
@ -16,6 +16,7 @@ test_common_SOURCES = \
|
||||
test_common_main.c \
|
||||
test_fifo_calls.c \
|
||||
test_list_calls.c \
|
||||
test_parse.c \
|
||||
test_string_calls.c \
|
||||
test_string_calls_unicode.c \
|
||||
test_os_calls.c \
|
||||
|
||||
@ -9,6 +9,7 @@ bin_to_hex(const char *input, int length);
|
||||
|
||||
Suite *make_suite_test_fifo(void);
|
||||
Suite *make_suite_test_list(void);
|
||||
Suite *make_suite_test_parse(void);
|
||||
Suite *make_suite_test_string(void);
|
||||
Suite *make_suite_test_string_unicode(void);
|
||||
Suite *make_suite_test_os_calls(void);
|
||||
|
||||
@ -48,6 +48,7 @@ int main (void)
|
||||
|
||||
sr = srunner_create (make_suite_test_fifo());
|
||||
srunner_add_suite(sr, make_suite_test_list());
|
||||
srunner_add_suite(sr, make_suite_test_parse());
|
||||
srunner_add_suite(sr, make_suite_test_string());
|
||||
srunner_add_suite(sr, make_suite_test_string_unicode());
|
||||
srunner_add_suite(sr, make_suite_test_os_calls());
|
||||
|
||||
319
tests/common/test_parse.c
Normal file
319
tests/common/test_parse.c
Normal file
@ -0,0 +1,319 @@
|
||||
#if defined(HAVE_CONFIG_H)
|
||||
#include "config_ac.h"
|
||||
#endif
|
||||
|
||||
#include "arch.h"
|
||||
#include "os_calls.h"
|
||||
#include "string_calls.h"
|
||||
#include "parse.h"
|
||||
|
||||
#include "test_common.h"
|
||||
|
||||
#define ELEMENTS(x) (sizeof(x) / sizeof(x[0]))
|
||||
|
||||
const static char
|
||||
utf8_simple_test_with_emoji[] =
|
||||
"Simple Test."
|
||||
"\xf0\x9f\x98\xa5"; // U+1F625 Disappointed But Relieved Face
|
||||
|
||||
const static char16_t
|
||||
utf16_simple_test_with_emoji[] =
|
||||
{
|
||||
'S', 'i', 'm', 'p', 'l', 'e', ' ', 'T', 'e', 's', 't', '.',
|
||||
0xd83d, 0xde25, // U+1F625
|
||||
0 // terminator
|
||||
};
|
||||
|
||||
/******************************************************************************/
|
||||
START_TEST(test_out_utf8_as_utf16_le)
|
||||
{
|
||||
struct stream *s;
|
||||
make_stream(s);
|
||||
init_stream(s, 8192);
|
||||
out_utf8_as_utf16_le(s, utf8_simple_test_with_emoji,
|
||||
sizeof(utf8_simple_test_with_emoji)); // Include term
|
||||
s_mark_end(s);
|
||||
|
||||
// Rewind the stream
|
||||
s->p = s->data;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji); ++i)
|
||||
{
|
||||
char16_t val;
|
||||
in_uint16_le(s, val);
|
||||
if (val != utf16_simple_test_with_emoji[i])
|
||||
{
|
||||
ck_abort_msg("test_out_utf8_as_utf16_le: "
|
||||
"Index %u expected %x, got %x",
|
||||
i, utf16_simple_test_with_emoji[i], val);
|
||||
}
|
||||
}
|
||||
|
||||
ck_assert_int_eq(s_check_end(s), 1);
|
||||
|
||||
free_stream(s);
|
||||
}
|
||||
END_TEST
|
||||
|
||||
/******************************************************************************/
|
||||
START_TEST(test_in_utf16_le_fixed_as_utf8)
|
||||
{
|
||||
struct stream *s;
|
||||
make_stream(s);
|
||||
init_stream(s, 8192);
|
||||
|
||||
// Write the stream without a terminator
|
||||
unsigned int i;
|
||||
for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji) - 1; ++i)
|
||||
{
|
||||
out_uint16_le(s, utf16_simple_test_with_emoji[i]);
|
||||
}
|
||||
s_mark_end(s);
|
||||
|
||||
// Rewind the stream
|
||||
s->p = s->data;
|
||||
|
||||
char buff[256];
|
||||
unsigned int len;
|
||||
|
||||
// Check the length call
|
||||
len = in_utf16_le_fixed_as_utf8_length(s, i);
|
||||
ck_assert_int_eq(len, sizeof(utf8_simple_test_with_emoji));
|
||||
|
||||
// Now read the string, checking for the same length
|
||||
unsigned int read_len;
|
||||
read_len = in_utf16_le_fixed_as_utf8(s, i, buff, sizeof(buff));
|
||||
ck_assert_int_eq(len, read_len);
|
||||
|
||||
// Should be at the end of the buffer
|
||||
ck_assert_int_eq(s_check_end(s), 1);
|
||||
|
||||
// Check the contents are as expected
|
||||
int cmp = memcmp(buff, utf8_simple_test_with_emoji,
|
||||
sizeof(utf8_simple_test_with_emoji));
|
||||
ck_assert_int_eq(cmp, 0);
|
||||
|
||||
free_stream(s);
|
||||
}
|
||||
END_TEST
|
||||
|
||||
/******************************************************************************/
|
||||
START_TEST(test_in_utf16_le_terminated_as_utf8)
|
||||
{
|
||||
struct stream *s;
|
||||
make_stream(s);
|
||||
init_stream(s, 8192);
|
||||
|
||||
// Write the stream with the terminator
|
||||
unsigned int i;
|
||||
for (i = 0; i < ELEMENTS(utf16_simple_test_with_emoji); ++i)
|
||||
{
|
||||
out_uint16_le(s, utf16_simple_test_with_emoji[i]);
|
||||
}
|
||||
s_mark_end(s);
|
||||
|
||||
// Rewind the stream
|
||||
s->p = s->data;
|
||||
|
||||
char buff[256];
|
||||
unsigned int len;
|
||||
|
||||
// Check the length call
|
||||
len = in_utf16_le_terminated_as_utf8_length(s);
|
||||
ck_assert_int_eq(len, sizeof(utf8_simple_test_with_emoji));
|
||||
|
||||
// Now read the string, checking for the same length
|
||||
unsigned int read_len;
|
||||
read_len = in_utf16_le_terminated_as_utf8(s, buff, sizeof(buff));
|
||||
ck_assert_int_eq(len, read_len);
|
||||
|
||||
// Should be at the end of the buffer
|
||||
ck_assert_int_eq(s_check_end(s), 1);
|
||||
|
||||
// Check the contents are as expected
|
||||
int cmp = memcmp(buff, utf8_simple_test_with_emoji,
|
||||
sizeof(utf8_simple_test_with_emoji));
|
||||
ck_assert_int_eq(cmp, 0);
|
||||
|
||||
free_stream(s);
|
||||
}
|
||||
END_TEST
|
||||
|
||||
/******************************************************************************/
|
||||
START_TEST(test_in_utf16_le_significant_chars)
|
||||
{
|
||||
struct stream *s;
|
||||
make_stream(s);
|
||||
init_stream(s, 8192);
|
||||
|
||||
const struct
|
||||
{
|
||||
struct
|
||||
{
|
||||
char16_t high; // Set to 0 for a single UTF-16 word
|
||||
char16_t low;
|
||||
} pair;
|
||||
char32_t expected;
|
||||
} tests[] =
|
||||
{
|
||||
// Single high surrogates are bad
|
||||
{ { 0, 0xd800 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xdbff }, UCS_REPLACEMENT_CHARACTER },
|
||||
// Single low surrogates are bad
|
||||
{ { 0, 0xdc00 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xdfff }, UCS_REPLACEMENT_CHARACTER },
|
||||
// Values before and after surrogate range
|
||||
{ { 0, 0xd7ff }, 0xd7ff },
|
||||
{ { 0, 0xe000 }, 0xe000 },
|
||||
// First and last non-surrogate pair values (don't use
|
||||
// 0xfffe and 0xffff for this test as they are non-characters,
|
||||
// and 0xfffd is the replacement character)
|
||||
{ { 0, 0 }, 0 },
|
||||
{ { 0, 0xfffc }, 0xfffc },
|
||||
{ { 0, 0xfffd }, UCS_REPLACEMENT_CHARACTER },
|
||||
// First and last surrogate pair values (don't use
|
||||
// 0x10fffe and 0x10ffff for this test as they are non-characters)
|
||||
{ { 0xd800, 0xdc00 }, 0x10000 },
|
||||
{ { 0xdbff, 0xdffd }, 0x10fffd },
|
||||
// End-of-plane non-characters (BMP) and the characters before them
|
||||
{ { 0xd83f, 0xdffd }, 0x1fffd },
|
||||
{ { 0xd83f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+1FFFE
|
||||
{ { 0xd83f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+1FFFF
|
||||
{ { 0xd87f, 0xdffd }, 0x2fffd },
|
||||
{ { 0xd87f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+2FFFE
|
||||
{ { 0xd87f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+2FFFF
|
||||
{ { 0xd8bf, 0xdffd }, 0x3fffd },
|
||||
{ { 0xd8bf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+3FFFE
|
||||
{ { 0xd8bf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+3FFFF
|
||||
{ { 0xd8ff, 0xdffd }, 0x4fffd },
|
||||
{ { 0xd8ff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+4FFFE
|
||||
{ { 0xd8ff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+4FFFF
|
||||
{ { 0xd93f, 0xdffd }, 0x5fffd },
|
||||
{ { 0xd93f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+5FFFE
|
||||
{ { 0xd93f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+5FFFF
|
||||
{ { 0xd97f, 0xdffd }, 0x6fffd },
|
||||
{ { 0xd97f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+6FFFE
|
||||
{ { 0xd97f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+6FFFF
|
||||
{ { 0xd9bf, 0xdffd }, 0x7fffd },
|
||||
{ { 0xd9bf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+7FFFE
|
||||
{ { 0xd9bf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+7FFFF
|
||||
{ { 0xd9ff, 0xdffd }, 0x8fffd },
|
||||
{ { 0xd9ff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+8FFFE
|
||||
{ { 0xd9ff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+8FFFF
|
||||
{ { 0xda3f, 0xdffd }, 0x9fffd },
|
||||
{ { 0xda3f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+9FFFE
|
||||
{ { 0xda3f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+9FFFF
|
||||
{ { 0xda7f, 0xdffd }, 0xafffd },
|
||||
{ { 0xda7f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+AFFFE
|
||||
{ { 0xda7f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+AFFFF
|
||||
{ { 0xdabf, 0xdffd }, 0xbfffd },
|
||||
{ { 0xdabf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+BFFFE
|
||||
{ { 0xdabf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+BFFFF
|
||||
{ { 0xdaff, 0xdffd }, 0xcfffd },
|
||||
{ { 0xdaff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+CFFFE
|
||||
{ { 0xdaff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+CFFFF
|
||||
{ { 0xdb3f, 0xdffd }, 0xdfffd },
|
||||
{ { 0xdb3f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+DFFFE
|
||||
{ { 0xdb3f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+DFFFF
|
||||
{ { 0xdb7f, 0xdffd }, 0xefffd },
|
||||
{ { 0xdb7f, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+EFFFE
|
||||
{ { 0xdb7f, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+EFFFF
|
||||
{ { 0xdbbf, 0xdffd }, 0xffffd },
|
||||
{ { 0xdbbf, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+FFFFE
|
||||
{ { 0xdbbf, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+FFFFF
|
||||
{ { 0xdbff, 0xdffd }, 0x10fffd },
|
||||
{ { 0xdbff, 0xdffe }, UCS_REPLACEMENT_CHARACTER }, // U+10FFFE
|
||||
{ { 0xdbff, 0xdfff }, UCS_REPLACEMENT_CHARACTER }, // U+10FFFF
|
||||
// Non-characters in "Arabic Presentation Forms-A"
|
||||
{ { 0, 0xfdd0 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd1 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd2 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd3 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd4 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd5 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd6 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd7 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd8 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdd9 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdda }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfddb }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfddc }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfddd }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdde }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfddf }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde0 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde1 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde2 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde3 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde4 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde5 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde6 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde7 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde8 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfde9 }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdea }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdeb }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdec }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfded }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdee }, UCS_REPLACEMENT_CHARACTER },
|
||||
{ { 0, 0xfdef }, UCS_REPLACEMENT_CHARACTER }
|
||||
};
|
||||
|
||||
unsigned int i;
|
||||
for (i = 0; i < ELEMENTS(tests); ++i)
|
||||
{
|
||||
char buff[256];
|
||||
unsigned int word_count;
|
||||
init_stream(s, 8192);
|
||||
|
||||
word_count = 0;
|
||||
if (tests[i].pair.high != 0)
|
||||
{
|
||||
out_uint16_le(s, tests[i].pair.high);
|
||||
++word_count;
|
||||
}
|
||||
out_uint16_le(s, tests[i].pair.low);
|
||||
++word_count;
|
||||
s_mark_end(s);
|
||||
|
||||
// Rewind the stream
|
||||
s->p = s->data;
|
||||
|
||||
// Read in one UTF-16 LE character as UTF-32
|
||||
in_utf16_le_fixed_as_utf8(s, word_count, buff, sizeof(buff));
|
||||
const char *p = buff;
|
||||
char32_t c32 = utf8_get_next_char(&p, NULL);
|
||||
|
||||
if (c32 != tests[i].expected)
|
||||
{
|
||||
ck_abort_msg("test_in_utf16_le_significant_chars: "
|
||||
"Index %u for {%x, %x}, expected %x, got %x",
|
||||
i, tests[i].pair.high, tests[i].pair.low,
|
||||
tests[i].expected, c32);
|
||||
}
|
||||
}
|
||||
|
||||
free_stream(s);
|
||||
}
|
||||
END_TEST
|
||||
|
||||
/******************************************************************************/
|
||||
|
||||
Suite *
|
||||
make_suite_test_parse(void)
|
||||
{
|
||||
Suite *s;
|
||||
TCase *tc_unicode;
|
||||
|
||||
s = suite_create("Parse");
|
||||
|
||||
tc_unicode = tcase_create("Unicode");
|
||||
suite_add_tcase(s, tc_unicode);
|
||||
tcase_add_test(tc_unicode, test_out_utf8_as_utf16_le);
|
||||
tcase_add_test(tc_unicode, test_in_utf16_le_fixed_as_utf8);
|
||||
tcase_add_test(tc_unicode, test_in_utf16_le_terminated_as_utf8);
|
||||
tcase_add_test(tc_unicode, test_in_utf16_le_significant_chars);
|
||||
|
||||
return s;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user