mirrors
/
xrdp
mirror of https://github.com/neutrinolabs/xrdp
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
xrdp/tests/common/test_string_calls_unicode.c

836 lines
28 KiB
C
Raw Permalink Blame History

/*
* The UTF-8 decoder tests are based on the UTF-8 decoder capability
* and stress test" 2015-08-26 by Markus Kuhn. A copy of that file
* named "UTF-8-test.txt" should be in the source directory for this file */
#if defined(HAVE_CONFIG_H)
#include "config_ac.h"
#endif
#include "string_calls.h"
#include "test_common.h"
// Abbreviate UCS_REPLACEMENT_CHARACTER for utf8_decode_sub_test arrays
#define URC UCS_REPLACEMENT_CHARACTER
struct utf8_decode_sub_test
{
const char *testref;
const char *utf8str;
// This array will contain 0 values after the initialised part
const char32_t expected[65];
};
// Abbreviate UCS_REPLACEMENT_CHARACTER for utf8_encode_sub_test arrays
#define E_URC { 0xef, 0xbf, 0xbd }
struct utf8_encode_sub_test
{
const char *testref;
char32_t c32;
unsigned int expected_len;
char expected_str[MAXLEN_UTF8_CHAR];
};
// Used as the simple test in UTF-8-test.txt
const static char greek_kosme[] =
"\xce\xba" // GREEK SMALL LETTER KAPPA
"\xe1\xbd\xb9" // GREEK SMALL LETTER OMICRON WITH OXIA
"\xcf\x83" // GREEK SMALL LETTER SIGMA
"\xce\xbc" // GREEK SMALL LETTER MU
"\xce\xb5"; // GREEK SMALL LETTER EPSILON
// See Issue #2603
const static char simple_test_with_emoji[] =
"Simple Test."
"\xf0\x9f\x98\xa5"; // U+1F625 Disappointed But Relieved Face
/******************************************************************************/
/**
* Function to decode a UTF-8 string and check the expected result
*
* @param st Pointer to the sub-test to run
*/
static void
run_utf8_decode_sub_test(const struct utf8_decode_sub_test *st)
{
char32_t c;
const char *p = st->utf8str;
unsigned int index = 0;
do
{
c = utf8_get_next_char(&p, NULL);
if (c != st->expected[index])
{
ck_abort_msg("Sub-test section %s Index %u expected %x, got %x",
st->testref,
index, st->expected[index], c);
}
++index;
}
while (c != 0);
}
/******************************************************************************/
/**
* Function to run an array of decode sub-tests
*
* @param st Pointer to the first sub-test to run
*/
static void
run_decode_sub_test_array(const struct utf8_decode_sub_test *st)
{
while (st->utf8str != NULL)
{
run_utf8_decode_sub_test(st++);
}
}
/******************************************************************************/
/**
* Function to encode a UTF-8 value and check the expected result
*
* @param st Pointer to the sub-test to run
*/
static void
run_utf8_encode_sub_test(const struct utf8_encode_sub_test *st)
{
char actual_str[MAXLEN_UTF8_CHAR];
unsigned int index;
unsigned int actual_len = utf_char32_to_utf8(st->c32, actual_str);
if (actual_len != st->expected_len)
{
ck_abort_msg("Sub-test %s Expected length of %u, got %u",
st->testref,
st->expected_len, actual_len);
}
for (index = 0 ; index < actual_len; ++index)
{
if (actual_str[index] != st->expected_str[index])
{
ck_abort_msg("Sub-test %s Character %u, expected %02x got %02x",
st->testref, index,
(int)(unsigned char)st->expected_str[index],
(int)(unsigned char)actual_str[index]);
}
}
}
/******************************************************************************/
/**
* Function to run an array of encode sub-tests
*
* @param st Pointer to the first sub-test to run
*/
static void
run_encode_sub_test_array(const struct utf8_encode_sub_test *st)
{
while (st->expected_len > 0)
{
run_utf8_encode_sub_test(st++);
}
}
/******************************************************************************/
START_TEST(test_get_next_char__section_1)
{
const struct utf8_decode_sub_test st =
{
"1",
greek_kosme,
{
0x03ba, // GREEK SMALL LETTER KAPPA
0x1f79, // GREEK SMALL LETTER OMICRON WITH OXIA
0x03c3, // GREEK SMALL LETTER SIGMA
0x03bc, // GREEK SMALL LETTER MU
0x03b5 // GREEK SMALL LETTER EPSILON
}
};
run_utf8_decode_sub_test(&st);
}
END_TEST
/******************************************************************************/
START_TEST(test_get_next_char__section_2)
{
struct utf8_decode_sub_test tests[] =
{
// 2.1 First possible sequence of a certain length
//
// (2.1.1 Is tested separately)
{ "2.1.2", "\xc2\x80", { 0x80 } },
{ "2.1.3", "\xe0\xa0\x80", { 0x800 } },
{ "2.1.4", "\xf0\x90\x80\x80", { 0x10000 } },
{ "2.1.5", "\xf8\x88\x80\x80\x80", { URC } },
{ "2.1.6", "\xfc\x84\x80\x80\x80\x80", { URC } },
// 2.2 Last possible sequence of a certain length
{ "2.2.1", "\x7f", { 0x7f } },
{ "2.2.2", "\xdf\xbf", { 0x7ff } },
// Use U+0000FFFC instead of U+0000FFFF as our decoder
// treats non-characters as an input error
{ "2.2.3", "\xef\xbf\xbc", { 0xfffc } },
// U+001FFFFF is out-of-range
{ "2.2.4", "\xf7\xbf\xbf\xbf", { URC } },
{ "2.2.5", "\xfb\xbf\xbf\xbf\xbf", { URC } },
{ "2.2.6", "\xfd\xbf\xbf\xbf\xbf\xbf", { URC } },
// 2.3 Other boundary conditions
{ "2.3.1", "\xed\x9f\xbf", { 0xd7ff } },
{ "2.3.2", "\xee\x80\x80", { 0xe000 } },
{ "2.3.3", "\xef\xbf\xbd", { 0xfffd } },
// Don't use U+10FFFF (non-character)
{ "2.3.4", "\xf4\x8f\xbf\xbd", { 0x10fffd } },
{ "2.3.5", "\xf4\x90\x80\x80", { URC } },
// Terminator
{ 0 }
};
// 2.1.1 is a '\0' which we use to terminate our strings. Test
// it separately
{
const char *p = "";
ck_assert_int_eq(utf8_get_next_char(&p, NULL), 0);
}
// Do the rest of the section 2 tests
run_decode_sub_test_array(tests);
}
END_TEST
/******************************************************************************/
START_TEST(test_get_next_char__section_3)
{
struct utf8_decode_sub_test tests[] =
{
// 3.1 Unexpected continuation bytes
//
// Each unexpected continuation byte should be separately
// signalled as a malformed sequence of its own.
{ "3.1.1", "\x80", { URC } },
{ "3.1.2", "\xbf", { URC } },
{ "3.1.3", "\x80\xbf", { URC, URC } },
{ "3.1.4", "\x80\xbf\x80", { URC, URC, URC } },
{ "3.1.5", "\x80\xbf\x80\xbf", { URC, URC, URC, URC } },
{ "3.1.6", "\x80\xbf\x80\xbf\x80", { URC, URC, URC, URC, URC } },
{
"3.1.7",
"\x80\xbf\x80\xbf\x80\xbf",
{ URC, URC, URC, URC, URC, URC }
},
{
"3.1.8",
"\x80\xbf\x80\xbf\x80\xbf\x80",
{ URC, URC, URC, URC, URC, URC, URC }
},
{
"3.1.9",
"\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f"
"\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f"
"\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf"
"\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf",
{
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC
}
},
// 3.2 Lonely start characters
{
"3.2.1",
"\xc0 \xc1 \xc2 \xc3 \xc4 \xc5 \xc6 \xc7 "
"\xc8 \xc9 \xca \xcb \xcc \xcd \xce \xcf "
"\xd0 \xd1 \xd2 \xd3 \xd4 \xd5 \xd6 \xd7 "
"\xd8 \xd9 \xda \xdb \xdc \xdd \xde \xdf ",
{
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' '
}
},
{
"3.2.2",
"\xe0 \xe1 \xe2 \xe3 \xe4 \xe5 \xe6 \xe7 "
"\xe8 \xe9 \xea \xeb \xec \xed \xee \xef ",
{
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' '
}
},
{
"3.2.3",
"\xf0 \xf1 \xf2 \xf3 \xf4 \xf5 \xf6 \xf7 ",
{
URC, ' ', URC, ' ', URC, ' ', URC, ' ',
URC, ' ', URC, ' ', URC, ' ', URC, ' '
}
},
{
"3.2.4",
"\xf8 \xf9 \xfa \xfb ",
{
URC, ' ', URC, ' ', URC, ' ', URC, ' '
}
},
{
"3.2.5", "\xfc \xfd ", { URC, ' ', URC, ' ' }
},
// 3.3 Sequences with last continuation byte missing
//
// From UTF-8-test.txt:-
// All bytes of an incomplete sequence should be signalled as
// a single malformed sequence, i.e., you should see only a
// single replacement character in each of the next 10 tests.
{ "3.3.1", "\xc0", { URC } },
{ "3.3.2", "\xe0\x80", { URC } },
{ "3.3.3", "\xf0\x80\x80", { URC } },
{ "3.3.4", "\xf8\x80\x80\x80", { URC } },
{ "3.3.5", "\xfc\x80\x80\x80\x80", { URC } },
{ "3.3.6", "\xdf", { URC } },
{ "3.3.7", "\xef\xbf", { URC } },
{ "3.3.8", "\xf7\xbf\xbf", { URC} },
{ "3.3.9", "\xfb\xbf\xbf\xbf", { URC } },
{ "3.3.10", "\xfd\xbf\xbf\xbf\xbf", { URC } },
// 3.4 Concatenation of incomplete sequences
{
"3,4",
"\xc0"
"\xe0\x80"
"\xf0\x80\x80"
"\xf8\x80\x80\x80"
"\xfc\x80\x80\x80\x80"
"\xdf"
"\xef\xbf"
"\xf7\xbf\xbf"
"\xfb\xbf\xbf\xbf"
"\xfd\xbf\xbf\xbf\xbf",
{
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC
}
},
// 3.5 Impossible bytes
{ "3.5.1", "\xfe", { URC } },
{ "3.5.2", "\xff", { URC } },
{ "3.5.3", "\xfe\xfe\xff\xff", { URC, URC, URC, URC } },
// Terminator
{ 0 }
};
run_decode_sub_test_array(tests);
}
END_TEST
/******************************************************************************/
START_TEST(test_get_next_char__section_4)
{
struct utf8_decode_sub_test tests[] =
{
// 4.1 Examples of an overlong ASCII character
//
// With a safe UTF-8 decoder, all of the following five
// overlong representations of the ASCII character slash ("/")
// should be rejected like a malformed UTF-8 sequence, for
// instance by substituting it with a replacement character. If
// you see a slash below, you do not have a safe UTF-8 decoder!
{ "4.1.1", "\xc0\xaf", { URC } },
{ "4.1.2", "\xe0\x80\xaf", { URC } },
{ "4.1.3", "\xf0\x80\x80\xaf", { URC } },
{ "4.1.4", "\xf8\x80\x80\x80\xaf", { URC } },
{ "4.1.5", "\xfc\x80\x80\x80\x80\xaf", { URC } },
// 4.2 Maximum overlong sequences
// Below you see the highest Unicode value that is still resulting
// in an overlong sequence if represented with the given number of
// bytes. This is a boundary test for safe UTF-8 decoders. All
// five characters should be rejected like malformed UTF-8
// sequences.
{ "4.2.1", "\xc1\xbf", { URC } },
{ "4.2.2", "\xe0\x9f\xbf", { URC } },
{ "4.2.3", "\xf0\x8f\xbf\xbf", { URC } },
{ "4.2.4", "\xf8\x87\xbf\xbf\xbf", { URC } },
{ "4.2.5", "\xfc\x83\xbf\xbf\xbf\xbf", { URC } },
// 4.3 Overlong representation of the NUL character
// The following five sequences should also be rejected like
// malformed UTF-8 sequences and should not be treated like the
// ASCII NUL character.
{ "4.3.1", "\xc0\x80", { URC } },
{ "4.3.2", "\xe0\x80\x80", { URC } },
{ "4.3.3", "\xf0\x80\x80\x80", { URC } },
{ "4.3.4", "\xf8\x80\x80\x80\x80", { URC } },
{ "4.3.5", "\xfc\x80\x80\x80\x80\x80", { URC } },
// Terminator
{ 0 }
};
run_decode_sub_test_array(tests);
}
END_TEST
/******************************************************************************/
START_TEST(test_get_next_char__section_5)
{
struct utf8_decode_sub_test tests[] =
{
// 5 Illegal code positions
// The following UTF-8 sequences should be rejected like
// malformed sequences, because they never represent valid
// ISO 10646 characters and a UTF-8 decoder that accepts them
// might introduce security problems comparable to overlong
// UTF-8 sequences.
// 5.1 Single UTF-16 surrogates
{ "5.1.1", "\xed\xa0\x80", { URC } },
{ "5.1.2", "\xed\xad\xbf", { URC } },
{ "5.1.3", "\xed\xae\x80", { URC } },
{ "5.1.4", "\xed\xaf\xbf", { URC } },
{ "5.1.5", "\xed\xb0\x80", { URC } },
{ "5.1.6", "\xed\xbe\x80", { URC } },
{ "5.1.7", "\xed\xbf\xbf", { URC } },
// 5.2 Paired UTF-16 surrogates
{ "5.2.1", "\xed\xa0\x80\xed\xb0\x80", { URC, URC } },
{ "5.2.2", "\xed\xa0\x80\xed\xbf\xbf", { URC, URC } },
{ "5.2.3", "\xed\xad\xbf\xed\xb0\x80", { URC, URC } },
{ "5.2.4", "\xed\xad\xbf\xed\xbf\xbf", { URC, URC } },
{ "5.2.5", "\xed\xae\x80\xed\xb0\x80", { URC, URC } },
{ "5.2.6", "\xed\xae\x80\xed\xbf\xbf", { URC, URC } },
{ "5.2.7", "\xed\xaf\xbf\xed\xb0\x80", { URC, URC } },
{ "5.2.8", "\xed\xaf\xbf\xed\xbf\xbf", { URC, URC } },
// 5.3 Noncharacter code positions
// The following "noncharacters" are "reserved for internal
// use" by applications, and according to older versions of
// the Unicode Standard "should never be interchanged". Unicode
// Corrigendum #9 dropped the latter restriction. Nevertheless,
// their presence in incoming UTF-8 data can remain a potential
// security risk, depending on what use is made of these codes
// subsequently. Examples of such internal use:
//
// - Some file APIs with 16-bit characters may use the integer
// value -1 = U+FFFF to signal an end-of-file (EOF) or error
// condition.
//
// - In some UTF-16 receivers, code point U+FFFE might trigger
// a byte-swap operation (to convert between UTF-16LE and
// UTF-16BE).
// With such internal use of noncharacters, it may be desirable
// and safer to block those code points in UTF-8 decoders, as
// they should never occur legitimately in incoming UTF-8 data,
// and could trigger unsafe behaviour in subsequent processing.
// Particularly problematic noncharacters in 16-bit applications:
{ "5.3.1", "\xef\xbf\xbe", { URC } },
{ "5.3.2", "\xef\xbf\xbf", { URC } },
// Other noncharacters:
{
"5.3.3",
// Non-characters in "Arabic Presentation Forms-A" (BMP)
"\xef\xb7\x90" "\xef\xb7\x91" "\xef\xb7\x92" "\xef\xb7\x93"
"\xef\xb7\x94" "\xef\xb7\x95" "\xef\xb7\x96" "\xef\xb7\x97"
"\xef\xb7\x98" "\xef\xb7\x99" "\xef\xb7\x9a" "\xef\xb7\x9b"
"\xef\xb7\x9c" "\xef\xb7\x9d" "\xef\xb7\x9e" "\xef\xb7\x9f"
"\xef\xb7\xa0" "\xef\xb7\xa1" "\xef\xb7\xa2" "\xef\xb7\xa3"
"\xef\xb7\xa4" "\xef\xb7\xa5" "\xef\xb7\xa6" "\xef\xb7\xa7"
"\xef\xb7\xa8" "\xef\xb7\xa9" "\xef\xb7\xaa" "\xef\xb7\xab"
"\xef\xb7\xac" "\xef\xb7\xad" "\xef\xb7\xae" "\xef\xb7\xaf",
{
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC
}
},
{
"5.3.4",
"\xf0\x9f\xbf\xbe" "\xf0\x9f\xbf\xbf" // U+0001FFFE U+0001FFFF
"\xf0\xaf\xbf\xbe" "\xf0\xaf\xbf\xbf" // U+0002FFFE U+0002FFFF
"\xf0\xbf\xbf\xbe" "\xf0\xbf\xbf\xbf" // U+0003FFFE U+0003FFFF
"\xf1\x8f\xbf\xbe" "\xf1\x8f\xbf\xbf" // U+0004FFFE U+0004FFFF
"\xf1\x9f\xbf\xbe" "\xf1\x9f\xbf\xbf" // U+0005FFFE U+0005FFFF
"\xf1\xaf\xbf\xbe" "\xf1\xaf\xbf\xbf" // U+0006FFFE U+0006FFFF
"\xf1\xbf\xbf\xbe" "\xf1\xbf\xbf\xbf" // U+0007FFFE U+0007FFFF
"\xf2\x8f\xbf\xbe" "\xf2\x8f\xbf\xbf" // U+0008FFFE U+0008FFFF
"\xf2\x9f\xbf\xbe" "\xf2\x9f\xbf\xbf" // U+0009FFFE U+0009FFFF
"\xf2\xaf\xbf\xbe" "\xf2\xaf\xbf\xbf" // U+000AFFFE U+000AFFFF
"\xf2\xbf\xbf\xbe" "\xf2\xbf\xbf\xbf" // U+000BFFFE U+000BFFFF
"\xf3\x8f\xbf\xbe" "\xf3\x8f\xbf\xbf" // U+000CFFFE U+000CFFFF
"\xf3\x9f\xbf\xbe" "\xf3\x9f\xbf\xbf" // U+000DFFFE U+000DFFFF
"\xf3\xaf\xbf\xbe" "\xf3\xaf\xbf\xbf" // U+000EFFFE U+000EFFFF
"\xf3\xbf\xbf\xbe" "\xf3\xbf\xbf\xbf" // U+000FFFFE U+000FFFFF
"\xf4\x8f\xbf\xbe" "\xf4\x8f\xbf\xbf",// U+0010FFFE U+0010FFFF
{
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC,
URC, URC, URC, URC, URC, URC, URC, URC
}
},
// Last line of UTF8-test.txt
{ "TheEnd", "THE END\n", { 'T', 'H', 'E', ' ', 'E', 'N', 'D', '\n'} },
// Terminator
{ 0 }
};
run_decode_sub_test_array(tests);
}
END_TEST
/******************************************************************************/
START_TEST(test_utf_char32_to_utf8)
{
struct utf8_encode_sub_test tests[] =
{
// E2.1 First possible sequence of a certain length
//
{ "E2.1.1", 0, 1, { 0 } },
{ "E2.1.2", 0x80, 2, { 0xc2, 0x80 } },
{ "E2.1.3", 0x800, 3, { 0xe0, 0xa0, 0x80 } },
{ "E2.1.4", 0x10000, 4, { 0xf0, 0x90, 0x80, 0x80 } },
// E2.2 Last possible sequence of a certain length
{ "E2.2.1", 0x7f, 1, { 0x7f } },
{ "E2.2.2", 0x7ff, 2, { 0xdf, 0xbf } },
{ "E2.2.3", 0xfffc, 3, { 0xef, 0xbf, 0xbc } }, // See 2.1.3 above
{ "E2.2.4", 0x1FFFFF, 3, E_URC }, // out-of-range
// E2.3 Other boundary conditions
{ "E2.3.1", 0xd7ff, 3, { 0xed, 0x9f, 0xbf } },
{ "E2.3.2", 0xe000, 3, { 0xee, 0x80, 0x80 } },
{ "E2.3.3", 0xfffd, 3, { 0xef, 0xbf, 0xbd } },
{ "E2.3.4", 0x10fffd, 4, { 0xf4, 0x8f, 0xbf, 0xbd } }, // See 2.3.4 above
// E2.3.5 - not tested
// E5.1 Single UTF-16 surrogates
{ "E5.1.1", 0xd800, 3, E_URC },
{ "E5.1.2", 0xdb7f, 3, E_URC },
{ "E5.1.3", 0xdb80, 3, E_URC },
{ "E5.1.4", 0xdbff, 3, E_URC },
{ "E5.1.5", 0xdc00, 3, E_URC },
{ "E5.1.6", 0xdf80, 3, E_URC },
{ "E5.1.7", 0xdfff, 3, E_URC },
// E5.3 Non-character code positions
{ "E5.3.3(0)", 0xfdd0, 3, E_URC },
{ "E5.3.3(1)", 0xfdd1, 3, E_URC },
{ "E5.3.3(2)", 0xfdd2, 3, E_URC },
{ "E5.3.3(3)", 0xfdd3, 3, E_URC },
{ "E5.3.3(4)", 0xfdd4, 3, E_URC },
{ "E5.3.3(5)", 0xfdd5, 3, E_URC },
{ "E5.3.3(6)", 0xfdd6, 3, E_URC },
{ "E5.3.3(7)", 0xfdd7, 3, E_URC },
{ "E5.3.3(8)", 0xfdd8, 3, E_URC },
{ "E5.3.3(9)", 0xfdd9, 3, E_URC },
{ "E5.3.3(10)", 0xfdda, 3, E_URC },
{ "E5.3.3(11)", 0xfddb, 3, E_URC },
{ "E5.3.3(12)", 0xfddc, 3, E_URC },
{ "E5.3.3(13)", 0xfddd, 3, E_URC },
{ "E5.3.3(14)", 0xfdde, 3, E_URC },
{ "E5.3.3(15)", 0xfddf, 3, E_URC },
{ "E5.3.3(16)", 0xfde0, 3, E_URC },
{ "E5.3.3(17)", 0xfde1, 3, E_URC },
{ "E5.3.3(18)", 0xfde2, 3, E_URC },
{ "E5.3.3(19)", 0xfde3, 3, E_URC },
{ "E5.3.3(20)", 0xfde4, 3, E_URC },
{ "E5.3.3(21)", 0xfde5, 3, E_URC },
{ "E5.3.3(22)", 0xfde6, 3, E_URC },
{ "E5.3.3(23)", 0xfde7, 3, E_URC },
{ "E5.3.3(24)", 0xfde8, 3, E_URC },
{ "E5.3.3(25)", 0xfde9, 3, E_URC },
{ "E5.3.3(26)", 0xfdea, 3, E_URC },
{ "E5.3.3(27)", 0xfdeb, 3, E_URC },
{ "E5.3.3(28)", 0xfdec, 3, E_URC },
{ "E5.3.3(29)", 0xfded, 3, E_URC },
{ "E5.3.3(30)", 0xfdee, 3, E_URC },
{ "E5.3.3(31)", 0xfdef, 3, E_URC },
{ "E5.3.4(0)", 0x1fffe, 3, E_URC },
{ "E5.3.4(1)", 0x1ffff, 3, E_URC },
{ "E5.3.4(2)", 0x2fffe, 3, E_URC },
{ "E5.3.4(3)", 0x2ffff, 3, E_URC },
{ "E5.3.4(4)", 0x3fffe, 3, E_URC },
{ "E5.3.4(5)", 0x3ffff, 3, E_URC },
{ "E5.3.4(6)", 0x4fffe, 3, E_URC },
{ "E5.3.4(7)", 0x4ffff, 3, E_URC },
{ "E5.3.4(8)", 0x5fffe, 3, E_URC },
{ "E5.3.4(9)", 0x5ffff, 3, E_URC },
{ "E5.3.4(10)", 0x6fffe, 3, E_URC },
{ "E5.3.4(11)", 0x6ffff, 3, E_URC },
{ "E5.3.4(12)", 0x7fffe, 3, E_URC },
{ "E5.3.4(13)", 0x7ffff, 3, E_URC },
{ "E5.3.4(14)", 0x8fffe, 3, E_URC },
{ "E5.3.4(15)", 0x8ffff, 3, E_URC },
{ "E5.3.4(16)", 0x9fffe, 3, E_URC },
{ "E5.3.4(17)", 0x9ffff, 3, E_URC },
{ "E5.3.4(18)", 0xafffe, 3, E_URC },
{ "E5.3.4(19)", 0xaffff, 3, E_URC },
{ "E5.3.4(20)", 0xbfffe, 3, E_URC },
{ "E5.3.4(21)", 0xbffff, 3, E_URC },
{ "E5.3.4(22)", 0xcfffe, 3, E_URC },
{ "E5.3.4(23)", 0xcffff, 3, E_URC },
{ "E5.3.4(24)", 0xdfffe, 3, E_URC },
{ "E5.3.4(25)", 0xdffff, 3, E_URC },
{ "E5.3.4(26)", 0xefffe, 3, E_URC },
{ "E5.3.4(27)", 0xeffff, 3, E_URC },
{ "E5.3.4(28)", 0xffffe, 3, E_URC },
{ "E5.3.4(29)", 0xfffff, 3, E_URC },
{ "E5.3.4(30)", 0x10fffe, 3, E_URC },
{ "E5.3.4(31)", 0x10ffff, 3, E_URC },
{ "E5.99.0", 'T', 1, { 'T' } },
{ "E5.99.1", 'H', 1, { 'H' } },
{ "E5.99.2", 'E', 1, { 'E' } },
{ "E5.99.3", ' ', 1, { ' ' } },
{ "E5.99.4", 'E', 1, { 'E' } },
{ "E5.99.5", 'N', 1, { 'N' } },
{ "E5.99.6", 'D', 1, { 'D' } },
// Terminator
{ 0 }
};
run_encode_sub_test_array(tests);
}
END_TEST
/******************************************************************************/
START_TEST(test_utf8_char_count)
{
// Check function can cope with NULL argument
ck_assert_int_eq(utf8_char_count(NULL), 0);
unsigned int kosme_strlen = strlen(greek_kosme);
unsigned int kosme_len = utf8_char_count(greek_kosme);
// All characters map to two bytes except for the 'omicrom with oxia'
// which maps to three
ck_assert_int_eq(kosme_strlen, 2 + 3 + 2 + 2 + 2);
ck_assert_int_eq(kosme_len, 5);
unsigned int simple_test_strlen = strlen(simple_test_with_emoji);
unsigned int simple_test_len = utf8_char_count(simple_test_with_emoji);
ck_assert_int_eq(simple_test_strlen,
(1 + 1 + 1 + 1 + 1 + 1 ) + // Simple
1 +
(1 + 1 + 1 + 1 ) + // Test
1 +
4); // emoji
// The emoji is 4 bytes - all others are 1
ck_assert_int_eq(simple_test_len, simple_test_strlen - 3);
}
END_TEST
/******************************************************************************/
START_TEST(test_utf8_as_utf16_word_count)
{
unsigned int kosme_count =
utf8_as_utf16_word_count(greek_kosme, strlen(greek_kosme));
ck_assert_int_eq(kosme_count, 5); // All characters in BMP
unsigned int simple_test_count =
utf8_as_utf16_word_count(simple_test_with_emoji,
strlen(simple_test_with_emoji));
ck_assert_int_eq(simple_test_count,
(1 + 1 + 1 + 1 + 1 + 1 ) + // Simple
1 +
(1 + 1 + 1 + 1 ) + // Test
1 +
2); // emoji
}
END_TEST
/******************************************************************************/
START_TEST(test_utf8_add_char_at)
{
#define TEST_SIZE sizeof(simple_test_with_emoji)
// Type pairing a string position with a Unicode char
struct pos_to_char_map
{
unsigned int pos;
char32_t c32;
};
// Buffer for constructing the string
char buff[TEST_SIZE];
// A pseudo-random map of the characters in simple_test_with_emoji
const struct pos_to_char_map map[] =
{
{ 0, 'l' },
{ 0, 'S' },
{ 1, 'i' },
{ 2, 'm' },
{ 4, 0x1f625 },
{ 4, '.' },
{ 4, 'e' },
{ 5, 'T' },
{ 3, 'p' },
{ 7, 't' },
{ 7, 'e' },
{ 8, 's' },
{ 6, ' ' },
{ 0 }
};
buff[0] = '\0';
// Construct the string in a pseudo-random fashion
const struct pos_to_char_map *p;
for (p = map; p->c32 != 0 ; ++p)
{
if (!utf8_add_char_at(buff, TEST_SIZE, p->c32, p->pos))
{
ck_abort_msg("test_utf8_add_char_at: "
"Can't insert char %x at pos %u",
p->c32,
p->pos);
}
}
// Should have reached the buffer size by now
ck_assert_int_eq(strlen(buff), TEST_SIZE - 1);
// Check the string is what we expect
ck_assert_int_eq(strcmp(buff, simple_test_with_emoji), 0);
// Try to insert another character
if (utf8_add_char_at(buff, TEST_SIZE, ' ', 0))
{
ck_abort_msg("test_utf8_add_char_at: "
"Insert succeeded but should have failed");
}
#undef TEST_SIZE
}
END_TEST
/******************************************************************************/
START_TEST(test_utf8_remove_char_at)
{
#define TEST_SIZE sizeof(simple_test_with_emoji)
// Type pairing a string position with a Unicode char
struct pos_to_char_map
{
unsigned int pos;
char32_t c32;
};
// Buffer for deconstructing the string
char buff[TEST_SIZE];
// A pseudo-random map of the characters in simple_test_with_emoji
const struct pos_to_char_map map[] =
{
{ 2, 'm' },
{ 7, 'e' },
{ 5, ' ' },
{ 1, 'i' },
{ 2, 'l' },
{ 3, 'T' },
{ 6, 0x1f625 },
{ 2, 'e' },
{ 3, 't' },
{ 3, '.' },
{ 2, 's' },
{ 1, 'p' },
{ 0, 'S' },
{ 0 }
};
char32_t c32;
strcpy(buff, simple_test_with_emoji);
// Deconstruct the string in a pseudo-random fashion
const struct pos_to_char_map *p;
for (p = map; p->c32 != 0 ; ++p)
{
c32 = utf8_remove_char_at(buff, p->pos);
if (c32 != p->c32)
{
ck_abort_msg("test_utf8_remove_char_at: "
"remove char at pos %u was %x, expected %x",
p->pos, c32, p->c32);
}
}
// Should have emptied the buffer by now
ck_assert_int_eq(buff[0], '\0');
// Try to remove other characters
c32 = utf8_remove_char_at(buff, 0);
ck_assert_int_eq(c32, 0);
c32 = utf8_remove_char_at(buff, 99);
ck_assert_int_eq(c32, 0);
ck_assert_int_eq(buff[0], '\0');
#undef TEST_SIZE
}
END_TEST
/******************************************************************************/
Suite *
make_suite_test_string_unicode(void)
{
Suite *s;
TCase *tc_unicode;
s = suite_create("String");
tc_unicode = tcase_create("Unicode");
suite_add_tcase(s, tc_unicode);
tcase_add_test(tc_unicode, test_get_next_char__section_1);
tcase_add_test(tc_unicode, test_get_next_char__section_2);
tcase_add_test(tc_unicode, test_get_next_char__section_3);
tcase_add_test(tc_unicode, test_get_next_char__section_4);
tcase_add_test(tc_unicode, test_get_next_char__section_5);
tcase_add_test(tc_unicode, test_utf_char32_to_utf8);
tcase_add_test(tc_unicode, test_utf8_char_count);
tcase_add_test(tc_unicode, test_utf8_as_utf16_word_count);
tcase_add_test(tc_unicode, test_utf8_add_char_at);
tcase_add_test(tc_unicode, test_utf8_remove_char_at);
return s;
}
Reference in New Issue View Git Blame Copy Permalink