objstrunicode: Basic implementation of unicode handling.
Squashed commit of the following: commit99dc21b67a
Author: Chris Angelico <rosuav@gmail.com> Date: Thu Jun 12 02:18:54 2014 +1000 Optimize as per TODO (thanks Damien!) commit5bf0153eca
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 08:42:06 2014 +1000 Test a default (= UTF-8) encode and decode commitc962057ac3
Merge:e2c9782
195de32
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 05:23:03 2014 +1000 Merge branch 'master' into unicode, resolving conflict on py/obj.h commite2c9782a65
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 05:05:57 2014 +1000 More whitespace fixups commit086a2a0f57
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 05:04:20 2014 +1000 Properly implement string slicing commit0d339a143e
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 02:24:11 2014 +1000 Support slicing in str_index_to_ptr, and fix a bounds error commit24371c7267
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 02:10:22 2014 +1000 Break out index-to-pointer calculation into a function commit616c24ac01
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 02:03:11 2014 +1000 Add tests of string slicing, which currently fail commita24d19f676
Author: Chris Angelico <rosuav@gmail.com> Date: Tue Jun 10 01:56:53 2014 +1000 Change string indexing to not precalculate the charlen, and add test for neg indexing commit0bcc7ab89e
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 22:09:17 2014 +1000 Clean up constant qstr declarations now that charlen isn't needed commit5473e1a1db
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 07:18:42 2014 +1000 Remove the charlen field from strings, calculating it when required commit5c1658ec71
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 07:11:27 2014 +1000 Get rid of mp_obj_str_get_data_len() which was used in only one place commita019ba968b
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 06:58:26 2014 +1000 Add a unichar_charlen() function to calculate length-in-characters from length-in-bytes commit44b0d5cff8
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 06:32:44 2014 +1000 Use utf8_get/next_char in building up a string's repr commit30d1bad33f
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 06:10:45 2014 +1000 Make utf8_get_char() and utf8_next_char() actually do what their names say commitbc990dad9a
Author: Chris Angelico <rosuav@gmail.com> Date: Sun Jun 8 02:10:59 2014 +1000 Revert "Add PEP 393-flags to strings and stub usage." This reverts commitc239f50952
. commitf9bebb28ad
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 15:41:48 2014 +1000 Whitespace fixes commit279de0c8eb
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 15:28:35 2014 +1000 Formatting/layout improvements - introduce macros for UTF-8 byte detection, add braces. No functional changes. commitf1911f53d5
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 11:56:02 2014 +1000 Make chr() Unicode-aware commitf51ad737b4
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 11:44:07 2014 +1000 Make a string's repr Unicode-aware commit01bd686846
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 11:33:43 2014 +1000 Expand the Unicode tests commit7bc91904f8
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 11:27:30 2014 +1000 Record byte lengths for byte strings commitbb13212071
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 11:25:06 2014 +1000 Make ord() Unicode-aware commit03f0cbe905
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 10:24:35 2014 +1000 Retain characters as UTF-8 encoded Unicode commite924659b85
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 08:37:27 2014 +1000 Add support for \u and \U escapes, but not \N (with explanatory comment) commit231031ac5f
Author: Chris Angelico <rosuav@gmail.com> Date: Sat Jun 7 05:09:35 2014 +1000 Add character length to qstr commit6df1b946fb
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 13:48:36 2014 +1000 Add test of UTF-8 encoded source file resulting in properly formed string commit16429b81a8
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 13:44:15 2014 +1000 Make len(s) return character length (even though creation's still buggy) commitcd2cf6663c
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 13:15:36 2014 +1000 HACK - When indexing a qstr, count its charlen. Stupidly inefficient but POC. All tests pass now, though string creation is still buggy. commit47c234584d
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 13:15:32 2014 +1000 objstr: Record character length separately from byte length CAUTION: Buggy, may crash stuff - qstr needs equivalent functionality too commitb0f41c72af
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 05:37:36 2014 +1000 Beginnings of UTF-8 support - construct strings from that many UTF-8-encoded chars, and subscript bytes the same way commit89452be641
Author: Chris Angelico <rosuav@gmail.com> Date: Fri Jun 6 05:28:47 2014 +1000 Update comments - now aiming for UTF-8 rather than PEP 393 strings commitc239f50952
Author: Chris Angelico <rosuav@gmail.com> Date: Wed Jun 4 05:28:12 2014 +1000 Add PEP 393-flags to strings and stub usage. The test suite all passes, but nothing has actually been changed.
This commit is contained in:
parent
83865347db
commit
64b468d873
|
@ -64,7 +64,7 @@ STATIC bool is_str_or_bytes(mp_obj_t o) {
|
|||
/******************************************************************************/
|
||||
/* str */
|
||||
|
||||
void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len) {
|
||||
void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *env, const byte *str_data, uint str_len, bool is_bytes) {
|
||||
// this escapes characters, but it will be very slow to print (calling print many times)
|
||||
bool has_single_quote = false;
|
||||
bool has_double_quote = false;
|
||||
|
@ -80,21 +80,33 @@ void mp_str_print_quoted(void (*print)(void *env, const char *fmt, ...), void *e
|
|||
quote_char = '"';
|
||||
}
|
||||
print(env, "%c", quote_char);
|
||||
for (const byte *s = str_data, *top = str_data + str_len; s < top; s++) {
|
||||
if (*s == quote_char) {
|
||||
print(env, "\\%c", quote_char);
|
||||
} else if (*s == '\\') {
|
||||
print(env, "\\\\");
|
||||
} else if (32 <= *s && *s <= 126) {
|
||||
print(env, "%c", *s);
|
||||
} else if (*s == '\n') {
|
||||
print(env, "\\n");
|
||||
} else if (*s == '\r') {
|
||||
print(env, "\\r");
|
||||
} else if (*s == '\t') {
|
||||
print(env, "\\t");
|
||||
const char *s = (const char *)str_data, *top = (const char *)str_data + str_len;
|
||||
while (s < top) {
|
||||
unichar ch;
|
||||
if (is_bytes) {
|
||||
ch = *(unsigned char *)s++; // Don't sign-extend bytes
|
||||
} else {
|
||||
print(env, "\\x%02x", *s);
|
||||
ch = utf8_get_char(s);
|
||||
s = utf8_next_char(s);
|
||||
}
|
||||
if (ch == quote_char) {
|
||||
print(env, "\\%c", quote_char);
|
||||
} else if (ch == '\\') {
|
||||
print(env, "\\\\");
|
||||
} else if (32 <= ch && ch <= 126) {
|
||||
print(env, "%c", ch);
|
||||
} else if (ch == '\n') {
|
||||
print(env, "\\n");
|
||||
} else if (ch == '\r') {
|
||||
print(env, "\\r");
|
||||
} else if (ch == '\t') {
|
||||
print(env, "\\t");
|
||||
} else if (ch < 0x100) {
|
||||
print(env, "\\x%02x", ch);
|
||||
} else if (ch < 0x10000) {
|
||||
print(env, "\\u%04x", ch);
|
||||
} else {
|
||||
print(env, "\\U%08x", ch);
|
||||
}
|
||||
}
|
||||
print(env, "%c", quote_char);
|
||||
|
@ -109,7 +121,7 @@ STATIC void str_print(void (*print)(void *env, const char *fmt, ...), void *env,
|
|||
if (is_bytes) {
|
||||
print(env, "b");
|
||||
}
|
||||
mp_str_print_quoted(print, env, str_data, str_len);
|
||||
mp_str_print_quoted(print, env, str_data, str_len, is_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -348,6 +360,59 @@ uncomparable:
|
|||
return MP_OBJ_NULL; // op not supported
|
||||
}
|
||||
|
||||
// Convert an index into a pointer to its lead byte. Out of bounds indexing will raise IndexError or
|
||||
// be capped to the first/last character of the string, depending on is_slice.
|
||||
STATIC const char *str_index_to_ptr(const char *self_data, uint self_len, mp_obj_t index, bool is_slice) {
|
||||
machine_int_t i;
|
||||
// Copied from mp_get_index; I don't want bounds checking, just give me
|
||||
// the integer as-is. (I can't bounds-check without scanning the whole
|
||||
// string; an out-of-bounds index will be caught in the loops below.)
|
||||
if (MP_OBJ_IS_SMALL_INT(index)) {
|
||||
i = MP_OBJ_SMALL_INT_VALUE(index);
|
||||
} else if (!mp_obj_get_int_maybe(index, &i)) {
|
||||
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, "string indices must be integers, not %s", mp_obj_get_type_str(index)));
|
||||
}
|
||||
const char *s, *top = self_data + self_len;
|
||||
if (i < 0)
|
||||
{
|
||||
// Negative indexing is performed by counting from the end of the string.
|
||||
for (s = top - 1; i; --s) {
|
||||
if (s < self_data) {
|
||||
if (is_slice) {
|
||||
return self_data;
|
||||
}
|
||||
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_IndexError, "string index out of range"));
|
||||
}
|
||||
if (!UTF8_IS_CONT(*s)) {
|
||||
++i;
|
||||
}
|
||||
}
|
||||
++s;
|
||||
} else if (!i) {
|
||||
return self_data; // Shortcut - str[0] is its base pointer
|
||||
} else {
|
||||
// Positive indexing, correspondingly, counts from the start of the string.
|
||||
// It's assumed that negative indexing will generally be used with small
|
||||
// absolute values (eg str[-1], not str[-1000000]), which means it'll be
|
||||
// more efficient this way.
|
||||
for (s = self_data; true; ++s) {
|
||||
if (s >= top) {
|
||||
if (is_slice) {
|
||||
return top;
|
||||
}
|
||||
nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_IndexError, "string index out of range"));
|
||||
}
|
||||
while (UTF8_IS_CONT(*s)) {
|
||||
++s;
|
||||
}
|
||||
if (!i--) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
|
||||
mp_obj_type_t *type = mp_obj_get_type(self_in);
|
||||
GET_STR_DATA_LEN(self_in, self_data, self_len);
|
||||
|
@ -355,20 +420,61 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
|
|||
// load
|
||||
#if MICROPY_PY_BUILTINS_SLICE
|
||||
if (MP_OBJ_IS_TYPE(index, &mp_type_slice)) {
|
||||
mp_bound_slice_t slice;
|
||||
if (!mp_seq_get_fast_slice_indexes(self_len, index, &slice)) {
|
||||
mp_obj_t ostart, ostop, ostep;
|
||||
mp_obj_slice_get(index, &ostart, &ostop, &ostep);
|
||||
if (ostep != mp_const_none && ostep != MP_OBJ_NEW_SMALL_INT(1)) {
|
||||
nlr_raise(mp_obj_new_exception_msg(&mp_type_NotImplementedError,
|
||||
"only slices with step=1 (aka None) are supported"));
|
||||
}
|
||||
return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start);
|
||||
|
||||
if (type == &mp_type_bytes) {
|
||||
machine_int_t start = 0, stop = self_len;
|
||||
if (ostart != mp_const_none) {
|
||||
start = MP_OBJ_SMALL_INT_VALUE(ostart);
|
||||
if (start < 0) {
|
||||
start = self_len + start;
|
||||
}
|
||||
}
|
||||
if (ostop != mp_const_none) {
|
||||
stop = MP_OBJ_SMALL_INT_VALUE(ostop);
|
||||
if (stop < 0) {
|
||||
stop = self_len + stop;
|
||||
}
|
||||
}
|
||||
return mp_obj_new_str_of_type(type, self_data + start, stop - start);
|
||||
}
|
||||
const char *pstart, *pstop;
|
||||
if (ostart != mp_const_none) {
|
||||
pstart = str_index_to_ptr((const char *)self_data, self_len, ostart, true);
|
||||
} else {
|
||||
pstart = (const char *)self_data;
|
||||
}
|
||||
if (ostop != mp_const_none) {
|
||||
// pstop will point just after the stop character. This depends on
|
||||
// the \0 at the end of the string.
|
||||
pstop = str_index_to_ptr((const char *)self_data, self_len, ostop, true);
|
||||
} else {
|
||||
pstop = (const char *)self_data + self_len;
|
||||
}
|
||||
if (pstop < pstart) {
|
||||
return MP_OBJ_NEW_QSTR(MP_QSTR_);
|
||||
}
|
||||
return mp_obj_new_str_of_type(type, (const byte *)pstart, pstop - pstart);
|
||||
}
|
||||
#endif
|
||||
uint index_val = mp_get_index(type, self_len, index, false);
|
||||
if (type == &mp_type_bytes) {
|
||||
uint index_val = mp_get_index(type, self_len, index, false);
|
||||
return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]);
|
||||
} else {
|
||||
return mp_obj_new_str((char*)self_data + index_val, 1, true);
|
||||
}
|
||||
const char *s = str_index_to_ptr((const char *)self_data, self_len, index, false);
|
||||
int len = 1;
|
||||
if (UTF8_IS_NONASCII(*s)) {
|
||||
// Count the number of 1 bits (after the first)
|
||||
for (char mask = 0x40; *s & mask; mask >>= 1) {
|
||||
++len;
|
||||
}
|
||||
}
|
||||
return mp_obj_new_str(s, len, true); // This will create a one-character string
|
||||
} else {
|
||||
return MP_OBJ_NULL; // op not supported
|
||||
}
|
||||
|
@ -1800,8 +1906,8 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) {
|
|||
uint mp_obj_str_get_len(mp_obj_t self_in) {
|
||||
// TODO This has a double check for the type, one in obj.c and one here
|
||||
if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) {
|
||||
GET_STR_LEN(self_in, l);
|
||||
return l;
|
||||
GET_STR_DATA_LEN(self_in, self_data, self_len);
|
||||
return unichar_charlen((const char *)self_data, self_len);
|
||||
} else {
|
||||
bad_implicit_conversion(self_in);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue