mirrors
/
micropython
mirror of https://github.com/micropython/micropython.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove the charlen field from strings, calculating it when required

This commit is contained in:
Chris Angelico 2014-06-08 07:18:42 +10:00
parent 5c1658ec71
commit 5473e1a1db
5 changed files with 15 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
py/makeqstrdata.py
View File

@ -59,8 +59,7 @@ def do_work(infiles):
for order, ident, qstr in sorted(qstrs.values(), key=lambda x: x[0]):
qhash = compute_hash(qstr)
qlen = len(qstr)
qchlen = len(qstr.decode("utf-8"))
print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qchlen & 0xff, (qchlen >> 8) & 0xff, qstr))
print('Q({}, (const byte*)"\\x{:02x}\\x{:02x}\\x{:02x}\\x{:02x}" "{}")'.format(ident, qhash & 0xff, (qhash >> 8) & 0xff, qlen & 0xff, (qlen >> 8) & 0xff, qstr))
return True

28
py/objstr.c
View File

@ -52,9 +52,6 @@ const mp_obj_t mp_const_empty_bytes;
// use this macro to extract the string data and length
#define GET_STR_DATA_LEN(str_obj_in, str_data, str_len) const byte *str_data; uint str_len; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_data = ((mp_obj_str_t*)str_obj_in)->data; }
// use this macro to extract the string data and both lengths
#define GET_STR_INFO(str_obj_in, str_data, str_len, str_charlen) const byte *str_data; uint str_len, str_charlen; if (MP_OBJ_IS_QSTR(str_obj_in)) { str_data = qstr_data(MP_OBJ_QSTR_VALUE(str_obj_in), &str_len); str_charlen = qstr_charlen(MP_OBJ_QSTR_VALUE(str_obj_in)); } else { str_len = ((mp_obj_str_t*)str_obj_in)->len; str_charlen = ((mp_obj_str_t*)str_obj_in)->charlen; str_data = ((mp_obj_str_t*)str_obj_in)->data; }
STATIC mp_obj_t mp_obj_new_str_iterator(mp_obj_t str);
STATIC mp_obj_t mp_obj_new_bytes_iterator(mp_obj_t str);
STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in);
@ -365,7 +362,7 @@ uncomparable:
STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
mp_obj_type_t *type = mp_obj_get_type(self_in);
GET_STR_INFO(self_in, self_data, self_len, self_charlen);
GET_STR_DATA_LEN(self_in, self_data, self_len);
if (value == MP_OBJ_SENTINEL) {
// load
#if MICROPY_PY_BUILTINS_SLICE
@ -378,7 +375,8 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) {
return mp_obj_new_str_of_type(type, self_data + slice.start, slice.stop - slice.start);
}
#endif
uint index_val = mp_get_index(type, self_charlen, index, false);
// TODO: Don't use mp_get_index() here
uint index_val = mp_get_index(type, unichar_charlen((const char *)self_data, self_len), index, false);
if (type == &mp_type_bytes) {
return MP_OBJ_NEW_SMALL_INT((mp_small_int_t)self_data[index_val]);
} else {
@ -1734,7 +1732,7 @@ const mp_obj_type_t mp_type_bytes = {
};
// the zero-length bytes
STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, 0, NULL};
STATIC const mp_obj_str_t empty_bytes_obj = {{&mp_type_bytes}, 0, 0, NULL};
const mp_obj_t mp_const_empty_bytes = (mp_obj_t)&empty_bytes_obj;
mp_obj_t mp_obj_str_builder_start(const mp_obj_type_t *type, uint len, byte **data) {
@ -1761,20 +1759,6 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uin
o->base.type = type;
o->len = len;
if (data) {
if (MP_OBJ_IS_STR(o)) {
// Count non-continuation bytes so we know how long the string is in characters.
const byte *endptr, *top = data + len;
uint charlen = 0;
for (endptr = data; endptr < top; ++endptr) {
if (!UTF8_IS_CONT(*endptr)) {
++charlen;
}
}
o->charlen = charlen;
} else {
// For byte strings, the 'character' length (really the "exposed length" or "Python length") equals the byte length.
o->charlen = len;
}
o->hash = qstr_compute_hash(data, len);
byte *p = m_new(byte, len + 1);
o->data = p;
@ -1844,8 +1828,8 @@ uint mp_obj_str_get_hash(mp_obj_t self_in) {
uint mp_obj_str_get_len(mp_obj_t self_in) {
// TODO This has a double check for the type, one in obj.c and one here
if (MP_OBJ_IS_STR(self_in) || MP_OBJ_IS_TYPE(self_in, &mp_type_bytes)) {
GET_STR_INFO(self_in, self_data, self_len, self_charlen); (void)self_data;
return self_charlen;
GET_STR_DATA_LEN(self_in, self_data, self_len);
return unichar_charlen((const char *)self_data, self_len);
} else {
bad_implicit_conversion(self_in);
}

5
py/objstr.h
View File

@ -30,13 +30,10 @@ typedef struct _mp_obj_str_t {
machine_uint_t hash : 16;
// len == number of bytes used in data, alloc = len + 1 because (at the moment) we also append a null byte
machine_uint_t len : 16;
// charlen == number of characters in the string - charlen <= len - 1, and is the value returned by len() in Python
machine_uint_t charlen : 16;
const void *data; //Character data is encoded UTF-8 and should not be blindly indexed.
} mp_obj_str_t;
// This is valid ONLY for pure-ASCII strings!
#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, sizeof(str) - 1, (const byte*)str};
#define MP_DEFINE_STR_OBJ(obj_name, str) mp_obj_str_t obj_name = {{&mp_type_str}, 0, sizeof(str) - 1, (const byte*)str};
mp_obj_t mp_obj_str_format(uint n_args, const mp_obj_t *args);
mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, uint len);

37
py/qstr.c
View File

@ -46,15 +46,13 @@
// For now we use very simple encoding, just to get the framework correct:
// - hash is 2 bytes (see function below)
// - length is 2 bytes
// - character length is 2 bytes
// - data follows
// - \0 terminated (for now, so they can be printed using printf)
#define Q_GET_HASH(q) ((q)[0] | ((q)[1] << 8))
#define Q_GET_ALLOC(q) (6 + Q_GET_LENGTH(q) + 1)
#define Q_GET_ALLOC(q) (4 + Q_GET_LENGTH(q) + 1)
#define Q_GET_LENGTH(q) ((q)[2] | ((q)[3] << 8))
#define Q_GET_CHARLEN(q) ((q)[4] | ((q)[5] << 8))
#define Q_GET_DATA(q) ((q) + 6)
#define Q_GET_DATA(q) ((q) + 4)
// this must match the equivalent function in makeqstrdata.py
// Note that this hashes the UTF-8 encoded data bytes.
@ -158,21 +156,13 @@ qstr qstr_from_strn(const char *str, uint len) {
qstr q = qstr_find_strn(str, len);
if (q == 0) {
machine_uint_t hash = qstr_compute_hash((const byte*)str, len);
byte *q_ptr = m_new(byte, 6 + len + 1);
uint charlen = 0;
for (const char *s = str; s < str + len; ++s) {
if (!UTF8_IS_CONT(*s)) {
++charlen;
}
}
byte *q_ptr = m_new(byte, 4 + len + 1);
q_ptr[0] = hash;
q_ptr[1] = hash >> 8;
q_ptr[2] = len;
q_ptr[3] = len >> 8;
q_ptr[4] = charlen;
q_ptr[5] = charlen >> 8;
memcpy(q_ptr + 6, str, len);
q_ptr[6 + len] = '\0';
memcpy(q_ptr + 4, str, len);
q_ptr[4 + len] = '\0';
q = qstr_add(q_ptr);
}
return q;
@ -180,7 +170,7 @@ qstr qstr_from_strn(const char *str, uint len) {
byte *qstr_build_start(uint len, byte **q_ptr) {
assert(len <= 65535);
*q_ptr = m_new(byte, 7 + len + 1);
*q_ptr = m_new(byte, 4 + len + 1);
(*q_ptr)[2] = len;
(*q_ptr)[3] = len >> 8;
return Q_GET_DATA(*q_ptr);
@ -194,15 +184,7 @@ qstr qstr_build_end(byte *q_ptr) {
machine_uint_t hash = qstr_compute_hash(str, len);
q_ptr[0] = hash;
q_ptr[1] = hash >> 8;
uint charlen = 0;
for (const byte *s = str; s < str + len; ++s) {
if (!UTF8_IS_CONT(*s)) {
++charlen;
}
}
q_ptr[4] = charlen;
q_ptr[5] = charlen >> 8;
q_ptr[6 + len] = '\0';
q_ptr[4 + len] = '\0';
q = qstr_add(q_ptr);
} else {
m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr));
@ -219,11 +201,6 @@ uint qstr_len(qstr q) {
return Q_GET_LENGTH(qd);
}
uint qstr_charlen(qstr q) {
const byte *qd = find_qstr(q);
return Q_GET_CHARLEN(qd);
}
// XXX to remove!
const char *qstr_str(qstr q) {
const byte *qd = find_qstr(q);

1
py/qstr.h
View File

@ -59,7 +59,6 @@ qstr qstr_build_end(byte *q_ptr);
machine_uint_t qstr_hash(qstr q);
const char* qstr_str(qstr q);
uint qstr_len(qstr q);
uint qstr_charlen(qstr q);
const byte* qstr_data(qstr q, uint *len);
void qstr_pool_info(uint *n_pool, uint *n_qstr, uint *n_str_data_bytes, uint *n_total_bytes);