mirrors/micropython
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Retain characters as UTF-8 encoded Unicode

Browse Source
This commit is contained in:
Chris Angelico 2014-06-07 10:24:35 +10:00
parent e924659b85
commit 03f0cbe905
2 changed files with 40 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
py/lexer.c
View File

@ -546,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
}
}
if (c != MP_LEXER_CHAR_EOF) {
vstr_add_char(&lex->vstr, c);
if (c < 0x110000 && !is_bytes) {
vstr_add_char(&lex->vstr, c);
} else if (c < 0x100 && is_bytes) {
vstr_add_byte(&lex->vstr, c);
} else {
assert(!"TODO: Throw an error, invalid escape code probably");
}
}
} else {
vstr_add_char(&lex->vstr, CUR_CHAR(lex));

38
py/vstr.c
View File

@ -199,12 +199,40 @@ void vstr_add_byte(vstr_t *vstr, byte b) {
}
void vstr_add_char(vstr_t *vstr, unichar c) {
// TODO UNICODE
byte *buf = (byte*)vstr_add_len(vstr, 1);
if (buf == NULL) {
return;
// TODO: Can this be simplified and deduplicated?
// Is it worth just calling vstr_add_len(vstr, 4)?
if (c < 0x80) {
byte *buf = (byte*)vstr_add_len(vstr, 1);
if (buf == NULL) {
return;
}
*buf = (byte)c;
} else if (c < 0x800) {
byte *buf = (byte*)vstr_add_len(vstr, 2);
if (buf == NULL) {
return;
}
buf[0] = (c >> 6) | 0xC0;
buf[1] = (c & 0x3F) | 0x80;
} else if (c < 0x10000) {
byte *buf = (byte*)vstr_add_len(vstr, 3);
if (buf == NULL) {
return;
}
buf[0] = (c >> 12) | 0xE0;
buf[1] = ((c >> 6) & 0x3F) | 0x80;
buf[2] = (c & 0x3F) | 0x80;
} else {
assert(c < 0x110000);
byte *buf = (byte*)vstr_add_len(vstr, 4);
if (buf == NULL) {
return;
}
buf[0] = (c >> 18) | 0xF0;
buf[1] = ((c >> 12) & 0x3F) | 0x80;
buf[2] = ((c >> 6) & 0x3F) | 0x80;
buf[3] = (c & 0x3F) | 0x80;
}
buf[0] = c;
}
void vstr_add_str(vstr_t *vstr, const char *str) {