[project @ 2005-04-16 05:09:32 by jmb]
Split out UTF-8 handling functions. Submit URL-encoded forms in sensible encoding: * First entry in accept-charset list, if present * Document encoding, otherwise We may want to explicitly look for UTF-8, to save converting. Convert cnv_str_local_enc/cnv_local_enc_str to use iconv (they're now veneers for utf8_[to/from]_enc). Provide mechanism for looking up local system charset (derived from system alphabet, under RISC OS) svn path=/import/netsurf/; revision=1647
This commit is contained in:
John Mark Bell
parent
37a4119ab8
commit
c17dc661ea
|
|
@ -1492,7 +1492,7 @@ void browser_form_submit(struct browser_window *bw, struct form *form,
|
|||
|
||||
switch (form->method) {
|
||||
case method_GET:
|
||||
data = form_url_encode(success);
|
||||
data = form_url_encode(form, success);
|
||||
if (!data) {
|
||||
form_free_successful(success);
|
||||
warn_user("NoMemory", 0);
|
||||
|
|
@ -1517,7 +1517,7 @@ void browser_form_submit(struct browser_window *bw, struct form *form,
|
|||
break;
|
||||
|
||||
case method_POST_URLENC:
|
||||
data = form_url_encode(success);
|
||||
data = form_url_encode(form, success);
|
||||
if (!data) {
|
||||
form_free_successful(success);
|
||||
warn_user("NoMemory", 0);
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
#define NDEBUG
|
||||
#include "netsurf/utils/log.h"
|
||||
#include "netsurf/utils/talloc.h"
|
||||
#include "netsurf/utils/utf8.h"
|
||||
#include "netsurf/utils/utils.h"
|
||||
|
||||
static void browser_window_textarea_callback(struct browser_window *bw,
|
||||
|
|
@ -37,121 +38,6 @@ static void browser_window_place_caret(struct browser_window *bw,
|
|||
wchar_t key, void *p),
|
||||
void *p);
|
||||
|
||||
/**
|
||||
* Convert a single UCS4 character into a UTF8 multibyte sequence
|
||||
*
|
||||
* Encoding of UCS values outside the UTF16 plane has been removed from
|
||||
* RFC3629. This macro conforms to RFC2279, however, as it is possible
|
||||
* that the platform specific keyboard input handler will generate a UCS4
|
||||
* value outside the UTF16 plane.
|
||||
*
|
||||
* \param c The character to process (0 <= c <= 0x7FFFFFFF)
|
||||
* \param s Pointer to 6 byte long output buffer
|
||||
* \param l Integer in which to store length of multibyte sequence
|
||||
*/
|
||||
#define ucs4_to_utf8(c, s, l) \
|
||||
do { \
|
||||
if ((c) < 0) \
|
||||
assert(0); \
|
||||
else if ((c) < 0x80) { \
|
||||
*(s) = (char)(c); \
|
||||
(l) = 1; \
|
||||
} \
|
||||
else if ((c) < 0x800) { \
|
||||
*(s) = 0xC0 | (((c) >> 6) & 0x1F); \
|
||||
*((s)+1) = 0x80 | ((c) & 0x3F); \
|
||||
(l) = 2; \
|
||||
} \
|
||||
else if ((c) < 0x10000) { \
|
||||
*(s) = 0xE0 | (((c) >> 12) & 0xF); \
|
||||
*((s)+1) = 0x80 | (((c) >> 6) & 0x3F); \
|
||||
*((s)+2) = 0x80 | ((c) & 0x3F); \
|
||||
(l) = 3; \
|
||||
} \
|
||||
else if ((c) < 0x200000) { \
|
||||
*(s) = 0xF0 | (((c) >> 18) & 0x7); \
|
||||
*((s)+1) = 0x80 | (((c) >> 12) & 0x3F); \
|
||||
*((s)+2) = 0x80 | (((c) >> 6) & 0x3F); \
|
||||
*((s)+3) = 0x80 | ((c) & 0x3F); \
|
||||
(l) = 4; \
|
||||
} \
|
||||
else if ((c) < 0x4000000) { \
|
||||
*(s) = 0xF8 | (((c) >> 24) & 0x3); \
|
||||
*((s)+1) = 0x80 | (((c) >> 18) & 0x3F); \
|
||||
*((s)+2) = 0x80 | (((c) >> 12) & 0x3F); \
|
||||
*((s)+3) = 0x80 | (((c) >> 6) & 0x3F); \
|
||||
*((s)+4) = 0x80 | ((c) & 0x3F); \
|
||||
(l) = 5; \
|
||||
} \
|
||||
else if ((c) <= 0x7FFFFFFF) { \
|
||||
*(s) = 0xFC | (((c) >> 30) & 0x1); \
|
||||
*((s)+1) = 0x80 | (((c) >> 24) & 0x3F); \
|
||||
*((s)+2) = 0x80 | (((c) >> 18) & 0x3F); \
|
||||
*((s)+3) = 0x80 | (((c) >> 12) & 0x3F); \
|
||||
*((s)+4) = 0x80 | (((c) >> 6) & 0x3F); \
|
||||
*((s)+5) = 0x80 | ((c) & 0x3F); \
|
||||
(l) = 6; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Calculate the length (in characters) of a NULL-terminated UTF8 string
|
||||
*
|
||||
* \param s The string
|
||||
* \param l Integer in which to store length
|
||||
*/
|
||||
#define utf8_length(s, l) \
|
||||
do { \
|
||||
char *__s = (s); \
|
||||
(l) = 0; \
|
||||
while (*__s != '\0') { \
|
||||
if ((*__s & 0x80) == 0x00) \
|
||||
__s += 1; \
|
||||
else if ((*__s & 0xE0) == 0xC0) \
|
||||
__s += 2; \
|
||||
else if ((*__s & 0xF0) == 0xE0) \
|
||||
__s += 3; \
|
||||
else if ((*__s & 0xF8) == 0xF0) \
|
||||
__s += 4; \
|
||||
else if ((*__s & 0xFC) == 0xF8) \
|
||||
__s += 5; \
|
||||
else if ((*__s & 0xFE) == 0xFC) \
|
||||
__s += 6; \
|
||||
else \
|
||||
assert(0); \
|
||||
(l)++; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/**
|
||||
* Find previous legal UTF8 char in string
|
||||
*
|
||||
* \param s The string
|
||||
* \param o Offset in the string to start at (updated on exit)
|
||||
*/
|
||||
#define utf8_prev(s, o) \
|
||||
do { \
|
||||
while ((o) != 0 && \
|
||||
!((((s)[--(o)] & 0x80) == 0x00) || \
|
||||
(((s)[(o)] & 0xC0) == 0xC0))) \
|
||||
/* do nothing */; \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Find next legal UTF8 char in string
|
||||
*
|
||||
* \param s The string
|
||||
* \param l Maximum offset in string
|
||||
* \param o Offset in the string to start at (updated on exit)
|
||||
*/
|
||||
#define utf8_next(s, l, o) \
|
||||
do { \
|
||||
while ((o) != (l) && \
|
||||
!((((s)[++(o)] & 0x80) == 0x00) || \
|
||||
(((s)[(o)] & 0xC0) == 0xC0))) \
|
||||
/* do nothing */; \
|
||||
} while(0)
|
||||
|
||||
/**
|
||||
* Handle clicks in a text area by placing the caret.
|
||||
*
|
||||
|
|
@ -300,7 +186,7 @@ void browser_window_textarea_callback(struct browser_window *bw,
|
|||
|
||||
if (!(key <= 0x001F || (0x007F <= key && key <= 0x009F))) {
|
||||
/* normal character insertion */
|
||||
ucs4_to_utf8(key, utf8, utf8_len);
|
||||
utf8_len = utf8_from_ucs4(key, utf8);
|
||||
|
||||
text = talloc_realloc(bw->current_content, text_box->text,
|
||||
char, text_box->length + 8);
|
||||
|
|
@ -368,7 +254,7 @@ void browser_window_textarea_callback(struct browser_window *bw,
|
|||
} else {
|
||||
/* delete a character */
|
||||
int prev_offset = char_offset;
|
||||
utf8_prev(text_box->text, char_offset);
|
||||
char_offset = utf8_prev(text_box->text, char_offset);
|
||||
|
||||
memmove(text_box->text + char_offset,
|
||||
text_box->text + prev_offset,
|
||||
|
|
@ -429,8 +315,8 @@ void browser_window_textarea_callback(struct browser_window *bw,
|
|||
|
||||
case 28: /* Right cursor -> */
|
||||
if ((unsigned int) char_offset != text_box->length) {
|
||||
utf8_next(text_box->text, text_box->length,
|
||||
char_offset);
|
||||
char_offset = utf8_next(text_box->text,
|
||||
text_box->length, char_offset);
|
||||
} else {
|
||||
if (!text_box->next)
|
||||
/* at end of text area: ignore */
|
||||
|
|
@ -445,7 +331,7 @@ void browser_window_textarea_callback(struct browser_window *bw,
|
|||
|
||||
case 29: /* Left cursor <- */
|
||||
if (char_offset != 0) {
|
||||
utf8_prev(text_box->text, char_offset);
|
||||
char_offset = utf8_prev(text_box->text, char_offset);
|
||||
} else {
|
||||
if (!text_box->prev)
|
||||
/* at start of text area: ignore */
|
||||
|
|
@ -656,14 +542,14 @@ void browser_window_input_callback(struct browser_window *bw,
|
|||
char *value;
|
||||
|
||||
/* have we exceeded max length of input? */
|
||||
utf8_length(input->gadget->value, utf8_len);
|
||||
utf8_len = utf8_length(input->gadget->value);
|
||||
if (utf8_len >= input->gadget->maxlength)
|
||||
return;
|
||||
|
||||
/* normal character insertion */
|
||||
|
||||
/* Insert key in gadget */
|
||||
ucs4_to_utf8(key, utf8, utf8_len);
|
||||
utf8_len = utf8_from_ucs4(key, utf8);
|
||||
|
||||
value = realloc(input->gadget->value,
|
||||
input->gadget->length + utf8_len + 1);
|
||||
|
|
@ -683,9 +569,10 @@ void browser_window_input_callback(struct browser_window *bw,
|
|||
|
||||
/* Insert key in text box */
|
||||
/* Convert space into NBSP */
|
||||
ucs4_to_utf8((input->gadget->type == GADGET_PASSWORD) ?
|
||||
utf8_len = utf8_from_ucs4(
|
||||
(input->gadget->type == GADGET_PASSWORD) ?
|
||||
'*' : (key == ' ') ? 160 : key,
|
||||
utf8, utf8_len);
|
||||
utf8);
|
||||
|
||||
value = talloc_realloc(bw->current_content, text_box->text,
|
||||
char, text_box->length + utf8_len + 1);
|
||||
|
|
@ -718,7 +605,8 @@ void browser_window_input_callback(struct browser_window *bw,
|
|||
/* Gadget */
|
||||
prev_offset = form_offset;
|
||||
/* Go to the previous valid UTF-8 character */
|
||||
utf8_prev(input->gadget->value, form_offset);
|
||||
form_offset = utf8_prev(input->gadget->value,
|
||||
form_offset);
|
||||
|
||||
memmove(input->gadget->value + form_offset,
|
||||
input->gadget->value + prev_offset,
|
||||
|
|
@ -729,7 +617,7 @@ void browser_window_input_callback(struct browser_window *bw,
|
|||
/* Text box */
|
||||
prev_offset = box_offset;
|
||||
/* Go to the previous valid UTF-8 character */
|
||||
utf8_prev(text_box->text, box_offset);
|
||||
box_offset = utf8_prev(text_box->text, box_offset);
|
||||
|
||||
memmove(text_box->text + box_offset,
|
||||
text_box->text + prev_offset,
|
||||
|
|
@ -810,20 +698,21 @@ void browser_window_input_callback(struct browser_window *bw,
|
|||
case 28: /* Right cursor -> */
|
||||
/* Text box */
|
||||
/* Go to the next valid UTF-8 character */
|
||||
utf8_next(text_box->text, text_box->length, box_offset);
|
||||
box_offset = utf8_next(text_box->text, text_box->length,
|
||||
box_offset);
|
||||
/* Gadget */
|
||||
/* Go to the next valid UTF-8 character */
|
||||
utf8_next(input->gadget->value, input->gadget->length,
|
||||
form_offset);
|
||||
form_offset = utf8_next(input->gadget->value,
|
||||
input->gadget->length, form_offset);
|
||||
break;
|
||||
|
||||
case 29: /* Left cursor -> */
|
||||
/* Text box */
|
||||
/* Go to the previous valid UTF-8 character */
|
||||
utf8_prev(text_box->text, box_offset);
|
||||
box_offset = utf8_prev(text_box->text, box_offset);
|
||||
/* Gadget */
|
||||
/* Go to the previous valid UTF-8 character */
|
||||
utf8_prev(input->gadget->value, form_offset);
|
||||
form_offset = utf8_prev(input->gadget->value, form_offset);
|
||||
break;
|
||||
|
||||
case 128: /* Ctrl + Left */
|
||||
|
|
|
|||
3
makefile
3
makefile
|
|
@ -21,7 +21,8 @@ OBJECTS_COMMON = content.o fetch.o fetchcache.o url_store.o # content/
|
|||
OBJECTS_COMMON += css.o css_enum.o parser.o ruleset.o scanner.o # css/
|
||||
OBJECTS_COMMON += box.o box_construct.o box_normalise.o form.o html.o \
|
||||
html_redraw.o layout.o list.o textplain.o # render/
|
||||
OBJECTS_COMMON += messages.o pool.o talloc.o url.o utils.o # utils/
|
||||
OBJECTS_COMMON += messages.o pool.o talloc.o url.o utf8.c \
|
||||
utils.o # utils/
|
||||
OBJECTS_COMMON += imagemap.o loginlist.o options.o selection.o \
|
||||
textinput.o tree.o # desktop/
|
||||
|
||||
|
|
|
|||
|
|
@ -1575,7 +1575,7 @@ bool box_iframe(BOX_SPECIAL_PARAMS)
|
|||
|
||||
bool box_form(BOX_SPECIAL_PARAMS)
|
||||
{
|
||||
char *action, *method, *enctype;
|
||||
char *action, *method, *enctype, *charset;
|
||||
form_method fmethod;
|
||||
struct form *form;
|
||||
|
||||
|
|
@ -1598,9 +1598,35 @@ bool box_form(BOX_SPECIAL_PARAMS)
|
|||
xmlFree(method);
|
||||
}
|
||||
|
||||
form = form_new(action, fmethod);
|
||||
/* acceptable encoding(s) for form data */
|
||||
if ((charset = (char *) xmlGetProp(n, (const xmlChar *) "accept-charset"))) {
|
||||
char *comma = strchr(charset, ',');
|
||||
if (!comma)
|
||||
/* only one => use it */
|
||||
comma = strdup(charset);
|
||||
else
|
||||
/* multiple => use first */
|
||||
comma = strndup(charset, comma - charset);
|
||||
|
||||
xmlFree(charset);
|
||||
charset = comma;
|
||||
}
|
||||
else if (content->data.html.encoding)
|
||||
/* none specified => try document encoding */
|
||||
charset = strdup(content->data.html.encoding);
|
||||
else
|
||||
/* none specified and no document encoding => 8859-1 */
|
||||
charset = strdup("ISO-8859-1");
|
||||
|
||||
if (!charset) {
|
||||
xmlFree(action);
|
||||
return false;
|
||||
}
|
||||
|
||||
form = form_new(action, fmethod, charset);
|
||||
if (!form) {
|
||||
xmlFree(action);
|
||||
free(charset);
|
||||
return false;
|
||||
}
|
||||
form->prev = content->data.html.forms;
|
||||
|
|
|
|||
|
|
@ -17,21 +17,22 @@
|
|||
#include "netsurf/render/box.h"
|
||||
#include "netsurf/render/form.h"
|
||||
#include "netsurf/utils/log.h"
|
||||
#include "netsurf/utils/utf8.h"
|
||||
#include "netsurf/utils/utils.h"
|
||||
|
||||
|
||||
static char *form_textarea_value(struct form_control *textarea);
|
||||
|
||||
|
||||
/**
|
||||
* Create a struct form.
|
||||
*
|
||||
* \param action URL to submit form to, used directly (not copied)
|
||||
* \param method method and enctype
|
||||
* \param charset characterset of form (not copied)
|
||||
* \return a new structure, or 0 on memory exhaustion
|
||||
*/
|
||||
|
||||
struct form *form_new(char *action, form_method method)
|
||||
struct form *form_new(char *action, form_method method, char *charset)
|
||||
{
|
||||
struct form *form;
|
||||
|
||||
|
|
@ -40,6 +41,7 @@ struct form *form_new(char *action, form_method method)
|
|||
return 0;
|
||||
form->action = action;
|
||||
form->method = method;
|
||||
form->charset = charset;
|
||||
form->controls = 0;
|
||||
form->last_control = 0;
|
||||
form->prev = 0;
|
||||
|
|
@ -465,15 +467,15 @@ char *form_textarea_value(struct form_control *textarea)
|
|||
/**
|
||||
* Encode controls using application/x-www-form-urlencoded.
|
||||
*
|
||||
* \param form form to which successful controls relate
|
||||
* \param control linked list of form_successful_control
|
||||
* \return URL-encoded form, or 0 on memory exhaustion
|
||||
*
|
||||
* \todo encoding conversion
|
||||
*/
|
||||
|
||||
char *form_url_encode(struct form_successful_control *control)
|
||||
char *form_url_encode(struct form *form,
|
||||
struct form_successful_control *control)
|
||||
{
|
||||
char *name, *value;
|
||||
char *name, *value, *n_temp, *v_temp;
|
||||
char *s = malloc(1), *s2;
|
||||
unsigned int len = 0, len1;
|
||||
|
||||
|
|
@ -482,11 +484,26 @@ char *form_url_encode(struct form_successful_control *control)
|
|||
s[0] = 0;
|
||||
|
||||
for (; control; control = control->next) {
|
||||
name = curl_escape(control->name, 0);
|
||||
value = curl_escape(control->value, 0);
|
||||
n_temp = utf8_to_enc(control->name, form->charset, 0);
|
||||
if (!n_temp) {
|
||||
free(s);
|
||||
return 0;
|
||||
}
|
||||
v_temp = utf8_to_enc(control->value, form->charset, 0);
|
||||
if (!v_temp) {
|
||||
free(n_temp);
|
||||
free(s);
|
||||
return 0;
|
||||
}
|
||||
name = curl_escape(n_temp, 0);
|
||||
value = curl_escape(v_temp, 0);
|
||||
len1 = len + strlen(name) + strlen(value) + 2;
|
||||
s2 = realloc(s, len1 + 1);
|
||||
if (!s2) {
|
||||
curl_free(value);
|
||||
curl_free(name);
|
||||
free(v_temp);
|
||||
free(n_temp);
|
||||
free(s);
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -495,6 +512,8 @@ char *form_url_encode(struct form_successful_control *control)
|
|||
len = len1;
|
||||
curl_free(name);
|
||||
curl_free(value);
|
||||
free(v_temp);
|
||||
free(n_temp);
|
||||
}
|
||||
if (len)
|
||||
s[len - 1] = 0;
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ typedef enum {
|
|||
struct form {
|
||||
char *action; /**< URL to submit to. */
|
||||
form_method method; /**< Method and enctype. */
|
||||
char *charset; /**< Charset to submit form in */
|
||||
struct form_control *controls; /**< Linked list of controls. */
|
||||
struct form_control *last_control; /**< Last control in list. */
|
||||
struct form *prev; /**< Previous form in doc. */
|
||||
|
|
@ -101,7 +102,7 @@ struct form_successful_control {
|
|||
struct form_successful_control *next; /**< Next in linked list. */
|
||||
};
|
||||
|
||||
struct form *form_new(char *action, form_method method);
|
||||
struct form *form_new(char *action, form_method method, char *charset);
|
||||
struct form_control *form_new_control(form_control_type type);
|
||||
void form_add_control(struct form *form, struct form_control *control);
|
||||
void form_free_control(struct form_control *control);
|
||||
|
|
@ -110,7 +111,8 @@ bool form_add_option(struct form_control *control, char *value, char *text,
|
|||
bool form_successful_controls(struct form *form,
|
||||
struct form_control *submit_button,
|
||||
struct form_successful_control **successful_controls);
|
||||
char *form_url_encode(struct form_successful_control *control);
|
||||
char *form_url_encode(struct form *form,
|
||||
struct form_successful_control *control);
|
||||
void form_free_successful(struct form_successful_control *control);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@
|
|||
* UCS conversion tables
|
||||
*/
|
||||
|
||||
#include "oslib/osbyte.h"
|
||||
#include "oslib/territory.h"
|
||||
#include "netsurf/riscos/ucstables.h"
|
||||
#include "netsurf/utils/utils.h"
|
||||
|
||||
/* Common values (ASCII) */
|
||||
#define common \
|
||||
|
|
@ -331,3 +333,42 @@ int *ucstable_from_alphabet(int alphabet)
|
|||
|
||||
return ucstable;
|
||||
}
|
||||
|
||||
static const char *localencodings[] = {
|
||||
"ISO-8859-1", /* BFont - 100 - just use Latin1, instead */
|
||||
"ISO-8859-1", /* do we want to use Acorn Latin1, instead? */
|
||||
"ISO-8859-2",
|
||||
"ISO-8859-3",
|
||||
"ISO-8859-4",
|
||||
"ISO-8859-5",
|
||||
"ISO-8859-6",
|
||||
"ISO-8869-7",
|
||||
"ISO-8859-8",
|
||||
"ISO-8859-9",
|
||||
"ISO-IR-182",
|
||||
"UTF-8",
|
||||
"ISO-8859-15",
|
||||
"ISO-8859-10",
|
||||
"ISO-8859-13",
|
||||
"ISO-8859-14",
|
||||
"CP866" /* Cyrillic2 - 120 */
|
||||
};
|
||||
|
||||
/**
|
||||
* Retrieve local encoding name, suitable for passing to iconv
|
||||
*/
|
||||
const char *local_encoding_name(void)
|
||||
{
|
||||
os_error *error;
|
||||
int alphabet;
|
||||
|
||||
error = xosbyte1(osbyte_ALPHABET_NUMBER, 127, 0, &alphabet);
|
||||
if (!error) {
|
||||
if (alphabet < 116)
|
||||
return localencodings[alphabet - 100];
|
||||
else if (alphabet == 120)
|
||||
return localencodings[16];
|
||||
}
|
||||
|
||||
return localencodings[0];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,280 @@
|
|||
/*
|
||||
* This file is part of NetSurf, http://netsurf.sourceforge.net/
|
||||
* Licensed under the GNU General Public License,
|
||||
* http://www.opensource.org/licenses/gpl-license
|
||||
* Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file UTF-8 manipulation functions (implementation)
|
||||
*/
|
||||
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
|
||||
#include <iconv.h>
|
||||
|
||||
#include "netsurf/utils/utf8.h"
|
||||
|
||||
static char *utf8_convert(const char *string, size_t len, const char *from,
|
||||
const char *to);
|
||||
|
||||
/**
|
||||
* Convert a UTF-8 multibyte sequence into a single UCS4 character
|
||||
*
|
||||
* Encoding of UCS values outside the UTF-16 plane has been removed from
|
||||
* RFC3629. This function conforms to RFC2279, however, as it is possible
|
||||
* that the platform specific keyboard input handler will generate a UCS4
|
||||
* value outside the UTF-16 plane.
|
||||
*
|
||||
* \param s The sequence to process
|
||||
* \param l Length of sequence
|
||||
* \return UCS4 character
|
||||
*/
|
||||
size_t utf8_to_ucs4(const char *s, size_t l)
|
||||
{
|
||||
size_t c = 0;
|
||||
|
||||
if (!s)
|
||||
assert(0);
|
||||
else if (l > 0 && *s < 0x80)
|
||||
c = *s;
|
||||
else if (l > 1 && (*s & 0xE0) == 0xC0 && (*(s+1) & 0xC0) == 0x80)
|
||||
c = ((*s & 0x1F) << 6) | (*(s+1) & 0x3F);
|
||||
else if (l > 2 && (*s & 0xF0) == 0xE0 && (*(s+1) & 0xC0) == 0x80 &&
|
||||
(*(s+2) & 0xC0) == 0x80)
|
||||
c = ((*s & 0x0F) << 12) | ((*(s+1) & 0x3F) << 6) |
|
||||
(*(s+2) & 0x3F);
|
||||
else if (l > 3 && (*s & 0xF8) == 0xF0 && (*(s+1) & 0xC0) == 0x80 &&
|
||||
(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80)
|
||||
c = ((*s & 0x0F) << 18) | ((*(s+1) & 0x3F) << 12) |
|
||||
((*(s+2) & 0x3F) << 6) | (*(s+3) & 0x3F);
|
||||
else if (l > 4 && (*s & 0xFC) == 0xF8 && (*(s+1) & 0xC0) == 0x80 &&
|
||||
(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
|
||||
(*(s+4) & 0xC0) == 0x80)
|
||||
c = ((*s & 0x0F) << 24) | ((*(s+1) & 0x3F) << 18) |
|
||||
((*(s+2) & 0x3F) << 12) | ((*(s+3) & 0x3F) << 6) |
|
||||
(*(s+4) & 0x3F);
|
||||
else if (l > 5 && (*s & 0xFE) == 0xFC && (*(s+1) & 0xC0) == 0x80 &&
|
||||
(*(s+2) & 0xC0) == 0x80 && (*(s+3) & 0xC0) == 0x80 &&
|
||||
(*(s+4) & 0xC0) == 0x80 && (*(s+5) & 0xC0) == 0x80)
|
||||
c = ((*s & 0x0F) << 28) | ((*(s+1) & 0x3F) << 24) |
|
||||
((*(s+2) & 0x3F) << 18) | ((*(s+3) & 0x3F) << 12) |
|
||||
((*(s+4) & 0x3F) << 6) | (*(s+5) & 0x3F);
|
||||
else
|
||||
assert(0);
|
||||
|
||||
return c;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a single UCS4 character into a UTF-8 multibyte sequence
|
||||
*
|
||||
* Encoding of UCS values outside the UTF-16 plane has been removed from
|
||||
* RFC3629. This function conforms to RFC2279, however, as it is possible
|
||||
* that the platform specific keyboard input handler will generate a UCS4
|
||||
* value outside the UTF-16 plane.
|
||||
*
|
||||
* \param c The character to process (0 <= c <= 0x7FFFFFFF)
|
||||
* \param s Pointer to 6 byte long output buffer
|
||||
* \return Length of multibyte sequence
|
||||
*/
|
||||
size_t utf8_from_ucs4(size_t c, char *s)
|
||||
{
|
||||
size_t l = 0;
|
||||
|
||||
if (c > 0x7FFFFFFF)
|
||||
assert(0);
|
||||
else if (c < 0x80) {
|
||||
*s = (char)c;
|
||||
l = 1;
|
||||
}
|
||||
else if (c < 0x800) {
|
||||
*s = 0xC0 | ((c >> 6) & 0x1F);
|
||||
*(s+1) = 0x80 | (c & 0x3F);
|
||||
l = 2;
|
||||
}
|
||||
else if (c < 0x10000) {
|
||||
*s = 0xE0 | ((c >> 12) & 0xF);
|
||||
*(s+1) = 0x80 | ((c >> 6) & 0x3F);
|
||||
*(s+2) = 0x80 | (c & 0x3F);
|
||||
l = 3;
|
||||
}
|
||||
else if (c < 0x200000) {
|
||||
*s = 0xF0 | ((c >> 18) & 0x7);
|
||||
*(s+1) = 0x80 | ((c >> 12) & 0x3F);
|
||||
*(s+2) = 0x80 | ((c >> 6) & 0x3F);
|
||||
*(s+3) = 0x80 | (c & 0x3F);
|
||||
l = 4;
|
||||
}
|
||||
else if (c < 0x4000000) {
|
||||
*s = 0xF8 | ((c >> 24) & 0x3);
|
||||
*(s+1) = 0x80 | ((c >> 18) & 0x3F);
|
||||
*(s+2) = 0x80 | ((c >> 12) & 0x3F);
|
||||
*(s+3) = 0x80 | ((c >> 6) & 0x3F);
|
||||
*(s+4) = 0x80 | (c & 0x3F);
|
||||
l = 5;
|
||||
}
|
||||
else if (c <= 0x7FFFFFFF) {
|
||||
*s = 0xFC | ((c >> 30) & 0x1);
|
||||
*(s+1) = 0x80 | ((c >> 24) & 0x3F);
|
||||
*(s+2) = 0x80 | ((c >> 18) & 0x3F);
|
||||
*(s+3) = 0x80 | ((c >> 12) & 0x3F);
|
||||
*(s+4) = 0x80 | ((c >> 6) & 0x3F);
|
||||
*(s+5) = 0x80 | (c & 0x3F);
|
||||
l = 6;
|
||||
}
|
||||
|
||||
return l;
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate the length (in characters) of a NULL-terminated UTF-8 string
|
||||
*
|
||||
* \param s The string
|
||||
* \return Length of string
|
||||
*/
|
||||
size_t utf8_length(const char *s)
|
||||
{
|
||||
const char *__s = s;
|
||||
int l = 0;
|
||||
while (*__s != '\0') {
|
||||
if ((*__s & 0x80) == 0x00)
|
||||
__s += 1;
|
||||
else if ((*__s & 0xE0) == 0xC0)
|
||||
__s += 2;
|
||||
else if ((*__s & 0xF0) == 0xE0)
|
||||
__s += 3;
|
||||
else if ((*__s & 0xF8) == 0xF0)
|
||||
__s += 4;
|
||||
else if ((*__s & 0xFC) == 0xF8)
|
||||
__s += 5;
|
||||
else if ((*__s & 0xFE) == 0xFC)
|
||||
__s += 6;
|
||||
else
|
||||
assert(0);
|
||||
l++;
|
||||
}
|
||||
|
||||
return l;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find previous legal UTF-8 char in string
|
||||
*
|
||||
* \param s The string
|
||||
* \param o Offset in the string to start at
|
||||
* \return Offset of first byte of previous legal character
|
||||
*/
|
||||
size_t utf8_prev(const char *s, size_t o)
|
||||
{
|
||||
while (o != 0 && !(((s[--o] & 0x80) == 0x00) ||
|
||||
((s[o] & 0xC0) == 0xC0)))
|
||||
/* do nothing */;
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find next legal UTF-8 char in string
|
||||
*
|
||||
* \param s The string
|
||||
* \param l Maximum offset in string
|
||||
* \param o Offset in the string to start at
|
||||
* \return Offset of first byte of next legal character
|
||||
*/
|
||||
size_t utf8_next(const char *s, size_t l, size_t o)
|
||||
{
|
||||
while (o != l && !(((s[++o] & 0x80) == 0x00) ||
|
||||
((s[o] & 0xC0) == 0xC0)))
|
||||
/* do nothing */;
|
||||
|
||||
return o;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Convert a UTF8 string into the named encoding
|
||||
*
|
||||
* \param string The NULL-terminated string to convert
|
||||
* \param encname The encoding name (suitable for passing to iconv)
|
||||
* \param len Length of input string to consider (in bytes), or 0
|
||||
* \return Pointer to converted string (on heap) or NULL on error
|
||||
*/
|
||||
char *utf8_to_enc(const char *string, const char *encname, size_t len)
|
||||
{
|
||||
return utf8_convert(string, len, "UTF-8", encname);
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a UTF8 string into the named encoding
|
||||
*
|
||||
* \param string The NULL-terminated string to convert
|
||||
* \param encname The encoding name (suitable for passing to iconv)
|
||||
* \param len Length of input string to consider (in bytes), or 0
|
||||
* \return Pointer to converted string (on heap) or NULL on error
|
||||
*/
|
||||
char *utf8_from_enc(const char *string, const char *encname, size_t len)
|
||||
{
|
||||
return utf8_convert(string, len, encname, "UTF-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a string from one encoding to another
|
||||
*
|
||||
* \param string The NULL-terminated string to convert
|
||||
* \param len Length of input string to consider (in bytes)
|
||||
* \param from The encoding name to convert from
|
||||
* \param to The encoding name to convert to
|
||||
* \return Pointer to converted string (on heap) or NULL on error
|
||||
*/
|
||||
char *utf8_convert(const char *string, size_t len, const char *from,
|
||||
const char *to)
|
||||
{
|
||||
iconv_t cd;
|
||||
char *ret, *temp, *out, *in;
|
||||
size_t slen, rlen;
|
||||
|
||||
if (!string || !from || !to)
|
||||
return NULL;
|
||||
|
||||
in = (char *)string;
|
||||
|
||||
cd = iconv_open(to, from);
|
||||
if (cd == (iconv_t)-1)
|
||||
return NULL;
|
||||
|
||||
slen = len ? len : strlen(string);
|
||||
/* Worst case = ACSII -> UCS4, so allocate an output buffer
|
||||
* 4 times larger than the input buffer, and add 4 bytes at
|
||||
* the end for the NULL terminator
|
||||
*/
|
||||
rlen = slen * 4 + 4;
|
||||
|
||||
temp = out = calloc(rlen, sizeof(char));
|
||||
if (!out) {
|
||||
iconv_close(cd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* perform conversion */
|
||||
if (iconv(cd, &in, &slen, &out, &rlen) == (size_t)-1) {
|
||||
free(temp);
|
||||
iconv_close(cd);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
iconv_close(cd);
|
||||
|
||||
if (rlen > 64 /* allow 64bytes wasted space */) {
|
||||
/* and allocate a more sensibly sized output buffer */
|
||||
ret = calloc(out - temp + 4, sizeof(char));
|
||||
memcpy(ret, temp, out - temp);
|
||||
free(temp);
|
||||
}
|
||||
else
|
||||
ret = temp;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
/*
|
||||
* This file is part of NetSurf, http://netsurf.sourceforge.net/
|
||||
* Licensed under the GNU General Public License,
|
||||
* http://www.opensource.org/licenses/gpl-license
|
||||
* Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
|
||||
*/
|
||||
|
||||
/**
|
||||
* \file UTF-8 manipulation functions (interface)
|
||||
*/
|
||||
|
||||
#ifndef _NETSURF_UTILS_UTF8_H_
|
||||
#define _NETSURF_UTILS_UTF8_H_
|
||||
|
||||
size_t utf8_to_ucs4(const char *s, size_t l);
|
||||
size_t utf8_from_ucs4(size_t c, char *s);
|
||||
|
||||
size_t utf8_length(const char *s);
|
||||
|
||||
size_t utf8_prev(const char *s, size_t o);
|
||||
size_t utf8_next(const char *s, size_t l, size_t o);
|
||||
|
||||
char *utf8_to_enc(const char *string, const char *encname, size_t len);
|
||||
char *utf8_from_enc(const char *string, const char *encname, size_t len);
|
||||
|
||||
#endif
|
||||
104
utils/utils.c
104
utils/utils.c
|
|
@ -22,6 +22,7 @@
|
|||
#define NDEBUG
|
||||
#include "netsurf/utils/log.h"
|
||||
#include "netsurf/utils/messages.h"
|
||||
#include "netsurf/utils/utf8.h"
|
||||
#include "netsurf/utils/utils.h"
|
||||
|
||||
|
||||
|
|
@ -110,56 +111,10 @@ char *cnv_space2nbsp(const char *s)
|
|||
* \param s string in local machine encoding. NUL or length terminated (which comes first).
|
||||
* \param length maximum number of bytes to consider at s.
|
||||
* \return malloc()'ed NUL termined string in UTF-8 encoding.
|
||||
*
|
||||
* Based on RISCOS-LATIN1 code from libiconv.
|
||||
* \todo: we should use libiconv to support more local encodings instead
|
||||
* of only RISCOS-LATIN1.
|
||||
*/
|
||||
char *cnv_local_enc_str(const char *s, size_t length)
|
||||
{
|
||||
size_t l_out, l_in;
|
||||
const char *s_in;
|
||||
char *d, *d_out;
|
||||
static const unsigned int riscos1_2uni[32] = {
|
||||
/* 0x80 */
|
||||
0x221a, 0x0174, 0x0175, 0x0083, 0x2573, 0x0176, 0x0177, 0x0087,
|
||||
0x21e6, 0x21e8, 0x21e9, 0x21e7, 0x2026, 0x2122, 0x2030, 0x2022,
|
||||
/* 0x90 */
|
||||
0x2018, 0x2019, 0x2039, 0x203a, 0x201c, 0x201d, 0x201e, 0x2013,
|
||||
0x2014, 0x2212, 0x0152, 0x0153, 0x2020, 0x2021, 0xfb01, 0xfb02,
|
||||
};
|
||||
|
||||
/* We're counting on the fact that all riscos1_2uni[] values are
|
||||
* between 0x80 (incl) and 0x1000 (excl).
|
||||
*/
|
||||
for (s_in = s, l_in = length, l_out = 1;
|
||||
*s_in != '\0' && l_in != 0;
|
||||
++s_in, --l_in)
|
||||
l_out += (*s_in >= 0x80 && *s_in < 0xA0) ? ((riscos1_2uni[*s_in - 0x80] < 0x800) ? 2 : 3) : 1;
|
||||
if ((d_out = (char *)malloc(l_out)) == NULL)
|
||||
return NULL;
|
||||
for (s_in = s, l_in = length, d = d_out;
|
||||
*s_in != '\0' && l_in != 0;
|
||||
++s_in, --l_in) {
|
||||
unsigned int uc = (*s_in >= 0x80 && *s_in < 0xA0) ? riscos1_2uni[*s_in - 0x80] : *s_in;
|
||||
const int cnt = (uc < 0x80) ? 1 : (uc < 0x800) ? 2 : 3;
|
||||
switch (cnt) {
|
||||
case 3:
|
||||
d[2] = 0x80 | (uc & 0x3F);
|
||||
uc = (uc >> 6) | 0x800;
|
||||
/* fall through */
|
||||
case 2:
|
||||
d[1] = 0x80 | (uc & 0x3F);
|
||||
uc = (uc >> 6) | 0xC0;
|
||||
/* fall through */
|
||||
case 1:
|
||||
d[0] = uc;
|
||||
}
|
||||
d += cnt;
|
||||
}
|
||||
*d = '\0';
|
||||
|
||||
return d_out;
|
||||
return utf8_from_enc(s, local_encoding_name(), length);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -169,7 +124,7 @@ char *cnv_local_enc_str(const char *s, size_t length)
|
|||
*/
|
||||
char *cnv_str_local_enc(const char *s)
|
||||
{
|
||||
return cnv_strn_local_enc(s, strlen(s), NULL);
|
||||
return cnv_strn_local_enc(s, 0);
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -177,61 +132,12 @@ return cnv_strn_local_enc(s, strlen(s), NULL);
|
|||
* Converts UTF-8 string <s> of <length> bytes to the machine local encoding.
|
||||
* Caller needs to free return value.
|
||||
*
|
||||
* When back_map is non-NULL, a ptr to a ptrdiff_t array is filled in which
|
||||
* needs to be free'd by the caller. The array contains per character
|
||||
* in the return string, a ptrdiff in the <s> UTF-8 encoded string.
|
||||
*
|
||||
* \todo: we should use libiconv to support more local encodings instead
|
||||
* of only ISOLATIN1.
|
||||
*/
|
||||
char *cnv_strn_local_enc(const char *s, int length, const ptrdiff_t **back_mapPP)
|
||||
char *cnv_strn_local_enc(const char *s, int length)
|
||||
{
|
||||
/* Buffer at d & back_mapP can be overdimentioned but is certainly
|
||||
* big enough to carry the end result.
|
||||
*/
|
||||
char *d, *d0;
|
||||
const char * const s0 = s;
|
||||
ptrdiff_t *back_mapP = NULL;
|
||||
|
||||
if (back_mapPP != NULL) {
|
||||
back_mapP = calloc(length + 1, sizeof(ptrdiff_t));
|
||||
if (!back_mapP)
|
||||
return NULL;
|
||||
*back_mapPP = back_mapP;
|
||||
}
|
||||
|
||||
d = calloc(length + 1, sizeof(char));
|
||||
if (!d)
|
||||
return NULL;
|
||||
|
||||
d0 = d;
|
||||
|
||||
while (length != 0) {
|
||||
int u, chars;
|
||||
|
||||
chars = length;
|
||||
u = xmlGetUTF8Char(s, &chars);
|
||||
if (chars <= 0) {
|
||||
s += 1;
|
||||
length -= 1;
|
||||
continue;
|
||||
}
|
||||
if (back_mapP != NULL)
|
||||
*back_mapP++ = s - s0;
|
||||
s += chars;
|
||||
length -= chars;
|
||||
if (u == 0x09 || u == 0x0a || u == 0x0d ||
|
||||
(0x20 <= u && u <= 0x7f) ||
|
||||
(0xa0 <= u && u <= 0xff))
|
||||
*d++ = u;
|
||||
else
|
||||
*d++ = '?';
|
||||
}
|
||||
if (back_mapP != NULL)
|
||||
*back_mapP = s - s0;
|
||||
*d = 0;
|
||||
|
||||
return d0;
|
||||
return utf8_to_enc(s, local_encoding_name(), length);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -20,20 +20,22 @@
|
|||
#define NOF_ELEMENTS(array) (sizeof(array)/sizeof(*(array)))
|
||||
#endif
|
||||
|
||||
void die(const char * const error);
|
||||
char * strip(char * const s);
|
||||
int whitespace(const char * str);
|
||||
char * squash_whitespace(const char * s);
|
||||
char *cnv_space2nbsp(const char *s);
|
||||
char *cnv_local_enc_str(const char *s, size_t length);
|
||||
char *cnv_str_local_enc(const char *s);
|
||||
char *cnv_strn_local_enc(const char *s, int length,
|
||||
const ptrdiff_t **back_mapPP);
|
||||
char *cnv_strn_local_enc(const char *s, int length);
|
||||
bool is_dir(const char *path);
|
||||
void regcomp_wrapper(regex_t *preg, const char *regex, int cflags);
|
||||
void clean_cookiejar(void);
|
||||
void unicode_transliterate(unsigned int c, char **r);
|
||||
char *human_friendly_bytesize(unsigned long bytesize);
|
||||
|
||||
/* Platform specific functions */
|
||||
void die(const char * const error);
|
||||
void warn_user(const char *warning, const char *detail);
|
||||
const char *local_encoding_name(void);
|
||||
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Reference in New Issue