netsurf/utils/utf8.c

567 lines
14 KiB
C

/*
* Copyright 2005 John M Bell <jmb202@ecs.soton.ac.uk>
*
* This file is part of NetSurf, http://www.netsurf-browser.org/
*
* NetSurf is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; version 2 of the License.
*
* NetSurf is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/** \file
* UTF-8 manipulation functions (implementation).
*/
#include <assert.h>
#include <errno.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <iconv.h>
#include <parserutils/charset/utf8.h>
#include "utils/config.h"
#include "utils/log.h"
#include "utils/utf8.h"
/**
* Convert a UTF-8 multibyte sequence into a single UCS4 character
*
* Encoding of UCS values outside the UTF-16 plane has been removed from
* RFC3629. This function conforms to RFC2279, however.
*
* \param s_in The sequence to process
* \param l Length of sequence
* \return UCS4 character
*/
uint32_t utf8_to_ucs4(const char *s_in, size_t l)
{
uint32_t ucs4;
size_t len;
parserutils_error perror;
perror = parserutils_charset_utf8_to_ucs4((const uint8_t *) s_in, l,
&ucs4, &len);
if (perror != PARSERUTILS_OK)
ucs4 = 0xfffd;
return ucs4;
}
/**
* Convert a single UCS4 character into a UTF-8 multibyte sequence
*
* Encoding of UCS values outside the UTF-16 plane has been removed from
* RFC3629. This function conforms to RFC2279, however.
*
* \param c The character to process (0 <= c <= 0x7FFFFFFF)
* \param s Pointer to 6 byte long output buffer
* \return Length of multibyte sequence
*/
size_t utf8_from_ucs4(uint32_t c, char *s)
{
uint8_t *in = (uint8_t *) s;
size_t len = 6;
parserutils_error perror;
perror = parserutils_charset_utf8_from_ucs4(c, &in, &len);
if (perror != PARSERUTILS_OK) {
s[0] = 0xef;
s[1] = 0xbf;
s[2] = 0xbd;
return 3;
}
return 6 - len;
}
/**
* Calculate the length (in characters) of a NULL-terminated UTF-8 string
*
* \param s The string
* \return Length of string
*/
size_t utf8_length(const char *s)
{
return utf8_bounded_length(s, strlen(s));
}
/**
* Calculated the length (in characters) of a bounded UTF-8 string
*
* \param s The string
* \param l Maximum length of input (in bytes)
* \return Length of string, in characters
*/
size_t utf8_bounded_length(const char *s, size_t l)
{
size_t len;
parserutils_error perror;
perror = parserutils_charset_utf8_length((const uint8_t *) s, l, &len);
if (perror != PARSERUTILS_OK)
return 0;
return len;
}
/**
* Calculate the length (in bytes) of a bounded UTF-8 string
*
* \param s The string
* \param l Maximum length of input (in bytes)
* \param c Maximum number of characters to measure
* \return Length of string, in bytes
*/
size_t utf8_bounded_byte_length(const char *s, size_t l, size_t c)
{
size_t len = 0;
while (len < l && c-- > 0)
len = utf8_next(s, l, len);
return len;
}
/**
* Calculate the length (in bytes) of a UTF-8 character
*
* \param s Pointer to start of character
* \return Length of character, in bytes
*/
size_t utf8_char_byte_length(const char *s)
{
size_t len;
parserutils_error perror;
perror = parserutils_charset_utf8_char_byte_length((const uint8_t *) s,
&len);
assert(perror == PARSERUTILS_OK);
return len;
}
/**
* Find previous legal UTF-8 char in string
*
* \param s The string
* \param o Offset in the string to start at
* \return Offset of first byte of previous legal character
*/
size_t utf8_prev(const char *s, size_t o)
{
uint32_t prev;
parserutils_error perror;
perror = parserutils_charset_utf8_prev((const uint8_t *) s, o, &prev);
assert(perror == PARSERUTILS_OK);
return prev;
}
/**
* Find next legal UTF-8 char in string
*
* \param s The string
* \param l Maximum offset in string
* \param o Offset in the string to start at
* \return Offset of first byte of next legal character
*/
size_t utf8_next(const char *s, size_t l, size_t o)
{
uint32_t next;
parserutils_error perror;
perror = parserutils_charset_utf8_next((const uint8_t *) s, l, o,
&next);
assert(perror == PARSERUTILS_OK);
return next;
}
/* Cache of previous iconv conversion descriptor used by utf8_convert */
static struct {
char from[32]; /**< Encoding name to convert from */
char to[32]; /**< Encoding name to convert to */
iconv_t cd; /**< Iconv conversion descriptor */
} last_cd;
static inline void utf8_clear_cd_cache(void)
{
last_cd.from[0] = '\0';
last_cd.to[0] = '\0';
last_cd.cd = 0;
}
/**
* Finalise the UTF-8 library
*/
void utf8_finalise(void)
{
if (last_cd.cd != 0)
iconv_close(last_cd.cd);
/* paranoia follows */
utf8_clear_cd_cache();
}
/**
* Convert a string from one encoding to another
*
* \param string The NULL-terminated string to convert
* \param len Length of input string to consider (in bytes), or 0
* \param from The encoding name to convert from
* \param to The encoding name to convert to
* \param result Pointer to location in which to store result.
* \param result_len Pointer to location in which to store result length.
* \return Appropriate utf8_convert_ret value
*/
static utf8_convert_ret utf8_convert(const char *string, size_t len,
const char *from, const char *to,
char **result, size_t *result_len)
{
iconv_t cd;
char *temp, *out, *in;
size_t slen, rlen;
assert(string && from && to && result);
if (string[0] == '\0') {
/* On AmigaOS, iconv() returns an error if we pass an
* empty string. This prevents iconv() being called as
* there is no conversion necessary anyway. */
*result = strdup("");
if (!(*result)) {
*result = NULL;
return UTF8_CONVERT_NOMEM;
}
return UTF8_CONVERT_OK;
}
if (strcasecmp(from, to) == 0) {
/* conversion from an encoding to itself == strdup */
slen = len ? len : strlen(string);
*(result) = strndup(string, slen);
if (!(*result)) {
*(result) = NULL;
return UTF8_CONVERT_NOMEM;
}
return UTF8_CONVERT_OK;
}
in = (char *)string;
/* we cache the last used conversion descriptor,
* so check if we're trying to use it here */
if (strncasecmp(last_cd.from, from, sizeof(last_cd.from)) == 0 &&
strncasecmp(last_cd.to, to, sizeof(last_cd.to)) == 0) {
cd = last_cd.cd;
}
else {
/* no match, so create a new cd */
cd = iconv_open(to, from);
if (cd == (iconv_t)-1) {
if (errno == EINVAL)
return UTF8_CONVERT_BADENC;
/* default to no memory */
return UTF8_CONVERT_NOMEM;
}
/* close the last cd - we don't care if this fails */
if (last_cd.cd)
iconv_close(last_cd.cd);
/* and copy the to/from/cd data into last_cd */
strncpy(last_cd.from, from, sizeof(last_cd.from));
strncpy(last_cd.to, to, sizeof(last_cd.to));
last_cd.cd = cd;
}
slen = len ? len : strlen(string);
/* Worst case = ASCII -> UCS4, so allocate an output buffer
* 4 times larger than the input buffer, and add 4 bytes at
* the end for the NULL terminator
*/
rlen = slen * 4 + 4;
temp = out = malloc(rlen);
if (!out)
return UTF8_CONVERT_NOMEM;
/* perform conversion */
if (iconv(cd, (void *) &in, &slen, &out, &rlen) == (size_t)-1) {
free(temp);
/* clear the cached conversion descriptor as it's invalid */
if (last_cd.cd)
iconv_close(last_cd.cd);
utf8_clear_cd_cache();
/** \todo handle the various cases properly
* There are 3 possible error cases:
* a) Insufficiently large output buffer
* b) Invalid input byte sequence
* c) Incomplete input sequence */
return UTF8_CONVERT_NOMEM;
}
*(result) = realloc(temp, out - temp + 4);
if (!(*result)) {
free(temp);
*(result) = NULL; /* for sanity's sake */
return UTF8_CONVERT_NOMEM;
}
/* NULL terminate - needs 4 characters as we may have
* converted to UTF-32 */
memset((*result) + (out - temp), 0, 4);
if (result_len != NULL) {
*result_len = (out - temp);
}
return UTF8_CONVERT_OK;
}
/**
* Convert a UTF8 string into the named encoding
*
* \param string The NULL-terminated string to convert
* \param encname The encoding name (suitable for passing to iconv)
* \param len Length of input string to consider (in bytes), or 0
* \param result Pointer to location to store result (allocated on heap)
* \return Appropriate utf8_convert_ret value
*/
utf8_convert_ret utf8_to_enc(const char *string, const char *encname,
size_t len, char **result)
{
return utf8_convert(string, len, "UTF-8", encname, result, NULL);
}
/**
* Convert a string in the named encoding into a UTF-8 string
*
* \param string The NULL-terminated string to convert
* \param encname The encoding name (suitable for passing to iconv)
* \param len Length of input string to consider (in bytes), or 0
* \param result Pointer to location to store result (allocated on heap)
* \return Appropriate utf8_convert_ret value
*/
utf8_convert_ret utf8_from_enc(const char *string, const char *encname,
size_t len, char **result, size_t *result_len)
{
return utf8_convert(string, len, encname, "UTF-8", result, result_len);
}
static utf8_convert_ret utf8_convert_html_chunk(iconv_t cd,
const char *chunk, size_t inlen,
char **out, size_t *outlen)
{
size_t ret, esclen;
uint32_t ucs4;
char *pescape, escape[11];
while (inlen > 0) {
ret = iconv(cd, (void *) &chunk, &inlen, (void *) out, outlen);
if (ret != (size_t) -1)
break;
if (errno != EILSEQ)
return UTF8_CONVERT_NOMEM;
ucs4 = utf8_to_ucs4(chunk, inlen);
esclen = snprintf(escape, sizeof(escape), "&#x%06x;", ucs4);
pescape = escape;
ret = iconv(cd, (void *) &pescape, &esclen,
(void *) out, outlen);
if (ret == (size_t) -1)
return UTF8_CONVERT_NOMEM;
esclen = utf8_next(chunk, inlen, 0);
chunk += esclen;
inlen -= esclen;
}
return UTF8_CONVERT_OK;
}
/**
* Convert a UTF-8 encoded string into a string of the given encoding,
* applying HTML escape sequences where necessary.
*
* \param string String to convert (NUL-terminated)
* \param encname Name of encoding to convert to
* \param len Length, in bytes, of the input string, or 0
* \param result Pointer to location to receive result
* \return Appropriate utf8_convert_ret value
*/
utf8_convert_ret utf8_to_html(const char *string, const char *encname,
size_t len, char **result)
{
iconv_t cd;
const char *in;
char *out, *origout;
size_t off, prev_off, inlen, outlen, origoutlen, esclen;
utf8_convert_ret ret;
char *pescape, escape[11];
if (len == 0)
len = strlen(string);
/* we cache the last used conversion descriptor,
* so check if we're trying to use it here */
if (strncasecmp(last_cd.from, "UTF-8", sizeof(last_cd.from)) == 0 &&
strncasecmp(last_cd.to, encname,
sizeof(last_cd.to)) == 0 &&
last_cd.cd != 0) {
cd = last_cd.cd;
}
else {
/* no match, so create a new cd */
cd = iconv_open(encname, "UTF-8");
if (cd == (iconv_t) -1) {
if (errno == EINVAL)
return UTF8_CONVERT_BADENC;
/* default to no memory */
return UTF8_CONVERT_NOMEM;
}
/* close the last cd - we don't care if this fails */
if (last_cd.cd)
iconv_close(last_cd.cd);
/* and copy the to/from/cd data into last_cd */
strncpy(last_cd.from, "UTF-8", sizeof(last_cd.from));
strncpy(last_cd.to, encname, sizeof(last_cd.to));
last_cd.cd = cd;
}
/* Worst case is ASCII -> UCS4, with all characters escaped:
* "&#xYYYYYY;", thus each input character may become a string
* of 10 UCS4 characters, each 4 bytes in length, plus four for
* terminating the string */
origoutlen = outlen = len * 10 * 4 + 4;
origout = out = malloc(outlen);
if (out == NULL) {
iconv_close(cd);
utf8_clear_cd_cache();
return UTF8_CONVERT_NOMEM;
}
/* Process input in chunks between characters we must escape */
prev_off = off = 0;
while (off < len) {
/* Must escape '&', '<', and '>' */
if (string[off] == '&' || string[off] == '<' ||
string[off] == '>') {
if (off - prev_off > 0) {
/* Emit chunk */
in = string + prev_off;
inlen = off - prev_off;
ret = utf8_convert_html_chunk(cd, in, inlen,
&out, &outlen);
if (ret != UTF8_CONVERT_OK) {
free(origout);
iconv_close(cd);
utf8_clear_cd_cache();
return ret;
}
}
/* Emit mandatory escape */
esclen = snprintf(escape, sizeof(escape),
"&#x%06x;", string[off]);
pescape = escape;
ret = utf8_convert_html_chunk(cd, pescape, esclen,
&out, &outlen);
if (ret != UTF8_CONVERT_OK) {
free(origout);
iconv_close(cd);
utf8_clear_cd_cache();
return ret;
}
prev_off = off = utf8_next(string, len, off);
} else {
off = utf8_next(string, len, off);
}
}
/* Process final chunk */
if (prev_off < len) {
in = string + prev_off;
inlen = len - prev_off;
ret = utf8_convert_html_chunk(cd, in, inlen, &out, &outlen);
if (ret != UTF8_CONVERT_OK) {
free(origout);
iconv_close(cd);
utf8_clear_cd_cache();
return ret;
}
}
/* Terminate string */
memset(out, 0, 4);
outlen -= 4;
/* Shrink-wrap */
*result = realloc(origout, origoutlen - outlen);
if (*result == NULL) {
free(origout);
return UTF8_CONVERT_NOMEM;
}
return UTF8_CONVERT_OK;
}
/**
* Save the given utf8 text to a file, converting to local encoding.
*
* \param utf8_text text to save to file
* \param path pathname to save to
* \return true iff the save succeeded
*/
bool utf8_save_text(const char *utf8_text, const char *path)
{
utf8_convert_ret ret;
char *conv;
FILE *out;
ret = utf8_to_local_encoding(utf8_text, strlen(utf8_text), &conv);
if (ret != UTF8_CONVERT_OK) {
LOG(("failed to convert to local encoding, return %d", ret));
return false;
}
out = fopen(path, "w");
if (out) {
int res = fputs(conv, out);
if (res < 0) {
LOG(("Warning: writing data failed"));
}
res = fputs("\n", out);
fclose(out);
free(conv);
return (res != EOF);
}
free(conv);
return false;
}