Add text encoding detection to the Locale Kit.
As usual we ask ICU to do the actual work. The TextEncoding constructor is fed with a sample of the text to identify (ICU docs recommend a few hundred bytes). The text is analyzed in various ways (bytes patterns such as UTF-8 escaping schemes, common letter sequences from known languages, byte order marks) and an encoding is determined. Replace code in StyledEdit by this new implementation. Note that ICU seems to always return some valid encoding, even with fed with obviously non-text data. This makes StyledEdit open the files no matter what, where it would error out before. Fixes #9395.
This commit is contained in:
Adrien Destugues
parent
aec3e63ed1
commit
fbb725bbdc
28
headers/os/locale/TextEncoding.h
Normal file
28
headers/os/locale/TextEncoding.h
Normal file
@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Copyright 2016, Haiku, inc.
|
||||
* Distributed under terms of the MIT license.
|
||||
*/
|
||||
|
||||
|
||||
#ifndef TEXTENCODING_H
|
||||
#define TEXTENCODING_H
|
||||
|
||||
|
||||
#include <String.h>
|
||||
|
||||
#include <stddef.h>
|
||||
|
||||
|
||||
class TextEncoding
|
||||
{
|
||||
public:
|
||||
TextEncoding(const char* data, size_t length);
|
||||
|
||||
BString GetName();
|
||||
|
||||
private:
|
||||
BString fName;
|
||||
};
|
||||
|
||||
|
||||
#endif /* !TEXTENCODING_H */
|
||||
@ -16,6 +16,7 @@
|
||||
#include <CharacterSetRoster.h>
|
||||
#include <MimeType.h>
|
||||
#include <String.h>
|
||||
#include <TextEncoding.h>
|
||||
#include <UTF8.h>
|
||||
|
||||
#include <algorithm>
|
||||
@ -32,7 +33,7 @@ using namespace std;
|
||||
#define B_TRANSLATION_CONTEXT "STXTTranslator"
|
||||
|
||||
#define READ_BUFFER_SIZE 32768
|
||||
#define DATA_BUFFER_SIZE 256
|
||||
#define DATA_BUFFER_SIZE 2048
|
||||
|
||||
// The input formats that this translator supports.
|
||||
static const translation_format sInputFormats[] = {
|
||||
@ -116,666 +117,6 @@ make_nth_translator(int32 n, image_id you, uint32 flags, ...)
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark - ascmagic.c from the BSD file tool
|
||||
/*
|
||||
* The following code has been taken from version 4.17 of the BSD file tool,
|
||||
* file ascmagic.c, modified for our purpose.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Copyright (c) Ian F. Darwin 1986-1995.
|
||||
* Software written by Ian F. Darwin and others;
|
||||
* maintained 1995-present by Christos Zoulas and others.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice immediately at the beginning of the file, without modification,
|
||||
* this list of conditions, and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
|
||||
* ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
/*
|
||||
* ASCII magic -- file types that we know based on keywords
|
||||
* that can appear anywhere in the file.
|
||||
* bool found = false;
|
||||
if (subtypeMimeSpecific != NULL) {
|
||||
mimeType->SetTo(subtypeMimeSpecific);
|
||||
if (mimeType->IsInstalled())
|
||||
found = true;
|
||||
}
|
||||
if (!found && subtypeMimeGeneric != NULL) {
|
||||
mimeType->SetTo(subtypeMimeGeneric);
|
||||
if (mimeType->IsInstalled())
|
||||
found = true;
|
||||
}
|
||||
if (!found)
|
||||
mimeType->SetTo("text/plain");
|
||||
|
||||
* Extensively modified by Eric Fischer <enf@pobox.com> in July, 2000,
|
||||
* to handle character codes other than ASCII on a unified basis.
|
||||
*
|
||||
* Joerg Wunsch <joerg@freebsd.org> wrote the original support for 8-bit
|
||||
* international characters, now subsumed into this file.
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <memory.h>
|
||||
#include <ctype.h>
|
||||
#include <stdlib.h>
|
||||
#include <unistd.h>
|
||||
#include "names.h"
|
||||
|
||||
typedef unsigned long my_unichar;
|
||||
|
||||
#define MAXLINELEN 300 /* longest sane line length */
|
||||
#define ISSPC(x) ((x) == ' ' || (x) == '\t' || (x) == '\r' || (x) == '\n' \
|
||||
|| (x) == 0x85 || (x) == '\f')
|
||||
|
||||
static int looks_ascii(const unsigned char *, size_t, my_unichar *, size_t *);
|
||||
static int looks_utf8(const unsigned char *, size_t, my_unichar *, size_t *);
|
||||
static int looks_unicode(const unsigned char *, size_t, my_unichar *, size_t *);
|
||||
static int looks_latin1(const unsigned char *, size_t, my_unichar *, size_t *);
|
||||
static int looks_extended(const unsigned char *, size_t, my_unichar *, size_t *);
|
||||
static void from_ebcdic(const unsigned char *, size_t, unsigned char *);
|
||||
static int ascmatch(const unsigned char *, const my_unichar *, size_t);
|
||||
|
||||
|
||||
static int
|
||||
file_ascmagic(const unsigned char *buf, size_t nbytes, BMimeType* mimeType,
|
||||
const char*& encoding)
|
||||
{
|
||||
size_t i;
|
||||
unsigned char *nbuf = NULL;
|
||||
my_unichar *ubuf = NULL;
|
||||
size_t ulen;
|
||||
struct names *p;
|
||||
int rv = -1;
|
||||
|
||||
const char *code = NULL;
|
||||
encoding = NULL;
|
||||
const char *type = NULL;
|
||||
const char *subtype = NULL;
|
||||
const char *subtypeMimeGeneric = NULL;
|
||||
const char *subtypeMimeSpecific = NULL;
|
||||
|
||||
int has_escapes = 0;
|
||||
int has_backspace = 0;
|
||||
int seen_cr = 0;
|
||||
|
||||
int n_crlf = 0;
|
||||
int n_lf = 0;
|
||||
int n_cr = 0;
|
||||
int n_nel = 0;
|
||||
|
||||
int last_line_end = -1;
|
||||
int has_long_lines = 0;
|
||||
|
||||
if ((nbuf = (unsigned char*)malloc((nbytes + 1) * sizeof(nbuf[0]))) == NULL)
|
||||
goto done;
|
||||
if ((ubuf = (my_unichar*)malloc((nbytes + 1) * sizeof(ubuf[0]))) == NULL)
|
||||
goto done;
|
||||
|
||||
/*
|
||||
* Then try to determine whether it's any character code we can
|
||||
* identify. Each of these tests, if it succeeds, will leave
|
||||
* the text converted into one-my_unichar-per-character Unicode in
|
||||
* ubuf, and the number of characters converted in ulen.
|
||||
*/
|
||||
if (nbytes == 0) {
|
||||
code = "UTF-8 Unicode";
|
||||
encoding = NULL; // "UTF-8";
|
||||
type = "text";
|
||||
rv = 1;
|
||||
} else if (looks_ascii(buf, nbytes, ubuf, &ulen)) {
|
||||
code = "ASCII";
|
||||
encoding = NULL; //"us-ascii";
|
||||
type = "text";
|
||||
if (nbytes == 1) {
|
||||
// no further tests
|
||||
rv = 1;
|
||||
}
|
||||
} else if (looks_utf8(buf, nbytes, ubuf, &ulen)) {
|
||||
code = "UTF-8 Unicode";
|
||||
encoding = NULL; // "UTF-8";
|
||||
type = "text";
|
||||
} else if ((i = looks_unicode(buf, nbytes, ubuf, &ulen)) != 0) {
|
||||
if (i == 1) {
|
||||
code = "Little-endian UTF-16 Unicode";
|
||||
encoding = "UTF-16";
|
||||
} else {
|
||||
code = "Big-endian UTF-16 Unicode";
|
||||
encoding = "UTF-16";
|
||||
}
|
||||
|
||||
type = "character data";
|
||||
} else if (looks_latin1(buf, nbytes, ubuf, &ulen)) {
|
||||
code = "ISO-8859";
|
||||
type = "text";
|
||||
encoding = "iso-8859-1";
|
||||
} else if (looks_extended(buf, nbytes, ubuf, &ulen)) {
|
||||
code = "Non-ISO extended-ASCII";
|
||||
type = "text";
|
||||
encoding = "unknown";
|
||||
} else {
|
||||
from_ebcdic(buf, nbytes, nbuf);
|
||||
|
||||
if (looks_ascii(nbuf, nbytes, ubuf, &ulen)) {
|
||||
code = "EBCDIC";
|
||||
type = "character data";
|
||||
encoding = "ebcdic";
|
||||
} else if (looks_latin1(nbuf, nbytes, ubuf, &ulen)) {
|
||||
code = "International EBCDIC";
|
||||
type = "character data";
|
||||
encoding = "ebcdic";
|
||||
} else {
|
||||
rv = 0;
|
||||
goto done; /* doesn't look like text at all */
|
||||
}
|
||||
}
|
||||
|
||||
if (nbytes <= 1) {
|
||||
if (rv == -1)
|
||||
rv = 0;
|
||||
goto done;
|
||||
}
|
||||
|
||||
/*
|
||||
* for troff, look for . + letter + letter or .\";
|
||||
* this must be done to disambiguate tar archives' ./file
|
||||
* and other trash from real troff input.
|
||||
*
|
||||
* I believe Plan 9 troff allows non-ASCII characters in the names
|
||||
* of macros, so this test might possibly fail on such a file.
|
||||
*/
|
||||
if (*ubuf == '.') {
|
||||
my_unichar *tp = ubuf + 1;
|
||||
|
||||
while (ISSPC(*tp))
|
||||
++tp; /* skip leading whitespace */
|
||||
if ((tp[0] == '\\' && tp[1] == '\"') ||
|
||||
(isascii((unsigned char)tp[0]) &&
|
||||
isalnum((unsigned char)tp[0]) &&
|
||||
isascii((unsigned char)tp[1]) &&
|
||||
isalnum((unsigned char)tp[1]) &&
|
||||
ISSPC(tp[2]))) {
|
||||
subtypeMimeGeneric = "text/x-source-code";
|
||||
subtypeMimeSpecific = "text/troff";
|
||||
subtype = "troff or preprocessor input";
|
||||
goto subtype_identified;
|
||||
}
|
||||
}
|
||||
|
||||
if ((*buf == 'c' || *buf == 'C') && ISSPC(buf[1])) {
|
||||
subtypeMimeGeneric = "text/x-source-code";
|
||||
subtypeMimeSpecific = "text/fortran";
|
||||
subtype = "fortran program";
|
||||
goto subtype_identified;
|
||||
}
|
||||
|
||||
/* look for tokens from names.h - this is expensive! */
|
||||
|
||||
i = 0;
|
||||
while (i < ulen) {
|
||||
size_t end;
|
||||
|
||||
/*
|
||||
* skip past any leading space
|
||||
*/
|
||||
while (i < ulen && ISSPC(ubuf[i]))
|
||||
i++;
|
||||
if (i >= ulen)
|
||||
break;
|
||||
|
||||
/*
|
||||
* find the next whitespace
|
||||
*/
|
||||
for (end = i + 1; end < nbytes; end++)
|
||||
if (ISSPC(ubuf[end]))
|
||||
break;
|
||||
|
||||
/*
|
||||
* compare the word thus isolated against the token list
|
||||
*/
|
||||
for (p = names; p < names + NNAMES; p++) {
|
||||
if (ascmatch((const unsigned char *)p->name, ubuf + i,
|
||||
end - i)) {
|
||||
subtype = types[p->type].human;
|
||||
subtypeMimeGeneric = types[p->type].generic_mime;
|
||||
subtypeMimeSpecific = types[p->type].specific_mime;
|
||||
goto subtype_identified;
|
||||
}
|
||||
}
|
||||
|
||||
i = end;
|
||||
}
|
||||
|
||||
subtype_identified:
|
||||
|
||||
/*
|
||||
* Now try to discover other details about the file.
|
||||
*/
|
||||
for (i = 0; i < ulen; i++) {
|
||||
if (ubuf[i] == '\n') {
|
||||
if (seen_cr)
|
||||
n_crlf++;
|
||||
else
|
||||
n_lf++;
|
||||
last_line_end = i;
|
||||
} else if (seen_cr)
|
||||
n_cr++;
|
||||
|
||||
seen_cr = (ubuf[i] == '\r');
|
||||
if (seen_cr)
|
||||
last_line_end = i;
|
||||
|
||||
if (ubuf[i] == 0x85) { /* X3.64/ECMA-43 "next line" character */
|
||||
n_nel++;
|
||||
last_line_end = i;
|
||||
}
|
||||
|
||||
/* If this line is _longer_ than MAXLINELEN, remember it. */
|
||||
if ((int)i > last_line_end + MAXLINELEN)
|
||||
has_long_lines = 1;
|
||||
|
||||
if (ubuf[i] == '\033')
|
||||
has_escapes = 1;
|
||||
if (ubuf[i] == '\b')
|
||||
has_backspace = 1;
|
||||
}
|
||||
|
||||
rv = 1;
|
||||
done:
|
||||
if (nbuf)
|
||||
free(nbuf);
|
||||
if (ubuf)
|
||||
free(ubuf);
|
||||
|
||||
if (rv) {
|
||||
// If we have identified the subtype, return it, otherwise just
|
||||
// text/plain.
|
||||
|
||||
bool found = false;
|
||||
if (subtypeMimeSpecific != NULL) {
|
||||
mimeType->SetTo(subtypeMimeSpecific);
|
||||
if (mimeType->IsInstalled())
|
||||
found = true;
|
||||
}
|
||||
if (!found && subtypeMimeGeneric != NULL) {
|
||||
mimeType->SetTo(subtypeMimeGeneric);
|
||||
if (mimeType->IsInstalled())
|
||||
found = true;
|
||||
}
|
||||
if (!found)
|
||||
mimeType->SetTo("text/plain");
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
static int
|
||||
ascmatch(const unsigned char *s, const my_unichar *us, size_t ulen)
|
||||
{
|
||||
size_t i;
|
||||
|
||||
for (i = 0; i < ulen; i++) {
|
||||
if (s[i] != us[i])
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (s[i])
|
||||
return 0;
|
||||
else
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* This table reflects a particular philosophy about what constitutes
|
||||
* "text," and there is room for disagreement about it.
|
||||
*
|
||||
* Version 3.31 of the file command considered a file to be ASCII if
|
||||
* each of its characters was approved by either the isascii() or
|
||||
* isalpha() function. On most systems, this would mean that any
|
||||
* file consisting only of characters in the range 0x00 ... 0x7F
|
||||
* would be called ASCII text, but many systems might reasonably
|
||||
* consider some characters outside this range to be alphabetic,
|
||||
* so the file command would call such characters ASCII. It might
|
||||
* have been more accurate to call this "considered textual on the
|
||||
* local system" than "ASCII."
|
||||
*
|
||||
* It considered a file to be "International language text" if each
|
||||
* of its characters was either an ASCII printing character (according
|
||||
* to the real ASCII standard, not the above test), a character in
|
||||
* the range 0x80 ... 0xFF, or one of the following control characters:
|
||||
* backspace, tab, line feed, vertical tab, form feed, carriage return,
|
||||
* escape. No attempt was made to determine the language in which files
|
||||
* of this type were written.
|
||||
*
|
||||
*
|
||||
* The table below considers a file to be ASCII if all of its characters
|
||||
* are either ASCII printing characters (again, according to the X3.4
|
||||
* standard, not isascii()) or any of the following controls: bell,
|
||||
* backspace, tab, line feed, form feed, carriage return, esc, nextline.
|
||||
*
|
||||
* I include bell because some programs (particularly shell scripts)
|
||||
* use it literally, even though it is rare in normal text. I exclude
|
||||
* vertical tab because it never seems to be used in real text. I also
|
||||
* include, with hesitation, the X3.64/ECMA-43 control nextline (0x85),
|
||||
* because that's what the dd EBCDIC->ASCII table maps the EBCDIC newline
|
||||
* character to. It might be more appropriate to include it in the 8859
|
||||
* set instead of the ASCII set, but it's got to be included in *something*
|
||||
* we recognize or EBCDIC files aren't going to be considered textual.
|
||||
* Some old Unix source files use SO/SI (^N/^O) to shift between Greek
|
||||
* and Latin characters, so these should possibly be allowed. But they
|
||||
* make a real mess on VT100-style displays if they're not paired properly,
|
||||
* so we are probably better off not calling them text.
|
||||
*
|
||||
* A file is considered to be ISO-8859 text if its characters are all
|
||||
* either ASCII, according to the above definition, or printing characters
|
||||
* from the ISO-8859 8-bit extension, characters 0xA0 ... 0xFF.
|
||||
*
|
||||
* Finally, a file is considered to be international text from some other
|
||||
* character code if its characters are all either ISO-8859 (according to
|
||||
* the above definition) or characters in the range 0x80 ... 0x9F, which
|
||||
* ISO-8859 considers to be control characters but the IBM PC and Macintosh
|
||||
* consider to be printing characters.
|
||||
*/
|
||||
|
||||
#define F 0 /* character never appears in text */
|
||||
#define T 1 /* character appears in plain ASCII text */
|
||||
#define I 2 /* character appears in ISO-8859 text */
|
||||
#define X 3 /* character appears in non-ISO extended ASCII (Mac, IBM PC) */
|
||||
|
||||
static char text_chars[256] = {
|
||||
/* BEL BS HT LF FF CR */
|
||||
F, F, F, F, F, F, F, T, T, T, T, F, T, T, F, F, /* 0x0X */
|
||||
/* ESC */
|
||||
F, F, F, F, F, F, F, F, F, F, F, T, F, F, F, F, /* 0x1X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x2X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x3X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x4X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x5X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, /* 0x6X */
|
||||
T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, F, /* 0x7X */
|
||||
/* NEL */
|
||||
X, X, X, X, X, T, X, X, X, X, X, X, X, X, X, X, /* 0x8X */
|
||||
X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, X, /* 0x9X */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xaX */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xbX */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xcX */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xdX */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, /* 0xeX */
|
||||
I, I, I, I, I, I, I, I, I, I, I, I, I, I, I, I /* 0xfX */
|
||||
};
|
||||
|
||||
static int
|
||||
looks_ascii(const unsigned char *buf, size_t nbytes, my_unichar *ubuf,
|
||||
size_t *ulen)
|
||||
{
|
||||
int i;
|
||||
|
||||
*ulen = 0;
|
||||
|
||||
for (i = 0; i < (int)nbytes; i++) {
|
||||
int t = text_chars[buf[i]];
|
||||
|
||||
if (t != T)
|
||||
return 0;
|
||||
|
||||
ubuf[(*ulen)++] = buf[i];
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int
|
||||
looks_latin1(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, size_t *ulen)
|
||||
{
|
||||
int i;
|
||||
|
||||
*ulen = 0;
|
||||
|
||||
for (i = 0; i < (int)nbytes; i++) {
|
||||
int t = text_chars[buf[i]];
|
||||
|
||||
if (t != T && t != I)
|
||||
return 0;
|
||||
|
||||
ubuf[(*ulen)++] = buf[i];
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int
|
||||
looks_extended(const unsigned char *buf, size_t nbytes, my_unichar *ubuf,
|
||||
size_t *ulen)
|
||||
{
|
||||
int i;
|
||||
|
||||
*ulen = 0;
|
||||
|
||||
for (i = 0; i < (int)nbytes; i++) {
|
||||
int t = text_chars[buf[i]];
|
||||
|
||||
if (t != T && t != I && t != X)
|
||||
return 0;
|
||||
|
||||
ubuf[(*ulen)++] = buf[i];
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int
|
||||
looks_utf8(const unsigned char *buf, size_t nbytes, my_unichar *ubuf, size_t *ulen)
|
||||
{
|
||||
int i, n;
|
||||
my_unichar c;
|
||||
int gotone = 0;
|
||||
|
||||
*ulen = 0;
|
||||
|
||||
for (i = 0; i < (int)nbytes; i++) {
|
||||
if ((buf[i] & 0x80) == 0) { /* 0xxxxxxx is plain ASCII */
|
||||
/*
|
||||
* Even if the whole file is valid UTF-8 sequences,
|
||||
* still reject it if it uses weird control characters.
|
||||
*/
|
||||
|
||||
if (text_chars[buf[i]] != T)
|
||||
return 0;
|
||||
|
||||
ubuf[(*ulen)++] = buf[i];
|
||||
} else if ((buf[i] & 0x40) == 0) { /* 10xxxxxx never 1st byte */
|
||||
return 0;
|
||||
} else { /* 11xxxxxx begins UTF-8 */
|
||||
int following;
|
||||
|
||||
if ((buf[i] & 0x20) == 0) { /* 110xxxxx */
|
||||
c = buf[i] & 0x1f;
|
||||
following = 1;
|
||||
} else if ((buf[i] & 0x10) == 0) { /* 1110xxxx */
|
||||
c = buf[i] & 0x0f;
|
||||
following = 2;
|
||||
} else if ((buf[i] & 0x08) == 0) { /* 11110xxx */
|
||||
c = buf[i] & 0x07;
|
||||
following = 3;
|
||||
} else if ((buf[i] & 0x04) == 0) { /* 111110xx */
|
||||
c = buf[i] & 0x03;
|
||||
following = 4;
|
||||
} else if ((buf[i] & 0x02) == 0) { /* 1111110x */
|
||||
c = buf[i] & 0x01;
|
||||
following = 5;
|
||||
} else
|
||||
return 0;
|
||||
|
||||
for (n = 0; n < following; n++) {
|
||||
i++;
|
||||
if (i >= (int)nbytes)
|
||||
goto done;
|
||||
|
||||
if ((buf[i] & 0x80) == 0 || (buf[i] & 0x40))
|
||||
return 0;
|
||||
|
||||
c = (c << 6) + (buf[i] & 0x3f);
|
||||
}
|
||||
|
||||
ubuf[(*ulen)++] = c;
|
||||
gotone = 1;
|
||||
}
|
||||
}
|
||||
done:
|
||||
return gotone; /* don't claim it's UTF-8 if it's all 7-bit */
|
||||
}
|
||||
|
||||
static int
|
||||
looks_unicode(const unsigned char *buf, size_t nbytes, my_unichar *ubuf,
|
||||
size_t *ulen)
|
||||
{
|
||||
int bigend;
|
||||
int i;
|
||||
|
||||
if (nbytes < 2)
|
||||
return 0;
|
||||
|
||||
if (buf[0] == 0xff && buf[1] == 0xfe)
|
||||
bigend = 0;
|
||||
else if (buf[0] == 0xfe && buf[1] == 0xff)
|
||||
bigend = 1;
|
||||
else
|
||||
return 0;
|
||||
|
||||
*ulen = 0;
|
||||
|
||||
for (i = 2; i + 1 < (int)nbytes; i += 2) {
|
||||
/* XXX fix to properly handle chars > 65536 */
|
||||
|
||||
if (bigend)
|
||||
ubuf[(*ulen)++] = buf[i + 1] + 256 * buf[i];
|
||||
else
|
||||
ubuf[(*ulen)++] = buf[i] + 256 * buf[i + 1];
|
||||
|
||||
if (ubuf[*ulen - 1] == 0xfffe)
|
||||
return 0;
|
||||
if (ubuf[*ulen - 1] < 128 &&
|
||||
text_chars[(size_t)ubuf[*ulen - 1]] != T)
|
||||
return 0;
|
||||
}
|
||||
|
||||
return 1 + bigend;
|
||||
}
|
||||
|
||||
#undef F
|
||||
#undef T
|
||||
#undef I
|
||||
#undef X
|
||||
|
||||
/*
|
||||
* This table maps each EBCDIC character to an (8-bit extended) ASCII
|
||||
* character, as specified in the rationale for the dd(1) command in
|
||||
* draft 11.2 (September, 1991) of the POSIX P1003.2 standard.
|
||||
*
|
||||
* Unfortunately it does not seem to correspond exactly to any of the
|
||||
* five variants of EBCDIC documented in IBM's _Enterprise Systems
|
||||
* Architecture/390: Principles of Operation_, SA22-7201-06, Seventh
|
||||
* Edition, July, 1999, pp. I-1 - I-4.
|
||||
*
|
||||
* Fortunately, though, all versions of EBCDIC, including this one, agree
|
||||
* on most of the printing characters that also appear in (7-bit) ASCII.
|
||||
* Of these, only '|', '!', '~', '^', '[', and ']' are in question at all.
|
||||
*
|
||||
* Fortunately too, there is general agreement that codes 0x00 through
|
||||
* 0x3F represent control characters, 0x41 a nonbreaking space, and the
|
||||
* remainder printing characters.
|
||||
*
|
||||
* This is sufficient to allow us to identify EBCDIC text and to distinguish
|
||||
* between old-style and internationalized examples of text.
|
||||
*/
|
||||
|
||||
static unsigned char ebcdic_to_ascii[] = {
|
||||
0, 1, 2, 3, 156, 9, 134, 127, 151, 141, 142, 11, 12, 13, 14, 15,
|
||||
16, 17, 18, 19, 157, 133, 8, 135, 24, 25, 146, 143, 28, 29, 30, 31,
|
||||
128, 129, 130, 131, 132, 10, 23, 27, 136, 137, 138, 139, 140, 5, 6, 7,
|
||||
144, 145, 22, 147, 148, 149, 150, 4, 152, 153, 154, 155, 20, 21, 158, 26,
|
||||
' ', 160, 161, 162, 163, 164, 165, 166, 167, 168, 213, '.', '<', '(', '+', '|',
|
||||
'&', 169, 170, 171, 172, 173, 174, 175, 176, 177, '!', '$', '*', ')', ';', '~',
|
||||
'-', '/', 178, 179, 180, 181, 182, 183, 184, 185, 203, ',', '%', '_', '>', '?',
|
||||
186, 187, 188, 189, 190, 191, 192, 193, 194, '`', ':', '#', '@', '\'','=', '"',
|
||||
195, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 196, 197, 198, 199, 200, 201,
|
||||
202, 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', '^', 204, 205, 206, 207, 208,
|
||||
209, 229, 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 210, 211, 212, '[', 214, 215,
|
||||
216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, ']', 230, 231,
|
||||
'{', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 232, 233, 234, 235, 236, 237,
|
||||
'}', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 238, 239, 240, 241, 242, 243,
|
||||
'\\',159, 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 244, 245, 246, 247, 248, 249,
|
||||
'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 250, 251, 252, 253, 254, 255
|
||||
};
|
||||
|
||||
#ifdef notdef
|
||||
/*
|
||||
* The following EBCDIC-to-ASCII table may relate more closely to reality,
|
||||
* or at least to modern reality. It comes from
|
||||
*
|
||||
* http://ftp.s390.ibm.com/products/oe/bpxqp9.html
|
||||
*
|
||||
* and maps the characters of EBCDIC code page 1047 (the code used for
|
||||
* Unix-derived software on IBM's 390 systems) to the corresponding
|
||||
* characters from ISO 8859-1.
|
||||
*
|
||||
* If this table is used instead of the above one, some of the special
|
||||
* cases for the NEL character can be taken out of the code.
|
||||
*/
|
||||
|
||||
static unsigned char ebcdic_1047_to_8859[] = {
|
||||
0x00,0x01,0x02,0x03,0x9C,0x09,0x86,0x7F,0x97,0x8D,0x8E,0x0B,0x0C,0x0D,0x0E,0x0F,
|
||||
0x10,0x11,0x12,0x13,0x9D,0x0A,0x08,0x87,0x18,0x19,0x92,0x8F,0x1C,0x1D,0x1E,0x1F,
|
||||
0x80,0x81,0x82,0x83,0x84,0x85,0x17,0x1B,0x88,0x89,0x8A,0x8B,0x8C,0x05,0x06,0x07,
|
||||
0x90,0x91,0x16,0x93,0x94,0x95,0x96,0x04,0x98,0x99,0x9A,0x9B,0x14,0x15,0x9E,0x1A,
|
||||
0x20,0xA0,0xE2,0xE4,0xE0,0xE1,0xE3,0xE5,0xE7,0xF1,0xA2,0x2E,0x3C,0x28,0x2B,0x7C,
|
||||
0x26,0xE9,0xEA,0xEB,0xE8,0xED,0xEE,0xEF,0xEC,0xDF,0x21,0x24,0x2A,0x29,0x3B,0x5E,
|
||||
0x2D,0x2F,0xC2,0xC4,0xC0,0xC1,0xC3,0xC5,0xC7,0xD1,0xA6,0x2C,0x25,0x5F,0x3E,0x3F,
|
||||
0xF8,0xC9,0xCA,0xCB,0xC8,0xCD,0xCE,0xCF,0xCC,0x60,0x3A,0x23,0x40,0x27,0x3D,0x22,
|
||||
0xD8,0x61,0x62,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0xAB,0xBB,0xF0,0xFD,0xFE,0xB1,
|
||||
0xB0,0x6A,0x6B,0x6C,0x6D,0x6E,0x6F,0x70,0x71,0x72,0xAA,0xBA,0xE6,0xB8,0xC6,0xA4,
|
||||
0xB5,0x7E,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7A,0xA1,0xBF,0xD0,0x5B,0xDE,0xAE,
|
||||
0xAC,0xA3,0xA5,0xB7,0xA9,0xA7,0xB6,0xBC,0xBD,0xBE,0xDD,0xA8,0xAF,0x5D,0xB4,0xD7,
|
||||
0x7B,0x41,0x42,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0xAD,0xF4,0xF6,0xF2,0xF3,0xF5,
|
||||
0x7D,0x4A,0x4B,0x4C,0x4D,0x4E,0x4F,0x50,0x51,0x52,0xB9,0xFB,0xFC,0xF9,0xFA,0xFF,
|
||||
0x5C,0xF7,0x53,0x54,0x55,0x56,0x57,0x58,0x59,0x5A,0xB2,0xD4,0xD6,0xD2,0xD3,0xD5,
|
||||
0x30,0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,0x39,0xB3,0xDB,0xDC,0xD9,0xDA,0x9F
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Copy buf[0 ... nbytes-1] into out[], translating EBCDIC to ASCII.
|
||||
*/
|
||||
static void
|
||||
from_ebcdic(const unsigned char *buf, size_t nbytes, unsigned char *out)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < (int)nbytes; i++) {
|
||||
out[i] = ebcdic_to_ascii[buf[i]];
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// #pragma mark -
|
||||
|
||||
|
||||
@ -889,10 +230,12 @@ identify_text(uint8* data, int32 bytesRead, BPositionIO* source,
|
||||
|
||||
bytesRead += readLater;
|
||||
|
||||
// TODO: identify encoding as possible!
|
||||
BMimeType type;
|
||||
if (!file_ascmagic((const unsigned char*)data, bytesRead, &type, encoding))
|
||||
TextEncoding textEncoding((char*)data, (size_t)bytesRead);
|
||||
encoding = textEncoding.GetName();
|
||||
if (strlen(encoding) == 0) {
|
||||
/* No valid character encoding found! */
|
||||
return B_NO_TRANSLATOR;
|
||||
}
|
||||
|
||||
float capability = TEXT_IN_CAPABILITY;
|
||||
if (bytesRead < 20)
|
||||
@ -904,12 +247,8 @@ identify_text(uint8* data, int32 bytesRead, BPositionIO* source,
|
||||
outInfo->quality = TEXT_IN_QUALITY;
|
||||
outInfo->capability = capability;
|
||||
|
||||
char description[B_MIME_TYPE_LENGTH];
|
||||
if (type.GetLongDescription(description) == B_OK)
|
||||
strlcpy(outInfo->name, description, sizeof(outInfo->name));
|
||||
else
|
||||
strlcpy(outInfo->name, B_TRANSLATE("Plain text file"),
|
||||
sizeof(outInfo->name));
|
||||
strlcpy(outInfo->name, B_TRANSLATE("Plain text file"),
|
||||
sizeof(outInfo->name));
|
||||
|
||||
//strlcpy(outInfo->MIME, type.Type(), sizeof(outInfo->MIME));
|
||||
strcpy(outInfo->MIME, "text/plain");
|
||||
|
||||
@ -21,6 +21,7 @@ local sources =
|
||||
LocaleRoster.cpp
|
||||
LocaleRosterData.cpp
|
||||
MutableLocaleRoster.cpp
|
||||
TextEncoding.cpp
|
||||
TimeZone.cpp
|
||||
|
||||
# in progress
|
||||
|
||||
29
src/kits/locale/TextEncoding.cpp
Normal file
29
src/kits/locale/TextEncoding.cpp
Normal file
@ -0,0 +1,29 @@
|
||||
/*
|
||||
* Copyright 2016, Haiku, inc.
|
||||
* Distributed under terms of the MIT license.
|
||||
*/
|
||||
|
||||
|
||||
#include "TextEncoding.h"
|
||||
|
||||
#include <unicode/ucsdet.h>
|
||||
|
||||
|
||||
TextEncoding::TextEncoding(const char* data, size_t length)
|
||||
{
|
||||
UErrorCode error = U_ZERO_ERROR;
|
||||
|
||||
UCharsetDetector* detector = ucsdet_open(&error);
|
||||
ucsdet_setText(detector, data, length, &error);
|
||||
const UCharsetMatch* encoding = ucsdet_detect(detector, &error);
|
||||
|
||||
fName = ucsdet_getName(encoding, &error);
|
||||
ucsdet_close(detector);
|
||||
}
|
||||
|
||||
|
||||
BString
|
||||
TextEncoding::GetName()
|
||||
{
|
||||
return fName;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user