Add seven kanji characters defined in the Windows 950 codepage to our
big5/win950 <-> UTF8 conversion tables. Per report by Roger Chang.
This commit is contained in:
Heikki Linnakangas
parent
8e1a8fe288
commit
2dbbf33f4a
177
src/backend/utils/mb/Unicode/UCS_to_big5.pl
Normal file
177
src/backend/utils/mb/Unicode/UCS_to_big5.pl
Normal file
@ -0,0 +1,177 @@
|
||||
#! /usr/bin/perl
|
||||
#
|
||||
# Copyright (c) 2001-2009, PostgreSQL Global Development Group
|
||||
#
|
||||
# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_big5.pl,v 1.1 2009/03/18 16:17:58 heikki Exp $
|
||||
#
|
||||
# Generate UTF-8 <--> BIG5 conversion tables from
|
||||
# map files provided by Unicode organization.
|
||||
# Unfortunately it is prohibited by the organization
|
||||
# to distribute the map files. So if you try to use this script,
|
||||
# you have to obtain the map files from the organization's ftp site.
|
||||
# ftp://www.unicode.org/Public/MAPPINGS/
|
||||
#
|
||||
# Our "big5" comes from BIG5.TXT, with the addition of the characters
|
||||
# in the range 0xf9d6-0xf9dc from CP950.TXT.
|
||||
#
|
||||
# BIG5.TXT format:
|
||||
# BIG5 code in hex
|
||||
# UCS-2 code in hex
|
||||
# # and Unicode name (not used in this script)
|
||||
#
|
||||
# CP950.TXT format:
|
||||
# CP950 code in hex
|
||||
# UCS-2 code in hex
|
||||
# # and Unicode name (not used in this script)
|
||||
|
||||
|
||||
require "ucs2utf.pl";
|
||||
|
||||
|
||||
#
|
||||
# first, generate UTF8 --> BIG5 table
|
||||
#
|
||||
$in_file = "BIG5.TXT";
|
||||
|
||||
open( FILE, $in_file ) || die( "cannot open $in_file" );
|
||||
|
||||
reset 'array';
|
||||
|
||||
while( <FILE> ){
|
||||
chop;
|
||||
if( /^#/ ){
|
||||
next;
|
||||
}
|
||||
( $c, $u, $rest ) = split;
|
||||
$ucs = hex($u);
|
||||
$code = hex($c);
|
||||
if( $code >= 0x80 && $ucs >= 0x0080){
|
||||
$utf = &ucs2utf($ucs);
|
||||
if( $array{ $utf } ne "" ){
|
||||
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
|
||||
next;
|
||||
}
|
||||
$count++;
|
||||
$array{ $utf } = $code;
|
||||
}
|
||||
}
|
||||
close( FILE );
|
||||
|
||||
$in_file = "CP950.TXT";
|
||||
|
||||
open( FILE, $in_file ) || die( "cannot open $in_file" );
|
||||
|
||||
while( <FILE> ){
|
||||
chop;
|
||||
if( /^#/ ){
|
||||
next;
|
||||
}
|
||||
( $c, $u, $rest ) = split;
|
||||
$ucs = hex($u);
|
||||
$code = hex($c);
|
||||
|
||||
# Pick only the ETEN extended characters in the range 0xf9d6 - 0xf9dc
|
||||
# from CP950.TXT
|
||||
if( $code >= 0x80 && $ucs >= 0x0080 &&
|
||||
$code >= 0xf9d6 && $code <= 0xf9dc ){
|
||||
$utf = &ucs2utf($ucs);
|
||||
if( $array{ $utf } ne "" ){
|
||||
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
|
||||
next;
|
||||
}
|
||||
$count++;
|
||||
$array{ $utf } = $code;
|
||||
}
|
||||
}
|
||||
close( FILE );
|
||||
|
||||
$file = lc("utf8_to_big5.map");
|
||||
open( FILE, "> $file" ) || die( "cannot open $file" );
|
||||
print FILE "static pg_utf_to_local ULmapBIG5[ $count ] = {\n";
|
||||
|
||||
for $index ( sort {$a <=> $b} keys( %array ) ){
|
||||
$code = $array{ $index };
|
||||
$count--;
|
||||
if( $count == 0 ){
|
||||
printf FILE " {0x%04x, 0x%04x}\n", $index, $code;
|
||||
} else {
|
||||
printf FILE " {0x%04x, 0x%04x},\n", $index, $code;
|
||||
}
|
||||
}
|
||||
|
||||
print FILE "};\n";
|
||||
close(FILE);
|
||||
|
||||
#
|
||||
# then generate BIG5 --> UTF8 table
|
||||
#
|
||||
$in_file = "BIG5.TXT";
|
||||
|
||||
open( FILE, $in_file ) || die( "cannot open $in_file" );
|
||||
|
||||
reset 'array';
|
||||
|
||||
while( <FILE> ){
|
||||
chop;
|
||||
if( /^#/ ){
|
||||
next;
|
||||
}
|
||||
( $c, $u, $rest ) = split;
|
||||
$ucs = hex($u);
|
||||
$code = hex($c);
|
||||
if( $code >= 0x80 && $ucs >= 0x0080){
|
||||
$utf = &ucs2utf($ucs);
|
||||
if( $array{ $utf } ne "" ){
|
||||
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
|
||||
next;
|
||||
}
|
||||
$count++;
|
||||
$array{ $code } = $utf;
|
||||
}
|
||||
}
|
||||
close( FILE );
|
||||
|
||||
$in_file = "CP950.TXT";
|
||||
|
||||
open( FILE, $in_file ) || die( "cannot open $in_file" );
|
||||
|
||||
while( <FILE> ){
|
||||
chop;
|
||||
if( /^#/ ){
|
||||
next;
|
||||
}
|
||||
( $c, $u, $rest ) = split;
|
||||
$ucs = hex($u);
|
||||
$code = hex($c);
|
||||
|
||||
# Pick only the ETEN extended characters in the range 0xf9d6 - 0xf9dc
|
||||
# from CP950.TXT
|
||||
if( $code >= 0x80 && $ucs >= 0x0080 &&
|
||||
$code >= 0xf9d6 && $code <= 0xf9dc ){
|
||||
$utf = &ucs2utf($ucs);
|
||||
if( $array{ $utf } ne "" ){
|
||||
printf STDERR "Warning: duplicate UTF8: %04x\n",$ucs;
|
||||
next;
|
||||
}
|
||||
$count++;
|
||||
$array{ $code } = $utf;
|
||||
}
|
||||
}
|
||||
close( FILE );
|
||||
|
||||
$file = lc("big5_to_utf8.map");
|
||||
open( FILE, "> $file" ) || die( "cannot open $file" );
|
||||
print FILE "static pg_local_to_utf LUmapBIG5[ $count ] = {\n";
|
||||
for $index ( sort {$a <=> $b} keys( %array ) ){
|
||||
$utf = $array{ $index };
|
||||
$count--;
|
||||
if( $count == 0 ){
|
||||
printf FILE " {0x%04x, 0x%04x}\n", $index, $utf;
|
||||
} else {
|
||||
printf FILE " {0x%04x, 0x%04x},\n", $index, $utf;
|
||||
}
|
||||
}
|
||||
|
||||
print FILE "};\n";
|
||||
close(FILE);
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
#
|
||||
# Copyright (c) 2001-2009, PostgreSQL Global Development Group
|
||||
#
|
||||
# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_most.pl,v 1.7 2009/02/10 19:29:39 petere Exp $
|
||||
# $PostgreSQL: pgsql/src/backend/utils/mb/Unicode/UCS_to_most.pl,v 1.8 2009/03/18 16:17:28 heikki Exp $
|
||||
#
|
||||
# Generate UTF-8 <--> character code conversion tables from
|
||||
# map files provided by Unicode organization.
|
||||
@ -47,7 +47,6 @@ require "ucs2utf.pl";
|
||||
'GBK' => 'CP936.TXT',
|
||||
'UHC' => 'CP949.TXT',
|
||||
'JOHAB' => 'JOHAB.TXT',
|
||||
'BIG5' => 'BIG5.TXT',
|
||||
);
|
||||
|
||||
@charsets = keys(filename);
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
static pg_local_to_utf LUmapBIG5[ 13710 ] = {
|
||||
static pg_local_to_utf LUmapBIG5[ 13717 ] = {
|
||||
{0xa140, 0xe38080},
|
||||
{0xa141, 0xefbc8c},
|
||||
{0xa142, 0xe38081},
|
||||
@ -13708,5 +13708,12 @@ static pg_local_to_utf LUmapBIG5[ 13710 ] = {
|
||||
{0xf9d2, 0xe9baa4},
|
||||
{0xf9d3, 0xe9bdbe},
|
||||
{0xf9d4, 0xe9bd89},
|
||||
{0xf9d5, 0xe9be98}
|
||||
{0xf9d5, 0xe9be98},
|
||||
{0xf9d6, 0xe7a281},
|
||||
{0xf9d7, 0xe98ab9},
|
||||
{0xf9d8, 0xe8a38f},
|
||||
{0xf9d9, 0xe5a2bb},
|
||||
{0xf9da, 0xe68192},
|
||||
{0xf9db, 0xe7b2a7},
|
||||
{0xf9dc, 0xe5abba}
|
||||
};
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
static pg_utf_to_local ULmapBIG5[ 13711 ] = {
|
||||
{0xc2a2, 0xa246},
|
||||
{0xc2a3, 0xa247},
|
||||
{0xc2a5, 0xa244},
|
||||
@ -2140,6 +2140,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe5a2b1, 0xe54c},
|
||||
{0xe5a2b3, 0xbc58},
|
||||
{0xe5a2ba, 0xe94d},
|
||||
{0xe5a2bb, 0xf9d9},
|
||||
{0xe5a2bc, 0xe94f},
|
||||
{0xe5a2bd, 0xe94a},
|
||||
{0xe5a2be, 0xbec1},
|
||||
@ -2508,6 +2509,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe5abb7, 0xe557},
|
||||
{0xe5abb8, 0xe55a},
|
||||
{0xe5abb9, 0xe55c},
|
||||
{0xe5abba, 0xf9dc},
|
||||
{0xe5abbb, 0xbc5f},
|
||||
{0xe5abbd, 0xe556},
|
||||
{0xe5abbf, 0xe554},
|
||||
@ -3370,6 +3372,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe6818c, 0xcec9},
|
||||
{0xe6818d, 0xabe9},
|
||||
{0xe68190, 0xaea3},
|
||||
{0xe68192, 0xf9da},
|
||||
{0xe68193, 0xcec5},
|
||||
{0xe68194, 0xcec1},
|
||||
{0xe68195, 0xaea4},
|
||||
@ -7482,6 +7485,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe7a1be, 0xe2f0},
|
||||
{0xe7a1bf, 0xb851},
|
||||
{0xe7a280, 0xdef0},
|
||||
{0xe7a281, 0xf9d6},
|
||||
{0xe7a283, 0xdeed},
|
||||
{0xe7a284, 0xdee8},
|
||||
{0xe7a285, 0xdeea},
|
||||
@ -8142,6 +8146,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe7b2a2, 0xdae7},
|
||||
{0xe7b2a3, 0xd6e1},
|
||||
{0xe7b2a5, 0xb5b0},
|
||||
{0xe7b2a7, 0xf9db},
|
||||
{0xe7b2a8, 0xdae9},
|
||||
{0xe7b2af, 0xdf56},
|
||||
{0xe7b2b1, 0xb864},
|
||||
@ -10208,6 +10213,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe8a38c, 0xdff8},
|
||||
{0xe8a38d, 0xdff3},
|
||||
{0xe8a38e, 0xdff4},
|
||||
{0xe8a38f, 0xf9d8},
|
||||
{0xe8a390, 0xdff9},
|
||||
{0xe8a392, 0xb8cf},
|
||||
{0xe8a394, 0xb8c7},
|
||||
@ -11806,6 +11812,7 @@ static pg_utf_to_local ULmapBIG5[ 13704 ] = {
|
||||
{0xe98ab5, 0xe8a1},
|
||||
{0xe98ab6, 0xe867},
|
||||
{0xe98ab7, 0xbe50},
|
||||
{0xe98ab9, 0xf9d7},
|
||||
{0xe98abb, 0xbe4f},
|
||||
{0xe98abc, 0xbe56},
|
||||
{0xe98b80, 0xe865},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user