147 lines
3.9 KiB
Tcl
147 lines
3.9 KiB
Tcl
|
|
#--------------------------------------------------------------------------
|
|
# Parameter $zName must be a path to the file UnicodeData.txt. This command
|
|
# reads the file and returns a list of mappings required to remove all
|
|
# diacritical marks from a unicode string. Each mapping is itself a list
|
|
# consisting of two elements - the unicode codepoint and the single ASCII
|
|
# character that it should be replaced with, or an empty string if the
|
|
# codepoint should simply be removed from the input. Examples:
|
|
#
|
|
# { 224 a } (replace codepoint 224 to "a")
|
|
# { 769 "" } (remove codepoint 769 from input)
|
|
#
|
|
# Mappings are only returned for non-upper case codepoints. It is assumed
|
|
# that the input has already been folded to lower case.
|
|
#
|
|
proc rd_load_unicodedata_text {zName} {
|
|
global tl_lookup_table
|
|
|
|
set fd [open $zName]
|
|
set lField {
|
|
code
|
|
character_name
|
|
general_category
|
|
canonical_combining_classes
|
|
bidirectional_category
|
|
character_decomposition_mapping
|
|
decimal_digit_value
|
|
digit_value
|
|
numeric_value
|
|
mirrored
|
|
unicode_1_name
|
|
iso10646_comment_field
|
|
uppercase_mapping
|
|
lowercase_mapping
|
|
titlecase_mapping
|
|
}
|
|
set lRet [list]
|
|
|
|
while { ![eof $fd] } {
|
|
set line [gets $fd]
|
|
if {$line == ""} continue
|
|
|
|
set fields [split $line ";"]
|
|
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
|
|
foreach $lField $fields {}
|
|
if { [llength $character_decomposition_mapping]!=2
|
|
|| [string is xdigit [lindex $character_decomposition_mapping 0]]==0
|
|
} {
|
|
continue
|
|
}
|
|
|
|
set iCode [expr "0x$code"]
|
|
set iAscii [expr "0x[lindex $character_decomposition_mapping 0]"]
|
|
set iDia [expr "0x[lindex $character_decomposition_mapping 1]"]
|
|
|
|
if {[info exists tl_lookup_table($iCode)]} continue
|
|
|
|
if { ($iAscii >= 97 && $iAscii <= 122)
|
|
|| ($iAscii >= 65 && $iAscii <= 90)
|
|
} {
|
|
lappend lRet [list $iCode [string tolower [format %c $iAscii]]]
|
|
set dia($iDia) 1
|
|
}
|
|
}
|
|
|
|
foreach d [array names dia] {
|
|
lappend lRet [list $d ""]
|
|
}
|
|
set lRet [lsort -integer -index 0 $lRet]
|
|
|
|
close $fd
|
|
set lRet
|
|
}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Parameter $zName must be a path to the file UnicodeData.txt. This command
|
|
# reads the file and returns a list of codepoints (integers). The list
|
|
# contains all codepoints in the UnicodeData.txt assigned to any "General
|
|
# Category" that is not a "Letter" or "Number".
|
|
#
|
|
proc an_load_unicodedata_text {zName} {
|
|
set fd [open $zName]
|
|
set lField {
|
|
code
|
|
character_name
|
|
general_category
|
|
canonical_combining_classes
|
|
bidirectional_category
|
|
character_decomposition_mapping
|
|
decimal_digit_value
|
|
digit_value
|
|
numeric_value
|
|
mirrored
|
|
unicode_1_name
|
|
iso10646_comment_field
|
|
uppercase_mapping
|
|
lowercase_mapping
|
|
titlecase_mapping
|
|
}
|
|
set lRet [list]
|
|
|
|
while { ![eof $fd] } {
|
|
set line [gets $fd]
|
|
if {$line == ""} continue
|
|
|
|
set fields [split $line ";"]
|
|
if {[llength $fields] != [llength $lField]} { error "parse error: $line" }
|
|
foreach $lField $fields {}
|
|
|
|
set iCode [expr "0x$code"]
|
|
set bAlnum [expr {
|
|
[lsearch {L N} [string range $general_category 0 0]] >= 0
|
|
|| $general_category=="Co"
|
|
}]
|
|
|
|
if { !$bAlnum } { lappend lRet $iCode }
|
|
}
|
|
|
|
close $fd
|
|
set lRet
|
|
}
|
|
|
|
proc tl_load_casefolding_txt {zName} {
|
|
global tl_lookup_table
|
|
|
|
set fd [open $zName]
|
|
while { ![eof $fd] } {
|
|
set line [gets $fd]
|
|
if {[string range $line 0 0] == "#"} continue
|
|
if {$line == ""} continue
|
|
|
|
foreach x {a b c d} {unset -nocomplain $x}
|
|
foreach {a b c d} [split $line ";"] {}
|
|
|
|
set a2 [list]
|
|
set c2 [list]
|
|
foreach elem $a { lappend a2 [expr "0x[string trim $elem]"] }
|
|
foreach elem $c { lappend c2 [expr "0x[string trim $elem]"] }
|
|
set b [string trim $b]
|
|
set d [string trim $d]
|
|
|
|
if {$b=="C" || $b=="S"} { set tl_lookup_table($a2) $c2 }
|
|
}
|
|
}
|
|
|
|
|