kuroko-lang/kuroko
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Stop using tupleOf in codec tools

Browse Source
This commit is contained in:
K. Lange 2022-07-05 11:42:57 +09:00
parent a429aa93dd
commit 19927ea13f
3 changed files with 19 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
modules/codecs/dbextra.krk
View File

@ -112,7 +112,7 @@ class XMacChineseTradIncrementalDecoder(AsciiIncrementalDecoder):
0xFE: 0x2122, 0xFE: 0x2122,
0xFF: 0x2026, 0xFF: 0x2026,
}) })
dbrange = tupleOf(*range(0xA1, 0xFC + 1)) dbrange = tuple(range(0xA1, 0xFC + 1))
trailrange = Big5HkscsIncrementalDecoder.trailrange trailrange = Big5HkscsIncrementalDecoder.trailrange
register_kuroko_codec(["x-mac-chinesetrad", "x-mac-trad-chinese"], register_kuroko_codec(["x-mac-chinesetrad", "x-mac-trad-chinese"],
@ -169,8 +169,8 @@ class XMacChineseSimpIncrementalDecoder(AsciiIncrementalDecoder):
0xFE: 0x2122, 0xFE: 0x2122,
0xFF: 0x2026, 0xFF: 0x2026,
}) })
dbrange = tupleOf(*range(0xA1, 0xFC + 1)) dbrange = tuple(range(0xA1, 0xFC + 1))
trailrange = tupleOf(*range(0xA1, 0xFE + 1)) trailrange = tuple(range(0xA1, 0xFE + 1))
register_kuroko_codec(["x-mac-chinesesimp", "x-mac-simp-chinese", "euc-cn", "euccn", "eucgb2312-cn"], register_kuroko_codec(["x-mac-chinesesimp", "x-mac-simp-chinese", "euc-cn", "euccn", "eucgb2312-cn"],
XMacChineseSimpIncrementalEncoder, XMacChineseSimpIncrementalDecoder) XMacChineseSimpIncrementalEncoder, XMacChineseSimpIncrementalDecoder)
@ -453,9 +453,9 @@ class EucJis2004IncrementalDecoder(AsciiIncrementalDecoder):
@lazy_property @lazy_property
def decoding_map(): def decoding_map():
return data_8bit.decode_euc04 return data_8bit.decode_euc04
dbrange = tupleOf(0x8E, *range(0xA1, 0xFE + 1)) dbrange = (0x8E,) + tuple(range(0xA1, 0xFE + 1))
tbrange = (0x8F,) tbrange = (0x8F,)
trailrange = tupleOf(*range(0xA1, 0xFE + 1)) trailrange = tuple(range(0xA1, 0xFE + 1))
register_kuroko_codec(["euc-jis-2004", "jisx0213", "eucjis2004", "euc_jis2004", register_kuroko_codec(["euc-jis-2004", "jisx0213", "eucjis2004", "euc_jis2004",
"euc_jisx0213", "eucjisx0213"], "euc_jisx0213", "eucjisx0213"],
@ -487,7 +487,7 @@ class ShiftJis2004IncrementalDecoder(AsciiIncrementalDecoder):
146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 224, 225, 226, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 224, 225, 226,
227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 227, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243,
244, 245, 246, 247, 248, 249, 250, 251, 252) 244, 245, 246, 247, 248, 249, 250, 251, 252)
trailrange = tupleOf(*range(64, 126 + 1), *range(128, 252 + 1)) trailrange = tuple(range(64, 126 + 1)) + tuple(range(128, 252 + 1))
register_kuroko_codec(["shift_jis-2004", "shiftjis2004", "sjis_2004", "s_jis_2004", register_kuroko_codec(["shift_jis-2004", "shiftjis2004", "sjis_2004", "s_jis_2004",
"shift_jisx0213", "shiftjisx0213", "sjisx0213", "s_jisx0213"], "shift_jisx0213", "shiftjisx0213", "sjisx0213", "s_jisx0213"],
@ -513,9 +513,9 @@ class AsciiJohabIncrementalDecoder(AsciiIncrementalDecoder):
@lazy_property @lazy_property
def decoding_map(): def decoding_map():
return data_8bit.decode_johab_ascii return data_8bit.decode_johab_ascii
dbrange = tupleOf(*range(0x84, 0xF9 + 1)) dbrange = tuple(range(0x84, 0xF9 + 1))
# Trail ranges for hangul and nonhangul are different, but this is their union. # Trail ranges for hangul and nonhangul are different, but this is their union.
trailrange = tupleOf(*range(0x31, 0x7E + 1), *range(0x81, 0xFE + 1)) trailrange = tuple(range(0x31, 0x7E + 1)) + tuple(range(0x81, 0xFE + 1))
register_kuroko_codec(["cp1361", "ms1361", "johab", "x-johab", "johab-ascii"], register_kuroko_codec(["cp1361", "ms1361", "johab", "x-johab", "johab-ascii"],
AsciiJohabIncrementalEncoder, AsciiJohabIncrementalDecoder) AsciiJohabIncrementalEncoder, AsciiJohabIncrementalDecoder)

6
modules/codecs/infrastructure.krk
View File

@ -455,15 +455,15 @@ class AsciiIncrementalEncoder(IncrementalEncoder):
for seq in seqs: for seq in seqs:
# TODO: where one mapped multi-codepoint sequence starts with another mapped # TODO: where one mapped multi-codepoint sequence starts with another mapped
# multi-codepoint sequence is still pathological. # multi-codepoint sequence is still pathological.
if tupleOf(*string_bit[:len(seq)]) == seq: if tuple(string_bit[:len(seq)]) == seq:
out.add(bytes(self.encoding_map[seq])) out.add(bytes(self.encoding_map[seq]))
offset += len(seq) offset += len(seq)
if offset >= len(string): if offset >= len(string):
return out.getvalue() return out.getvalue()
i = string[offset] i = string[offset]
break break
else if (not final) and (tupleOf(*string_bit) == else if (not final) and (tuple(string_bit) ==
tupleOf(*list(seq)[:testable_length])): tuple(list(seq)[:testable_length])):
self.pending_lead = "".join([chr(i) for i in string_bit]) self.pending_lead = "".join([chr(i) for i in string_bit])
return out.getvalue() return out.getvalue()
if ord(i) < 0x80 and ord(i) not in self.ascii_exceptions: if ord(i) < 0x80 and ord(i) not in self.ascii_exceptions:

16
tools/codectools/gen_dbdata.krk
View File

@ -100,18 +100,18 @@ let encode_big5web = {}
let encode_big5hkscs = {} let encode_big5hkscs = {}
let encode_big5hkscs_extras = {} let encode_big5hkscs_extras = {}
let encode_gbk = {} let encode_gbk = {}
let dbrange_eucjp = tupleOf(0x8E, *range(0xA1, 0xFE + 1)) let dbrange_eucjp = (0x8E,) + tuple(range(0xA1, 0xFE + 1))
let dbrange_shiftjis = tupleOf(*range(0x81, 0x9F + 1), *range(0xE0, 0xFC + 1)) let dbrange_shiftjis = tuple(range(0x81, 0x9F + 1)) + tuple(range(0xE0, 0xFC + 1))
let dbrange_uhc = tupleOf(*range(0x81, 0xFE + 1)) let dbrange_uhc = tuple(range(0x81, 0xFE + 1))
let dbrange_big5 = tupleOf(*range(0x81, 0xFE + 1)) let dbrange_big5 = tuple(range(0x81, 0xFE + 1))
let tbrange_eucjp = (0x8F,) let tbrange_eucjp = (0x8F,)
let tbrange_shiftjis = () let tbrange_shiftjis = ()
let tbrange_uhc = () let tbrange_uhc = ()
let tbrange_big5 = () let tbrange_big5 = ()
let trailrange_eucjp = tupleOf(*range(0xA1, 0xFE + 1)) let trailrange_eucjp = tuple(range(0xA1, 0xFE + 1))
let trailrange_shiftjis = tupleOf(*range(0x40, 0x7E + 1), *range(0x80, 0xFC + 1)) let trailrange_shiftjis = tuple(range(0x40, 0x7E + 1)) + tuple(range(0x80, 0xFC + 1))
let trailrange_uhc = tupleOf(*range(0x41, 0xFE + 1)) # As WHATWG define it (a superset of the actual range) let trailrange_uhc = tuple(range(0x41, 0xFE + 1)) # As WHATWG define it (a superset of the actual range)
let trailrange_big5 = tupleOf(*range(0x40, 0x7E + 1), *range(0xA1, 0xFE + 1)) let trailrange_big5 = tuple(range(0x40, 0x7E + 1)) + tuple(range(0xA1, 0xFE + 1))
for pointer, ucs in enumerate(indices['jis0208']): for pointer, ucs in enumerate(indices['jis0208']):
if ucs == None: continue if ucs == None: continue