diff --git a/modules/syntax/__init__.krk b/modules/syntax/__init__.krk new file mode 100644 index 0000000..e69de29 diff --git a/modules/syntax/highlighter.krk b/modules/syntax/highlighter.krk new file mode 100644 index 0000000..eb23d4a --- /dev/null +++ b/modules/syntax/highlighter.krk @@ -0,0 +1,247 @@ +''' +Read strings of Kuroko source code and calculate syntax highlighting properties. +''' + +import string + +let FLAG_NONE = 0 +let FLAG_KEYWORD = 1 +let FLAG_STRING = 2 +let FLAG_COMMENT = 3 +let FLAG_TYPE = 4 +let FLAG_PRAGMA = 5 +let FLAG_NUMERAL = 6 +let FLAG_ERROR = 7 +let FLAG_DIFFPLUS = 8 +let FLAG_DIFFMINUS = 9 +let FLAG_NOTICE = 10 +let FLAG_BOLD = 11 +let FLAG_LINK = 12 +let FLAG_ESCAPE = 13 + +let keywords = [ + 'and','class','def','else','export','for','if','in','import','let','not', + 'or','return','while','try','except','raise','continue','break','as','from', + 'elif', 'lambda' +] + +let types = [ + 'self','super','len','str','int','float','dir','repr','list','dict','range', + 'object','exception','isinstance','type','print' +] + +let special = [ + 'True', 'False', 'None' +] + +def toTerminal(processed,colors=None): + if not colors: + colors = { + FLAG_NONE: '\[[0;39m', + FLAG_KEYWORD: '\[[0;94m', + FLAG_STRING: '\[[0;32m', + FLAG_COMMENT: '\[[0;90m', + FLAG_TYPE: '\[[0;33m', + FLAG_PRAGMA: '\[[0;31m', + FLAG_NUMERAL: '\[[0;35m', + FLAG_ERROR: '\[[0;97;41m', + FLAG_DIFFPLUS: '\[[0;92m', + FLAG_DIFFMINUS: '\[[0;91m', + FLAG_NOTICE: '\[[0;30;103m', + FLAG_BOLD: '\[[0;1m', + FLAG_LINK: '\[[0;4;94m', + FLAG_ESCAPE: '\[[0;92m', + } + for line in processed: + for unit in line: + print(colors[unit[1]] + unit[0],end='') + print('\[[0m') + +class State(): + def __init__(self, code): + self.lines = code.strip().split('\n') + + def highlight(self): + self.line_no = 0 + self.linestates = [] + for line in self.lines: + self.col_no = 0 + self.line = line + self.current = None + self.states = [] + self.state = 0 + while True: + self.state = self.highlightOnce() + if self.state == None: + break + if self.current: + self.states.append(self.current) + self.linestates.append(self.states) + self.line_no++ + + def process(self): + let final = [] + for i = 0; i < len(self.lines); i++: + let l = self.lines[i] + let s = self.linestates[i] + let out = [] + for state in s: + let w = l[state[0]:state[0]+state[1]] + out.append((w,state[2])) + final.append(out) + return final + + def charat(self): + return self.charrel(0) + + def nextchar(self): + return self.charrel(1) + + def lastchar(self): + return self.charrel(-1) + + def charrel(self,i): + if self.col_no + i >= 0 and self.col_no + i < len(self.line): + return self.line[self.col_no+i] + return None + + def paint(self, width, flag): + if not self.current: + self.current = [self.col_no, width, flag] + else: + if self.current[2] != flag: + self.states.append(self.current) + self.current = [self.col_no, width, flag] + else: + self.current[1] += width + self.col_no += width + + def isdigit(self, c): + return c in string.digits + + def isxdigit(self, c): + return c in string.hexdigits + + def isalpha(self, c): + return c in string.ascii_letters + + def cKeywordQualifier(self, c): + return c in (string.ascii_letters + string.digits + '_') + + def findKeywords(self, words, flag, checker): + if checker(self.lastchar()): return False + if not checker(self.charat()): return False + for word in words: + let d = 0 + while self.col_no + d < len(self.line) and d < len(word) and self.charrel(d) == word[d]: + d++ + if d == len(word) and (self.col_no + d >= len(self.line) or not checker(self.charrel(d))): + self.paint(len(word),flag) + return True + return False + + def skip(self): + self.paint(1,FLAG_NONE) + + def paintComment(self): + while self.charat() != None: + if self.findKeywords(['TODO','XXX'], FLAG_NOTICE, self.cKeywordQualifier): continue + else if self.findKeywords(['FIXME'], FLAG_ERROR, self.cKeywordQualifier): continue + else: self.paint(1, FLAG_COMMENT) + return None + +class KurokoHighlighter(State): + + def paintTriple(self, strType): + while self.charat() != None: + if self.charat() == strType: + self.paint(1, FLAG_STRING) + if self.charat() == strType and self.nextchar() == strType: + self.paint(2, FLAG_STRING) + return 0 + else: + self.paint(1, FLAG_STRING) + return int(strType == "'") + 1 + + def paintString(self, strType): + while self.charat() != None: + if self.charat() == '\\' and self.nextchar() == strType: + self.paint(2, FLAG_ESCAPE) + else if self.charat() == strType: + self.paint(1, FLAG_STRING) + return 0 + else if self.charat() == '\\': + if self.nextchar() == 'x': + self.paint(4, FLAG_ESCAPE) + else if self.nextchar() == 'u': + self.paint(6, FLAG_ESCAPE) + else if self.nextchar() == None: + self.paint(1, FLAG_ESCAPE) + return int(strType == "'") + 3 + else: + self.paint(2, FLAG_ESCAPE) + else: + self.paint(1, FLAG_STRING) + return 0 + + def paintNumeral(self): + if self.charat() == '0' and (self.nextchar() == 'x' or self.nextchar() == 'X'): + self.paint(2, FLAG_NUMERAL) + while self.isxdigit(self.charat()): + self.paint(1, FLAG_NUMERAL) + else if self.charat() == '0' and (self.nextchar() == 'o' or self.nextchar() == 'O'): + self.paint(2, FLAG_NUMERAL) + while self.charat() in '01234567': + self.flag(1, FLAG_NUMERAL) + else if self.charat() == '0' and (self.nextchar() == 'b' or self.nextchar() == 'B'): + self.paint(2, FLAG_NUMERAL) + while self.charat() == '0' or self.charat() == '1': + self.paint(1, FLAG_NUMERAL) + else: + while self.isdigit(self.charat()): + self.paint(1, FLAG_NUMERAL) + if self.charat() == '.' and self.isdigit(self.nextchar()): + self.paint(1, FLAG_NUMERAL) + while self.isdigit(self.charat()): + self.paint(1, FLAG_NUMERAL) + return 0 + + def highlightOnce(self): + if self.state <= 0: + if self.charat() == '#': + self.paintComment() + else if self.charat() == '@': + self.paint(1, FLAG_TYPE) + while self.cKeywordQualifier(self.charat()): + self.paint(1, FLAG_TYPE) + return 0 + else if self.charat() == '"' or self.charat() == "'": + let strType = self.charat() + if self.nextchar() == strType and self.charrel(2) == strType: + self.paint(3, FLAG_STRING) + return self.paintTriple(strType) + self.paint(1, FLAG_STRING) + return self.paintString(strType) + else if self.findKeywords(keywords, FLAG_KEYWORD, self.cKeywordQualifier): + return 0 + else if self.lastchar() != '.' and self.findKeywords(types, FLAG_TYPE, self.cKeywordQualifier): + return 0 + else if self.findKeywords(special, FLAG_NUMERAL, self.cKeywordQualifier): + return 0 + else if not self.cKeywordQualifier(self.lastchar()) and self.isdigit(self.charat()): + self.paintNumeral() + return 0 + else if self.charat() != None: + self.skip() + return 0 + else if self.state == 1: + return self.paintTriple('"') + else if self.state == 2: + return self.paintTriple("'") + else if self.state == 3: + return self.paintString('"') + else if self.state == 4: + return self.paintString("'") + return None + + diff --git a/test/testSyntaxHighlighter.krk b/test/testSyntaxHighlighter.krk new file mode 100644 index 0000000..6b87b5d --- /dev/null +++ b/test/testSyntaxHighlighter.krk @@ -0,0 +1,13 @@ +# This is a comment. +'''This is a big fat string.''' +from fileio import open +from syntax.highlighter import KurokoHighlighter, toTerminal + +let h +with open('test/testSyntaxHighlighter.krk') as f: + h = KurokoHighlighter(f.read()) + +h.highlight() +print(h.linestates) +print(h.process()) +toTerminal(h.process()) diff --git a/test/testSyntaxHighlighter.krk.expect b/test/testSyntaxHighlighter.krk.expect new file mode 100644 index 0000000..7cc26cd --- /dev/null +++ b/test/testSyntaxHighlighter.krk.expect @@ -0,0 +1,15 @@ +[[[0, 20, 3]], [[0, 31, 2]], [[0, 4, 1], [4, 8, 0], [12, 6, 1], [18, 5, 0]], [[0, 4, 1], [4, 20, 0], [24, 6, 1], [30, 30, 0]], [], [[0, 3, 1], [3, 2, 0]], [[0, 10, 0], [10, 32, 2], [42, 2, 0], [44, 2, 1], [46, 3, 0]], [[0, 35, 0]], [], [[0, 13, 0]], [[0, 5, 4], [5, 14, 0]], [[0, 5, 4], [5, 13, 0]], [[0, 23, 0]]] +[[('# This is a comment.', 3)], [('\'\'\'This is a big fat string.\'\'\'', 2)], [('from', 1), (' fileio ', 0), ('import', 1), (' open', 0)], [('from', 1), (' syntax.highlighter ', 0), ('import', 1), (' KurokoHighlighter, toTerminal', 0)], [], [('let', 1), (' h', 0)], [('with open(', 0), ('\'test/testSyntaxHighlighter.krk\'', 2), (') ', 0), ('as', 1), (' f:', 0)], [(' h = KurokoHighlighter(f.read())', 0)], [], [('h.highlight()', 0)], [('print', 4), ('(h.linestates)', 0)], [('print', 4), ('(h.process())', 0)], [('toTerminal(h.process())', 0)]] +# This is a comment. +'''This is a big fat string.''' +from fileio import open +from syntax.highlighter import KurokoHighlighter, toTerminal + +let h +with open('test/testSyntaxHighlighter.krk') as f: + h = KurokoHighlighter(f.read()) + +h.highlight() +print(h.linestates) +print(h.process()) +toTerminal(h.process())