Changed token type handling so that different chars can resolve to the same

token type (part of #3236). "/" now equals "\" and ":" to mean "divide by".


git-svn-id: file:///srv/svn/repos/haiku/haiku/trunk@28921 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Stephan Aßmus 2009-01-17 14:19:36 +00:00
parent b93465f1e0
commit 85d74ff62d

View File

@ -19,31 +19,30 @@
static const int32 kMaxDigits = 64; static const int32 kMaxDigits = 64;
enum { enum {
TOKEN_IDENTIFIER = 'a', TOKEN_IDENTIFIER = 0,
TOKEN_CONSTANT = '0', TOKEN_CONSTANT,
TOKEN_PLUS = '+', TOKEN_PLUS,
TOKEN_MINUS = '-', TOKEN_MINUS,
TOKEN_STAR = '*', TOKEN_STAR,
TOKEN_SLASH = '/', TOKEN_SLASH,
TOKEN_MODULO = '%', TOKEN_MODULO,
TOKEN_POWER = '^', TOKEN_POWER,
TOKEN_OPENING_BRACKET = '(', TOKEN_OPENING_BRACKET,
TOKEN_CLOSING_BRACKET = ')', TOKEN_CLOSING_BRACKET,
TOKEN_AND = '&', TOKEN_AND,
TOKEN_OR = '|', TOKEN_OR,
TOKEN_NOT = '~', TOKEN_NOT,
TOKEN_NONE = ' ', TOKEN_NONE,
TOKEN_END_OF_LINE = '\n', TOKEN_END_OF_LINE
}; };
struct Token { struct Token {
Token() Token()
: string(""), : string(""),
@ -168,29 +167,57 @@ class Tokenizer {
} else { } else {
int32 type = TOKEN_NONE;
switch (*fCurrentChar) { switch (*fCurrentChar) {
case '+': case '+':
type = TOKEN_PLUS;
break;
case '-': case '-':
type = TOKEN_MINUS;
break;
case '*': case '*':
type = TOKEN_STAR;
break;
case '/': case '/':
case '\\':
case ':':
type = TOKEN_SLASH;
break;
case '^':
case '%': case '%':
type = TOKEN_MODULO;
break;
case '^':
type = TOKEN_POWER;
break;
case '(': case '(':
type = TOKEN_OPENING_BRACKET;
break;
case ')': case ')':
type = TOKEN_CLOSING_BRACKET;
break;
case '&': case '&':
type = TOKEN_AND;
break;
case '|': case '|':
type = TOKEN_OR;
break;
case '~': case '~':
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type = TOKEN_NOT;
*fCurrentChar); break;
fCurrentChar++;
case '\n':
type = TOKEN_END_OF_LINE;
break; break;
default: default:
throw ParseException("unexpected character", _CurrentPos()); throw ParseException("unexpected character", _CurrentPos());
} }
fCurrentToken = Token(fCurrentChar, 1, _CurrentPos(), type);
fCurrentChar++;
} }
//printf("next token: '%s'\n", fCurrentToken.string.String()); //printf("next token: '%s'\n", fCurrentToken.string.String());