Aren/haiku
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added "position in original character stream" marker to Token class

Browse Source 
git-svn-id: file:///srv/svn/repos/haiku/trunk/current@490 a95241bf-73f2-0310-859d-f6bbb57e9c96
This commit is contained in:
Tyler Dauwalder 2002-07-28 04:13:30 +00:00
parent dc685e9ad8
commit 224e92720e
3 changed files with 64 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
headers/private/storage/sniffer/Parser.h
View File

@ -51,6 +51,7 @@ public:
void Unset(); void Unset();
status_t InitCheck() const; status_t InitCheck() const;
bool IsEmpty() const; bool IsEmpty() const;
size_t Pos() const;
char Get(); char Get();
void Unget(); void Unget();
@ -83,7 +84,7 @@ const char* tokenTypeToString(TokenType type);
class Token { class Token {
public: public:
Token(TokenType type = EmptyToken); Token(TokenType type = EmptyToken, const size_t posInStream = -1);
virtual ~Token(); virtual ~Token();
TokenType Type() const; TokenType Type() const;
virtual const char* String() const; virtual const char* String() const;
@ -92,11 +93,12 @@ public:
bool operator==(Token &ref); bool operator==(Token &ref);
protected: protected:
TokenType fType; TokenType fType;
size_t fPosInStream;
}; };
class StringToken : public Token { class StringToken : public Token {
public: public:
StringToken(const char *string); StringToken(const char *string, const size_t posInStream);
virtual ~StringToken(); virtual ~StringToken();
virtual const char* String() const; virtual const char* String() const;
protected: protected:
@ -105,7 +107,7 @@ protected:
class IntToken : public Token { class IntToken : public Token {
public: public:
IntToken(const int32 value); IntToken(const int32 value, const size_t posInStream);
virtual int32 Int() const; virtual int32 Int() const;
virtual double Float() const; virtual double Float() const;
protected: protected:
@ -114,7 +116,7 @@ protected:
class FloatToken : public Token { class FloatToken : public Token {
public: public:
FloatToken(const double value); FloatToken(const double value, const size_t posInStream);
virtual double Float() const; virtual double Float() const;
protected: protected:
double fValue; double fValue;
@ -135,10 +137,10 @@ public:
bool IsEmpty(); bool IsEmpty();
private: private:
void AddToken(TokenType type); void AddToken(TokenType type, size_t posInStream);
void AddString(const char *str); void AddString(const char *str, size_t posInStream);
void AddInt(const char *str); void AddInt(const char *str, size_t posInStream);
void AddFloat(const char *str); void AddFloat(const char *str, size_t posInStream);
BList fTokenList; BList fTokenList;
status_t fCStatus; status_t fCStatus;

85
src/kits/storage/sniffer/Parser.cpp
View File

@ -27,8 +27,8 @@ using namespace Sniffer;
TokenStream stream; TokenStream stream;
// Private parsing functions // Private parsing functions
/*
double parsePriority(); double parsePriority();
/*
BList* parseExprList(); BList* parseExprList();
Expr* parseExpr(); Expr* parseExpr();
Range parseRange(); Range parseRange();
@ -63,7 +63,7 @@ Sniffer::parse(const char *rule, Rule *result, BString *parseError = NULL) {
double priority; double priority;
BList* exprList; BList* exprList;
// priority = parsePriority(); priority = parsePriority();
} catch (Err *err) { } catch (Err *err) {
if (parseError && err) if (parseError && err)
@ -168,6 +168,11 @@ CharStream::IsEmpty() const {
return fPos >= fLen; return fPos >= fLen;
} }
size_t
CharStream::Pos() const {
return fPos;
}
char char
CharStream::Get() { CharStream::Get() {
if (fCStatus != B_OK) if (fCStatus != B_OK)
@ -194,8 +199,9 @@ CharStream::Unget() {
// Token // Token
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
Token::Token(TokenType type) Token::Token(TokenType type, const size_t posInStream)
: fType(type) : fType(type)
, fPosInStream(posInStream)
{ {
// if (type != EmptyToken) // if (type != EmptyToken)
// cout << "New Token, fType == " << tokenTypeToString(fType) << endl; // cout << "New Token, fType == " << tokenTypeToString(fType) << endl;
@ -272,8 +278,8 @@ Token::operator==(Token &ref) {
// StringToken // StringToken
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
StringToken::StringToken(const char *string) StringToken::StringToken(const char *string, const size_t posInStream)
: Token(CharacterString) : Token(CharacterString, posInStream)
, fString(NULL) , fString(NULL)
{ {
if (string) { if (string) {
@ -296,8 +302,8 @@ StringToken::String() const {
// IntToken // IntToken
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
IntToken::IntToken(const int32 value) IntToken::IntToken(const int32 value, const size_t posInStream)
: Token(Integer) : Token(Integer, posInStream)
, fValue(value) , fValue(value)
{ {
} }
@ -316,8 +322,8 @@ IntToken::Float() const {
// FloatToken // FloatToken
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
FloatToken::FloatToken(const double value) FloatToken::FloatToken(const double value, const size_t posInStream)
: Token(FloatingPoint) : Token(FloatingPoint, posInStream)
, fValue(value) , fValue(value)
{ {
} }
@ -381,6 +387,7 @@ TokenStream::SetTo(const char *string) {
char lastLastChar; // For three char lookahead char lastLastChar; // For three char lookahead
bool keepLooping = true; bool keepLooping = true;
while (keepLooping) { while (keepLooping) {
size_t pos = stream.Pos();
char ch = stream.Get(); char ch = stream.Get();
switch (state) { switch (state) {
case tsssStart: case tsssStart:
@ -437,11 +444,11 @@ TokenStream::SetTo(const char *string) {
state = tsssIntOrFloat; state = tsssIntOrFloat;
break; break;
case '&': AddToken(Ampersand); break; case '&': AddToken(Ampersand, pos); break;
case '(': AddToken(LeftParen); break; case '(': AddToken(LeftParen, pos); break;
case ')': AddToken(RightParen); break; case ')': AddToken(RightParen, pos); break;
case ':': AddToken(Colon); break; case ':': AddToken(Colon, pos); break;
case '[': AddToken(LeftBracket); break; case '[': AddToken(LeftBracket, pos); break;
case '\\': case '\\':
charStr = ""; // Clear our string charStr = ""; // Clear our string
@ -449,8 +456,8 @@ TokenStream::SetTo(const char *string) {
escapedState = tsssUnquoted; // Unquoted strings begin with an escaped character escapedState = tsssUnquoted; // Unquoted strings begin with an escaped character
break; break;
case ']': AddToken(RightBracket); break; case ']': AddToken(RightBracket, pos); break;
case '|': AddToken(Divider); break; case '|': AddToken(Divider, pos); break;
default: default:
throw new Err(std::string("Sniffer scanner error: unexpected character '") + ch + "'"); throw new Err(std::string("Sniffer scanner error: unexpected character '") + ch + "'");
@ -464,7 +471,7 @@ TokenStream::SetTo(const char *string) {
state = tsssEscape; // Handle the escape sequence state = tsssEscape; // Handle the escape sequence
break; break;
case '\'': case '\'':
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
state = tsssStart; state = tsssStart;
break; break;
case 0x3: case 0x3:
@ -486,7 +493,7 @@ TokenStream::SetTo(const char *string) {
state = tsssEscape; // Handle the escape sequence state = tsssEscape; // Handle the escape sequence
break; break;
case '"': case '"':
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
state = tsssStart; state = tsssStart;
break; break;
case 0x3: case 0x3:
@ -513,11 +520,11 @@ TokenStream::SetTo(const char *string) {
state = tsssFloat; state = tsssFloat;
} else if (ch == 0x3 && stream.IsEmpty()) { } else if (ch == 0x3 && stream.IsEmpty()) {
// Terminate the number and then the loop // Terminate the number and then the loop
AddInt(charStr.c_str()); AddInt(charStr.c_str(), pos);
keepLooping = false; keepLooping = false;
} else { } else {
// Terminate the number // Terminate the number
AddInt(charStr.c_str()); AddInt(charStr.c_str(), pos);
// Push the last char back on and try again // Push the last char back on and try again
stream.Unget(); stream.Unget();
@ -550,11 +557,11 @@ TokenStream::SetTo(const char *string) {
lastChar = ch; lastChar = ch;
state = tsssOneHex; state = tsssOneHex;
} else if (isWhiteSpace(ch) || isPunctuation(ch)) { } else if (isWhiteSpace(ch) || isPunctuation(ch)) {
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
stream.Unget(); // So punctuation gets handled properly stream.Unget(); // So punctuation gets handled properly
state = tsssStart; state = tsssStart;
} else if (ch == 0x3 && stream.IsEmpty()) { } else if (ch == 0x3 && stream.IsEmpty()) {
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
keepLooping = false; keepLooping = false;
} else } else
throw Err(std::string("Sniffer scanner error: unexpected character '") + ch + "'"); throw Err(std::string("Sniffer scanner error: unexpected character '") + ch + "'");
@ -568,7 +575,7 @@ TokenStream::SetTo(const char *string) {
state = tsssFloat; state = tsssFloat;
} else { } else {
// Terminate the number // Terminate the number
AddInt(charStr.c_str()); AddInt(charStr.c_str(), pos);
// Push the last char back on and try again // Push the last char back on and try again
stream.Unget(); stream.Unget();
@ -581,7 +588,7 @@ TokenStream::SetTo(const char *string) {
charStr += ch; charStr += ch;
else { else {
// Terminate the number // Terminate the number
AddFloat(charStr.c_str()); AddFloat(charStr.c_str(), pos);
// Push the last char back on and try again // Push the last char back on and try again
stream.Unget(); stream.Unget();
@ -616,7 +623,7 @@ TokenStream::SetTo(const char *string) {
throw new Err(std::string("Sniffer scanner error: negative floating point numbers are useless and thus signs (both + and -) are disallowed on floating points")); throw new Err(std::string("Sniffer scanner error: negative floating point numbers are useless and thus signs (both + and -) are disallowed on floating points"));
else { else {
// Terminate the number // Terminate the number
AddInt(charStr.c_str()); AddInt(charStr.c_str(), pos);
// Push the last char back on and try again // Push the last char back on and try again
stream.Unget(); stream.Unget();
@ -629,13 +636,13 @@ TokenStream::SetTo(const char *string) {
escapedState = state; // Save our state escapedState = state; // Save our state
state = tsssEscape; // Handle the escape sequence state = tsssEscape; // Handle the escape sequence
} else if (isWhiteSpace(ch) || isPunctuation(ch)) { } else if (isWhiteSpace(ch) || isPunctuation(ch)) {
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
stream.Unget(); // In case it's punctuation, let tsssStart handle it stream.Unget(); // In case it's punctuation, let tsssStart handle it
state = tsssStart; state = tsssStart;
} else if (ch == '\'' || ch == '"') { } else if (ch == '\'' || ch == '"') {
throw new Err(std::string("Sniffer scanner error: illegal unquoted character '") + ch + "'"); throw new Err(std::string("Sniffer scanner error: illegal unquoted character '") + ch + "'");
} else if (ch == 0x3 && stream.IsEmpty()) { } else if (ch == 0x3 && stream.IsEmpty()) {
AddString(charStr.c_str()); AddString(charStr.c_str(), pos);
keepLooping = false; keepLooping = false;
} else { } else {
charStr += ch; charStr += ch;
@ -745,30 +752,30 @@ TokenStream::IsEmpty() {
} }
void void
TokenStream::AddToken(TokenType type) { TokenStream::AddToken(TokenType type, size_t posInStream) {
Token *token = new Token(type); Token *token = new Token(type, posInStream);
fTokenList.AddItem(token); fTokenList.AddItem(token);
} }
void void
TokenStream::AddString(const char *str) { TokenStream::AddString(const char *str, size_t posInStream) {
Token *token = new StringToken(str); Token *token = new StringToken(str, posInStream);
fTokenList.AddItem(token); fTokenList.AddItem(token);
} }
void void
TokenStream::AddInt(const char *str) { TokenStream::AddInt(const char *str, size_t posInStream) {
// Convert the string to an int // Convert the string to an int
int32 value = atol(str); int32 value = atol(str);
Token *token = new IntToken(value); Token *token = new IntToken(value, posInStream);
fTokenList.AddItem(token); fTokenList.AddItem(token);
} }
void void
TokenStream::AddFloat(const char *str) { TokenStream::AddFloat(const char *str, size_t posInStream) {
// Convert the string to a float // Convert the string to a float
double value = atof(str); double value = atof(str);
Token *token = new FloatToken(value); Token *token = new FloatToken(value, posInStream);
fTokenList.AddItem(token); fTokenList.AddItem(token);
} }
@ -917,3 +924,11 @@ Sniffer::tokenTypeToString(TokenType type) {
} }
} }
//------------------------------------------------------------------------------
// Parsing functions
//------------------------------------------------------------------------------
double
parsePriority() {
// stream.Read(LeftBracket);
}

8
src/tests/kits/storage/ParserTest.cpp
View File

@ -31,10 +31,10 @@ ParserTest::ScannerTest() {
// Define some useful macros for dynamically allocating // Define some useful macros for dynamically allocating
// various Token classes // various Token classes
#define T(type) (new Token(type)) #define T(type) (new Token(type, -1))
#define S(str) (new StringToken(str)) #define S(str) (new StringToken(str, -1))
#define I(val) (new IntToken(val)) #define I(val) (new IntToken(val, -1))
#define F(val) (new FloatToken(val)) #define F(val) (new FloatToken(val, -1))
struct test_case { struct test_case {
const char *rule; const char *rule;