mirrors
/
stb
mirror of https://github.com/nothings/stb
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

stb_c_lexer: allow including stb_c_lexer.h without defining overrides (all tokens are always defined; token values have changed)

This commit is contained in:
Sean Barrett 2021-07-11 16:37:34 -07:00
parent 0be82e4814
commit fd7807e92d
3 changed files with 77 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
stb_c_lexer.h
View File

@ -42,6 +42,7 @@
// //
// See end of file for license information. // See end of file for license information.
#ifdef STB_C_LEXER_IMPLEMENTATION
#ifndef STB_C_LEXER_DEFINITIONS #ifndef STB_C_LEXER_DEFINITIONS
// to change the default parsing rules, copy the following lines // to change the default parsing rules, copy the following lines
// into your C/C++ file *before* including this, and then replace // into your C/C++ file *before* including this, and then replace
@ -54,7 +55,6 @@
#error "Can only use stb_c_lexer in contexts where the preprocessor symbols 'Y' and 'N' are not defined" #error "Can only use stb_c_lexer in contexts where the preprocessor symbols 'Y' and 'N' are not defined"
#endif #endif
#define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit #define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit
#define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit #define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit
#define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit #define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit
@ -102,7 +102,7 @@
#define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions #define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions
// --END-- // --END--
#endif
#endif #endif
#ifndef INCLUDE_STB_C_LEXER_H #ifndef INCLUDE_STB_C_LEXER_H
@ -172,78 +172,41 @@ extern void stb_c_lexer_get_location(const stb_lexer *lexer, const char *where,
} }
#endif #endif
// Hacky definitions so we can easily #if on them
#define Y(x) 1
#define N(x) 0
// Config variable that influence which lexer tokens get declared need to go here
#if STB_C_LEX_C_DECIMAL_INTS(x) || STB_C_LEX_C_HEX_INTS(x) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x)
#define STB__clex_define_int
#endif
#if (STB_C_LEX_C_ARITHEQ(x) && STB_C_LEX_C_SHIFTS(x)) || STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x)
#define STB__clex_define_shifts
#endif
// Now pick a definition of Y/N that's conducive to
// defining the enum of token names.
#if STB_C_LEX_DEFINE_ALL_TOKEN_NAMES(x) || defined(STB_C_LEXER_SELF_TEST)
#undef N
#define N(a) Y(a)
#else
#undef N
#define N(a)
#endif
#undef Y
#define Y(a) a,
enum enum
{ {
CLEX_eof = 256, CLEX_eof = 256,
CLEX_parse_error, CLEX_parse_error,
CLEX_intlit ,
#ifdef STB__clex_define_int CLEX_floatlit ,
CLEX_intlit, CLEX_id ,
#endif CLEX_dqstring ,
CLEX_sqstring ,
STB_C_LEX_C_DECIMAL_FLOATS( CLEX_floatlit ) CLEX_charlit ,
STB_C_LEX_C_IDENTIFIERS( CLEX_id ) CLEX_eq ,
STB_C_LEX_C_DQ_STRINGS( CLEX_dqstring ) CLEX_noteq ,
STB_C_LEX_C_SQ_STRINGS( CLEX_sqstring ) CLEX_lesseq ,
STB_C_LEX_C_CHARS( CLEX_charlit ) CLEX_greatereq ,
STB_C_LEX_C_COMPARISONS( CLEX_eq ) CLEX_andand ,
STB_C_LEX_C_COMPARISONS( CLEX_noteq ) CLEX_oror ,
STB_C_LEX_C_COMPARISONS( CLEX_lesseq ) CLEX_shl ,
STB_C_LEX_C_COMPARISONS( CLEX_greatereq ) CLEX_shr ,
STB_C_LEX_C_LOGICAL( CLEX_andand ) CLEX_plusplus ,
STB_C_LEX_C_LOGICAL( CLEX_oror ) CLEX_minusminus ,
STB_C_LEX_C_SHIFTS( CLEX_shl ) CLEX_pluseq ,
STB_C_LEX_C_SHIFTS( CLEX_shr ) CLEX_minuseq ,
STB_C_LEX_C_INCREMENTS( CLEX_plusplus ) CLEX_muleq ,
STB_C_LEX_C_INCREMENTS( CLEX_minusminus ) CLEX_diveq ,
STB_C_LEX_C_ARITHEQ( CLEX_pluseq ) CLEX_modeq ,
STB_C_LEX_C_ARITHEQ( CLEX_minuseq ) CLEX_andeq ,
STB_C_LEX_C_ARITHEQ( CLEX_muleq ) CLEX_oreq ,
STB_C_LEX_C_ARITHEQ( CLEX_diveq ) CLEX_xoreq ,
STB_C_LEX_C_ARITHEQ( CLEX_modeq ) CLEX_arrow ,
STB_C_LEX_C_BITWISEEQ( CLEX_andeq ) CLEX_eqarrow ,
STB_C_LEX_C_BITWISEEQ( CLEX_oreq )
STB_C_LEX_C_BITWISEEQ( CLEX_xoreq )
STB_C_LEX_C_ARROW( CLEX_arrow )
STB_C_LEX_EQUAL_ARROW( CLEX_eqarrow )
#ifdef STB__clex_define_shifts
CLEX_shleq, CLEX_shreq, CLEX_shleq, CLEX_shreq,
#endif
CLEX_first_unused_token CLEX_first_unused_token
}; };
#undef Y
#undef N
#endif // INCLUDE_STB_C_LEXER_H #endif // INCLUDE_STB_C_LEXER_H
#ifdef STB_C_LEXER_IMPLEMENTATION #ifdef STB_C_LEXER_IMPLEMENTATION

49
tests/test_c_lexer.c
View File

@ -1 +1,50 @@
#include "stb_c_lexer.h" #include "stb_c_lexer.h"
#define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit
#define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit
#define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit
#define STB_C_LEX_C_DECIMAL_FLOATS Y // "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?) CLEX_floatlit
#define STB_C_LEX_C99_HEX_FLOATS N // "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+ CLEX_floatlit
#define STB_C_LEX_C_IDENTIFIERS Y // "[_a-zA-Z][_a-zA-Z0-9]*" CLEX_id
#define STB_C_LEX_C_DQ_STRINGS Y // double-quote-delimited strings with escapes CLEX_dqstring
#define STB_C_LEX_C_SQ_STRINGS N // single-quote-delimited strings with escapes CLEX_ssstring
#define STB_C_LEX_C_CHARS Y // single-quote-delimited character with escape CLEX_charlits
#define STB_C_LEX_C_COMMENTS Y // "/* comment */"
#define STB_C_LEX_CPP_COMMENTS Y // "// comment to end of line\n"
#define STB_C_LEX_C_COMPARISONS Y // "==" CLEX_eq "!=" CLEX_noteq "<=" CLEX_lesseq ">=" CLEX_greatereq
#define STB_C_LEX_C_LOGICAL Y // "&&" CLEX_andand "||" CLEX_oror
#define STB_C_LEX_C_SHIFTS Y // "<<" CLEX_shl ">>" CLEX_shr
#define STB_C_LEX_C_INCREMENTS Y // "++" CLEX_plusplus "--" CLEX_minusminus
#define STB_C_LEX_C_ARROW Y // "->" CLEX_arrow
#define STB_C_LEX_EQUAL_ARROW N // "=>" CLEX_eqarrow
#define STB_C_LEX_C_BITWISEEQ Y // "&=" CLEX_andeq "|=" CLEX_oreq "^=" CLEX_xoreq
#define STB_C_LEX_C_ARITHEQ Y // "+=" CLEX_pluseq "-=" CLEX_minuseq
// "*=" CLEX_muleq "/=" CLEX_diveq "%=" CLEX_modeq
// if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ:
// "<<=" CLEX_shleq ">>=" CLEX_shreq
#define STB_C_LEX_PARSE_SUFFIXES N // letters after numbers are parsed as part of those numbers, and must be in suffix list below
#define STB_C_LEX_DECIMAL_SUFFIXES "" // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage
#define STB_C_LEX_HEX_SUFFIXES "" // e.g. "uUlL"
#define STB_C_LEX_OCTAL_SUFFIXES "" // e.g. "uUlL"
#define STB_C_LEX_FLOAT_SUFFIXES "" //
#define STB_C_LEX_0_IS_EOF Y // if Y, ends parsing at '\0'; if N, returns '\0' as token
#define STB_C_LEX_INTEGERS_AS_DOUBLES N // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N
#define STB_C_LEX_MULTILINE_DSTRINGS N // allow newlines in double-quoted strings
#define STB_C_LEX_MULTILINE_SSTRINGS N // allow newlines in single-quoted strings
#define STB_C_LEX_USE_STDLIB Y // use strtod,strtol for parsing #s; otherwise inaccurate hack
#define STB_C_LEX_DOLLAR_IDENTIFIER Y // allow $ as an identifier character
#define STB_C_LEX_FLOAT_NO_DECIMAL Y // allow floats that have no decimal point if they have an exponent
#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES N // if Y, all CLEX_ token names are defined, even if never returned
// leaving it as N should help you catch config bugs
#define STB_C_LEX_DISCARD_PREPROCESSOR Y // discard C-preprocessor directives (e.g. after prepocess
// still have #line, #pragma, etc)
//#define STB_C_LEX_ISWHITE(str) ... // return length in bytes of whitespace characters if first char is whitespace
#define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions
#include "stb_c_lexer.h"

49
tests/test_cpp_compilation.cpp
View File

@ -183,53 +183,4 @@ void dummy3(void)
stb_textedit_paste(0,0,0,0); stb_textedit_paste(0,0,0,0);
} }
#if 0
#define STB_C_LEX_C_DECIMAL_INTS Y // "0|[1-9][0-9]*" CLEX_intlit
#define STB_C_LEX_C_HEX_INTS Y // "0x[0-9a-fA-F]+" CLEX_intlit
#define STB_C_LEX_C_OCTAL_INTS Y // "[0-7]+" CLEX_intlit
#define STB_C_LEX_C_DECIMAL_FLOATS Y // "[0-9]*(.[0-9]*([eE][-+]?[0-9]+)?) CLEX_floatlit
#define STB_C_LEX_C99_HEX_FLOATS N // "0x{hex}+(.{hex}*)?[pP][-+]?{hex}+ CLEX_floatlit
#define STB_C_LEX_C_IDENTIFIERS Y // "[_a-zA-Z][_a-zA-Z0-9]*" CLEX_id
#define STB_C_LEX_C_DQ_STRINGS Y // double-quote-delimited strings with escapes CLEX_dqstring
#define STB_C_LEX_C_SQ_STRINGS N // single-quote-delimited strings with escapes CLEX_ssstring
#define STB_C_LEX_C_CHARS Y // single-quote-delimited character with escape CLEX_charlits
#define STB_C_LEX_C_COMMENTS Y // "/* comment */"
#define STB_C_LEX_CPP_COMMENTS Y // "// comment to end of line\n"
#define STB_C_LEX_C_COMPARISONS Y // "==" CLEX_eq "!=" CLEX_noteq "<=" CLEX_lesseq ">=" CLEX_greatereq
#define STB_C_LEX_C_LOGICAL Y // "&&" CLEX_andand "||" CLEX_oror
#define STB_C_LEX_C_SHIFTS Y // "<<" CLEX_shl ">>" CLEX_shr
#define STB_C_LEX_C_INCREMENTS Y // "++" CLEX_plusplus "--" CLEX_minusminus
#define STB_C_LEX_C_ARROW Y // "->" CLEX_arrow
#define STB_C_LEX_EQUAL_ARROW N // "=>" CLEX_eqarrow
#define STB_C_LEX_C_BITWISEEQ Y // "&=" CLEX_andeq "|=" CLEX_oreq "^=" CLEX_xoreq
#define STB_C_LEX_C_ARITHEQ Y // "+=" CLEX_pluseq "-=" CLEX_minuseq
// "*=" CLEX_muleq "/=" CLEX_diveq "%=" CLEX_modeq
// if both STB_C_LEX_SHIFTS & STB_C_LEX_ARITHEQ:
// "<<=" CLEX_shleq ">>=" CLEX_shreq
#define STB_C_LEX_PARSE_SUFFIXES N // letters after numbers are parsed as part of those numbers, and must be in suffix list below
#define STB_C_LEX_DECIMAL_SUFFIXES "" // decimal integer suffixes e.g. "uUlL" -- these are returned as-is in string storage
#define STB_C_LEX_HEX_SUFFIXES "" // e.g. "uUlL"
#define STB_C_LEX_OCTAL_SUFFIXES "" // e.g. "uUlL"
#define STB_C_LEX_FLOAT_SUFFIXES "" //
#define STB_C_LEX_0_IS_EOF Y // if Y, ends parsing at '\0'; if N, returns '\0' as token
#define STB_C_LEX_INTEGERS_AS_DOUBLES N // parses integers as doubles so they can be larger than 'int', but only if STB_C_LEX_STDLIB==N
#define STB_C_LEX_MULTILINE_DSTRINGS N // allow newlines in double-quoted strings
#define STB_C_LEX_MULTILINE_SSTRINGS N // allow newlines in single-quoted strings
#define STB_C_LEX_USE_STDLIB Y // use strtod,strtol for parsing #s; otherwise inaccurate hack
#define STB_C_LEX_DOLLAR_IDENTIFIER Y // allow $ as an identifier character
#define STB_C_LEX_FLOAT_NO_DECIMAL Y // allow floats that have no decimal point if they have an exponent
#define STB_C_LEX_DEFINE_ALL_TOKEN_NAMES N // if Y, all CLEX_ token names are defined, even if never returned
// leaving it as N should help you catch config bugs
#define STB_C_LEX_DISCARD_PREPROCESSOR Y // discard C-preprocessor directives (e.g. after prepocess
// still have #line, #pragma, etc)
//#define STB_C_LEX_ISWHITE(str) ... // return length in bytes of whitespace characters if first char is whitespace
#define STB_C_LEXER_DEFINITIONS // This line prevents the header file from replacing your definitions
#endif
#include "stb_c_lexer.h" #include "stb_c_lexer.h"