Add the %extra_context directive to lemon, as an alternative to %extra_argument.

Use this to improve the performance of the parser.

FossilOrigin-Name: be47a6f5262a43f477700579512fe7112a0872faedcbbe5c3383d13a08af6440
This commit is contained in:
drh 2018-04-21 13:51:42 +00:00
parent f3d7bbb767
commit fb32c44e41
8 changed files with 125 additions and 56 deletions

View File

@ -99,6 +99,9 @@ Show only the basis for each parser state in the report file.
<li><b>-c</b>
Do not compress the generated action tables. The parser will be a
little larger and slower, but it will detect syntax errors sooner.
<li><b>-d</b><i>directory</i>
Write all output files into <i>directory</i>. Normally, output files
are written into the directory that contains the input grammar file.
<li><b>-D<i>name</i></b>
Define C preprocessor macro <i>name</i>. This macro is usable by
"<tt><a href='#pifdef'>%ifdef</a></tt>" and
@ -679,6 +682,30 @@ of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of the 4th parameter
in the most recent call to Parse().</p>
<p>The <tt>%extra_context</tt> directive works the same except that it
is passed in on the ParseAlloc() or ParseInit() routines instead of
on Parse().
<a name='extractx'></a>
<h4>The <tt>%extra_context</tt> directive</h4>
The <tt>%extra_context</tt> directive instructs Lemon to add a 2th parameter
to the parameter list of the ParseAlloc() and ParseInif() functions. Lemon
doesn't do anything itself with these extra argument, but it does
store the value make it available to C-code action routines, destructors,
and so forth. For example, if the grammar file contains:</p>
<p><pre>
%extra_context { MyStruct *pAbc }
</pre></p>
<p>Then the ParseAlloc() and ParseInit() functions will have an 2th parameter
of type "MyStruct*" and all action routines will have access to
a variable named "pAbc" that is the value of that 2th parameter.</p>
<p>The <tt>%extra_argument</tt> directive works the same except that it
is passed in on the Parse() routine instead of on ParseAlloc()/ParseInit().
<a name='pfallback'></a>
<h4>The <tt>%fallback</tt> directive</h4>

View File

@ -1,5 +1,5 @@
C Fix\sUPSERT\sso\sthat\sit\splays\snicely\swith\sAUTOINCREMENT.
D 2018-04-21T03:06:29.663
C Add\sthe\s%extra_context\sdirective\sto\slemon,\sas\san\salternative\sto\s%extra_argument.\nUse\sthis\sto\simprove\sthe\sperformance\sof\sthe\sparser.
D 2018-04-21T13:51:42.935
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439
@ -35,7 +35,7 @@ F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55
F configure 80e2dcad8ab88aacc58b55eb0e395f79184b45fcfaa3f36fc20d2e71cfa0a7e4 x
F configure.ac d4529ebb26ae046269334f1dac65f2b1d6927c2efe22b2ec24dce24dfe4f83dd
F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
F doc/lemon.html 278113807f49d12d04179a93fab92b5b917a08771152ca7949d34e928efa3941
F doc/lemon.html ac63db056bce24b7368e29319cd1a7eb5f1798cc85922d96a80b6c3a4ff9f51b
F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710
F doc/vfs-shm.txt e101f27ea02a8387ce46a05be2b1a902a021d37a
F ext/README.md fd5f78013b0a2bc6f0067afb19e6ad040e89a10179b4f6f03eee58fac5f169bd
@ -481,7 +481,7 @@ F src/os_win.c eb03c6d52f893bcd7fdd4c6006674c13c1b5e49543fec98d605201af2997171c
F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f
F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388
F src/parse.y 1095d04ac7bc6d2cc69167daf6969c1276970c5d14578e259561d2a5477312f3
F src/parse.y 07784439d25f0bc64a656eece4caecc549b147d213f513cdbeb8430345ec2911
F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd
F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@ -497,7 +497,7 @@ F src/shell.c.in 8ab4687da814ddc4adf6ea0fcd43ea1eb2784ee6915674dd690759241b7a24b
F src/sqlite.h.in aa9bd3ae4a077c7002059cb418271abe52214b0227b2a734bc44736b24cbcc40
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
F src/sqliteInt.h 6f26888c81854b1cc8c9a285d75411636a7c6444227e1953a39b2c52323441d5
F src/sqliteInt.h e62bf3794b21b073ebeb484833b451f4d0e11abb0cf263c7371ac8375dc68523
F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@ -554,7 +554,7 @@ F src/test_windirent.c a895e2c068a06644eef91a7f0a32182445a893b9a0f33d0cdb4283dca
F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a90484215
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
F src/tokenize.c 5b0c661a85f783d35b9883830736eeb63be4aefc4f6b7d9cd081d48782c041e2
F src/tokenize.c bbde32eac9eb1280f5292bcdfef66f5a57e43176cbf9347e0efab9f75e133f97
F src/treeview.c 14d5d1254702ec96876aa52642cb31548612384134970409fae333b25b39d6bb
F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995
F src/update.c 0d53281948be1c7f7242151fe9adcdcb02eb9faeb1ee4c98cffd67c12adc3599
@ -1643,8 +1643,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
F tool/lemon.c 56eb42d92187547c58646482a49686e114eb4c0e8363820c70b5134bd6bb954f
F tool/lempar.c 468a155e8729cfbccfe1d85bf60d064f1dab76167a51149ec5c7928a2de63953
F tool/lemon.c 735516a0aac3aeebc548b5db2acdda7a518e8d7f642b8debb0f46b01cd6ce5af
F tool/lempar.c 8ce83cbec62cba95819760de2ebab98a1b0d00861af5876bd456a29e1158229d
F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@ -1724,7 +1724,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 9cd20475ff3b2ca1a58e441194c921780d25bdb9b9c744a6b4541b888194efb8
R 1a3926e888b44fb26b041e4405035512
P 359725ab36339b443b7745e84f6d27991038ceb063c653805dde17f3eb5a03c5
R f18739f01eafa1a7e4d368678c4dae1f
U drh
Z a5448d6a099bd200ea71d047714044d6
Z 626b0434bcf85f73e88ad9b2ab679a11

View File

@ -1 +1 @@
359725ab36339b443b7745e84f6d27991038ceb063c653805dde17f3eb5a03c5
be47a6f5262a43f477700579512fe7112a0872faedcbbe5c3383d13a08af6440

View File

@ -24,8 +24,9 @@
%token_type {Token}
%default_type {Token}
// The generated parser function takes a 4th argument as follows:
%extra_argument {Parse *pParse}
// An extra argument to the constructor for the parser, which is available
// to all actions.
%extra_context {Parse *pParse}
// This code runs whenever there is a syntax error
//

View File

@ -4213,10 +4213,10 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
** The interface to the LEMON-generated parser
*/
#ifndef SQLITE_AMALGAMATION
void *sqlite3ParserAlloc(void*(*)(u64));
void *sqlite3ParserAlloc(void*(*)(u64), Parse*);
void sqlite3ParserFree(void*, void(*)(void*));
#endif
void sqlite3Parser(void*, int, Token, Parse*);
void sqlite3Parser(void*, int, Token);
#ifdef YYTRACKMAXSTACKDEPTH
int sqlite3ParserStackPeak(void*);
#endif

View File

@ -496,9 +496,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
/* sqlite3ParserTrace(stdout, "parser: "); */
#ifdef sqlite3Parser_ENGINEALWAYSONSTACK
pEngine = &sEngine;
sqlite3ParserInit(pEngine);
sqlite3ParserInit(pEngine, pParse);
#else
pEngine = sqlite3ParserAlloc(sqlite3Malloc);
pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse);
if( pEngine==0 ){
sqlite3OomFault(db);
return SQLITE_NOMEM_BKPT;
@ -542,7 +542,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
}else{
pParse->sLastToken.z = zSql;
pParse->sLastToken.n = n;
sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
lastTokenParsed = tokenType;
zSql += n;
if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;

View File

@ -396,6 +396,7 @@ struct lemon {
struct symbol *wildcard; /* Token that matches anything */
char *name; /* Name of the generated parser */
char *arg; /* Declaration of the 3th argument to parser */
char *ctx; /* Declaration of 2nd argument to constructor */
char *tokentype; /* Type of terminal symbols in the parser stack */
char *vartype; /* The default type of non-terminal symbols */
char *start; /* Name of the start symbol for the grammar */
@ -2493,6 +2494,9 @@ to follow the previous rule.");
}else if( strcmp(x,"extra_argument")==0 ){
psp->declargslot = &(psp->gp->arg);
psp->insertLineMacro = 0;
}else if( strcmp(x,"extra_context")==0 ){
psp->declargslot = &(psp->gp->ctx);
psp->insertLineMacro = 0;
}else if( strcmp(x,"token_type")==0 ){
psp->declargslot = &(psp->gp->tokentype);
psp->insertLineMacro = 0;
@ -4198,16 +4202,36 @@ void ReportTable(
while( i>=1 && (ISALNUM(lemp->arg[i-1]) || lemp->arg[i-1]=='_') ) i--;
fprintf(out,"#define %sARG_SDECL %s;\n",name,lemp->arg); lineno++;
fprintf(out,"#define %sARG_PDECL ,%s\n",name,lemp->arg); lineno++;
fprintf(out,"#define %sARG_FETCH %s = yypParser->%s\n",
fprintf(out,"#define %sARG_PARAM ,%s\n",name,&lemp->arg[i]); lineno++;
fprintf(out,"#define %sARG_FETCH %s=yypParser->%s;\n",
name,lemp->arg,&lemp->arg[i]); lineno++;
fprintf(out,"#define %sARG_STORE yypParser->%s = %s\n",
fprintf(out,"#define %sARG_STORE yypParser->%s=%s;\n",
name,&lemp->arg[i],&lemp->arg[i]); lineno++;
}else{
fprintf(out,"#define %sARG_SDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PDECL\n",name); lineno++;
fprintf(out,"#define %sARG_SDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PDECL\n",name); lineno++;
fprintf(out,"#define %sARG_PARAM\n",name); lineno++;
fprintf(out,"#define %sARG_FETCH\n",name); lineno++;
fprintf(out,"#define %sARG_STORE\n",name); lineno++;
}
if( lemp->ctx && lemp->ctx[0] ){
i = lemonStrlen(lemp->ctx);
while( i>=1 && ISSPACE(lemp->ctx[i-1]) ) i--;
while( i>=1 && (ISALNUM(lemp->ctx[i-1]) || lemp->ctx[i-1]=='_') ) i--;
fprintf(out,"#define %sCTX_SDECL %s;\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PDECL ,%s\n",name,lemp->ctx); lineno++;
fprintf(out,"#define %sCTX_PARAM ,%s\n",name,&lemp->ctx[i]); lineno++;
fprintf(out,"#define %sCTX_FETCH %s=yypParser->%s;\n",
name,lemp->ctx,&lemp->ctx[i]); lineno++;
fprintf(out,"#define %sCTX_STORE yypParser->%s=%s;\n",
name,&lemp->ctx[i],&lemp->ctx[i]); lineno++;
}else{
fprintf(out,"#define %sCTX_SDECL\n",name); lineno++;
fprintf(out,"#define %sCTX_PDECL\n",name); lineno++;
fprintf(out,"#define %sCTX_PARAM\n",name); lineno++;
fprintf(out,"#define %sCTX_FETCH\n",name); lineno++;
fprintf(out,"#define %sCTX_STORE\n",name); lineno++;
}
if( mhflag ){
fprintf(out,"#endif\n"); lineno++;
}

View File

@ -66,8 +66,10 @@
** zero the stack is dynamically sized using realloc()
** ParseARG_SDECL A static variable declaration for the %extra_argument
** ParseARG_PDECL A parameter declaration for the %extra_argument
** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter
** ParseARG_STORE Code to store %extra_argument into yypParser
** ParseARG_FETCH Code to extract %extra_argument from yypParser
** ParseCTX_* As ParseARG_ except for %extra_context
** YYERRORSYMBOL is the code number of the error symbol. If not
** defined, then do no error processing.
** YYNSTATE the combined number of states.
@ -211,6 +213,7 @@ struct yyParser {
int yyerrcnt; /* Shifts left before out of the error */
#endif
ParseARG_SDECL /* A place to hold %extra_argument */
ParseCTX_SDECL /* A place to hold %extra_context */
#if YYSTACKDEPTH<=0
int yystksz; /* Current side of the stack */
yyStackEntry *yystack; /* The parser's stack */
@ -315,28 +318,29 @@ static int yyGrowStack(yyParser *p){
/* Initialize a new parser that has already been allocated.
*/
void ParseInit(void *yypParser){
yyParser *pParser = (yyParser*)yypParser;
void ParseInit(void *yypRawParser ParseCTX_PDECL){
yyParser *yypParser = (yyParser*)yypRawParser;
ParseCTX_STORE
#ifdef YYTRACKMAXSTACKDEPTH
pParser->yyhwm = 0;
yypParser->yyhwm = 0;
#endif
#if YYSTACKDEPTH<=0
pParser->yytos = NULL;
pParser->yystack = NULL;
pParser->yystksz = 0;
if( yyGrowStack(pParser) ){
pParser->yystack = &pParser->yystk0;
pParser->yystksz = 1;
yypParser->yytos = NULL;
yypParser->yystack = NULL;
yypParser->yystksz = 0;
if( yyGrowStack(yypParser) ){
yypParser->yystack = &yypParser->yystk0;
yypParser->yystksz = 1;
}
#endif
#ifndef YYNOERRORRECOVERY
pParser->yyerrcnt = -1;
yypParser->yyerrcnt = -1;
#endif
pParser->yytos = pParser->yystack;
pParser->yystack[0].stateno = 0;
pParser->yystack[0].major = 0;
yypParser->yytos = yypParser->yystack;
yypParser->yystack[0].stateno = 0;
yypParser->yystack[0].major = 0;
#if YYSTACKDEPTH>0
pParser->yystackEnd = &pParser->yystack[YYSTACKDEPTH-1];
yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1];
#endif
}
@ -353,11 +357,14 @@ void ParseInit(void *yypParser){
** A pointer to a parser. This pointer is used in subsequent calls
** to Parse and ParseFree.
*/
void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){
yyParser *pParser;
pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
if( pParser ) ParseInit(pParser);
return pParser;
void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){
yyParser *yypParser;
yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) );
if( yypParser ){
ParseCTX_STORE
ParseInit(yypParser ParseCTX_PARAM);
}
return (void*)yypParser;
}
#endif /* Parse_ENGINEALWAYSONSTACK */
@ -374,7 +381,8 @@ static void yy_destructor(
YYCODETYPE yymajor, /* Type code for object to destroy */
YYMINORTYPE *yypminor /* The object to be destroyed */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
switch( yymajor ){
/* Here is inserted the actions which take place when a
** terminal or non-terminal is destroyed. This can happen
@ -596,7 +604,8 @@ static int yy_find_reduce_action(
** The following routine is called if the stack overflows.
*/
static void yyStackOverflow(yyParser *yypParser){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt);
@ -608,7 +617,8 @@ static void yyStackOverflow(yyParser *yypParser){
/******** Begin %stack_overflow code ******************************************/
%%
/******** End %stack_overflow code ********************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument var */
ParseARG_STORE /* Suppress warning about unused %extra_argument var */
ParseCTX_STORE
}
/*
@ -701,12 +711,13 @@ static void yy_reduce(
unsigned int yyruleno, /* Number of the rule by which to reduce */
int yyLookahead, /* Lookahead token, or YYNOCODE if none */
ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */
ParseCTX_PDECL /* %extra_context */
){
int yygoto; /* The next state */
int yyact; /* The next action */
yyStackEntry *yymsp; /* The top of the parser's stack */
int yysize; /* Amount to pop the stack */
ParseARG_FETCH;
ParseARG_FETCH
(void)yyLookahead;
(void)yyLookaheadToken;
yymsp = yypParser->yytos;
@ -789,7 +800,8 @@ static void yy_reduce(
static void yy_parse_failed(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt);
@ -801,7 +813,8 @@ static void yy_parse_failed(
/************ Begin %parse_failure code ***************************************/
%%
/************ End %parse_failure code *****************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}
#endif /* YYNOERRORRECOVERY */
@ -813,12 +826,14 @@ static void yy_syntax_error(
int yymajor, /* The major type of the error token */
ParseTOKENTYPE yyminor /* The minor type of the error token */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#define TOKEN yyminor
/************ Begin %syntax_error code ****************************************/
%%
/************ End %syntax_error code ******************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}
/*
@ -827,7 +842,8 @@ static void yy_syntax_error(
static void yy_accept(
yyParser *yypParser /* The parser */
){
ParseARG_FETCH;
ParseARG_FETCH
ParseCTX_FETCH
#ifndef NDEBUG
if( yyTraceFILE ){
fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt);
@ -842,7 +858,8 @@ static void yy_accept(
/*********** Begin %parse_accept code *****************************************/
%%
/*********** End %parse_accept code *******************************************/
ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */
ParseARG_STORE /* Suppress warning about unused %extra_argument variable */
ParseCTX_STORE
}
/* The main parser program.
@ -878,14 +895,14 @@ void Parse(
#ifdef YYERRORSYMBOL
int yyerrorhit = 0; /* True if yymajor has invoked an error */
#endif
yyParser *yypParser; /* The parser */
yyParser *yypParser = (yyParser*)yyp; /* The parser */
ParseCTX_FETCH
ParseARG_STORE
yypParser = (yyParser*)yyp;
assert( yypParser->yytos!=0 );
#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY)
yyendofinput = (yymajor==0);
#endif
ParseARG_STORE;
#ifndef NDEBUG
if( yyTraceFILE ){
@ -903,7 +920,7 @@ void Parse(
do{
yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor);
if( yyact >= YY_MIN_REDUCE ){
yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor);
yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor ParseCTX_PARAM);
}else if( yyact <= YY_MAX_SHIFTREDUCE ){
yy_shift(yypParser,yyact,yymajor,yyminor);
#ifndef YYNOERRORRECOVERY