From 2e51716b1520fc6e681e9578d681e2e60ae6590f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2019 02:09:47 +0000 Subject: [PATCH 1/3] Increase the size of the yy_lookahead table so that it is never necessary to down bounds checking on the index. FossilOrigin-Name: bafd872398e58766e996963372c7acc03a1e20a6d39a3867ca45d3ea0ed2ac1d --- manifest | 20 +++++++++++--------- manifest.uuid | 2 +- tool/lemon.c | 19 ++++++++++++++++++- tool/lempar.c | 6 ++++-- 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index e3c0e4ca6c..2e8403bc0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\s"ORDER\sBY\s...\sNULLS\sFIRST"\sand\s"ORDER\sBY\s...\sNULLS\sLAST".\sUse\sthis\sto\sfix\sticket\s[f8a7060e]. -D 2019-08-27T19:59:21.714 +C Increase\sthe\ssize\sof\sthe\syy_lookahead\stable\sso\sthat\sit\sis\snever\snecessary\sto\ndown\sbounds\schecking\son\sthe\sindex. +D 2019-08-28T02:09:47.441 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1756,8 +1756,8 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c 9ec344d29cbeb03fdc0fce668eedfb7495792170de933adf95cf8d6904a166ad F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c cc43148d2793ab93668e74a6a6319bb71c2e65c1ffcd6deeaa59e12be9362a43 -F tool/lempar.c 4b868d0a5eb9a67cd1b546274227c0d449dad073c663b331b686fa2253e1c86f +F tool/lemon.c 3b40d248784ada53b1210fb389ebfc149f3043eebcfbe477646430137071798b +F tool/lempar.c be669a41410f127485e7d52b0c50f5b449c745aa23e06940b9d1a9c9eda58633 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1837,8 +1837,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 336235db2b1167cdb2feb64b47eb6368c97c43ee9641b7bfccc5775a41dd0d0e ad816d01d4bd3908ff2c574d79d1a29b6d732df308f2f1b1cdf7c0bc7c1bd7cf -R 5139358becb963924affc3638fcd0d5e -T +closed ad816d01d4bd3908ff2c574d79d1a29b6d732df308f2f1b1cdf7c0bc7c1bd7cf -U dan -Z 534b1b1d3945585b80108a8c2d91a394 +P 94085fb3e756bc984237b74b6e29c68462ad860870c64dcb5124feaeec387660 +R 622629c7f73f36222c78d3454b5494fa +T *branch * lemon-optimization +T *sym-lemon-optimization * +T -sym-trunk * +U drh +Z 2510adf038772aa7dc037a8992922686 diff --git a/manifest.uuid b/manifest.uuid index 12718058d1..24efefb261 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94085fb3e756bc984237b74b6e29c68462ad860870c64dcb5124feaeec387660 \ No newline at end of file +bafd872398e58766e996963372c7acc03a1e20a6d39a3867ca45d3ea0ed2ac1d \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 4416f67bb2..3daa6d38cf 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4153,6 +4153,7 @@ void ReportTable( struct rule *rp; struct acttab *pActtab; int i, j, n, sz; + int nLookAhead; int szActionType; /* sizeof(YYACTIONTYPE) */ int szCodeType; /* sizeof(YYCODETYPE) */ const char *name; @@ -4403,13 +4404,29 @@ void ReportTable( if( la<0 ) la = lemp->nsymbol; if( j==0 ) fprintf(out," /* %5d */ ", i); fprintf(out, " %4d,", la); - if( j==9 || i==n-1 ){ + if( j==9 ){ fprintf(out, "\n"); lineno++; j = 0; }else{ j++; } } + /* Add extra entries to the end of the yy_lookahead[] table so that + ** yy_shift_ofst[]+iToken will always be a valid index into the array, + ** even for the largest possible value of yy_shift_ofst[] and iToken. */ + nLookAhead = lemp->nterminal + lemp->nactiontab; + while( interminal); + if( j==9 ){ + fprintf(out, "\n"); lineno++; + j = 0; + }else{ + j++; + } + i++; + } + if( j>0 ) fprintf(out, "\n"); lineno++; fprintf(out, "};\n"); lineno++; /* Output the yy_shift_ofst[] table */ diff --git a/tool/lempar.c b/tool/lempar.c index 1c088080d4..baf6a217df 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -521,11 +521,13 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ if( iLookAhead Date: Wed, 28 Aug 2019 11:31:11 +0000 Subject: [PATCH 2/3] Further improvements to parser speed by enlarging lookup tables to eliminate the need to do range checking on the index prior to lookup. FossilOrigin-Name: 47d3e091ae49eb7947af5abef9b5b96b16b86d349e51fe0677795649be6db473 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- tool/lemon.c | 4 +++- tool/lempar.c | 17 +++++------------ 4 files changed, 16 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 2e8403bc0f..167921e6c3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\ssize\sof\sthe\syy_lookahead\stable\sso\sthat\sit\sis\snever\snecessary\sto\ndown\sbounds\schecking\son\sthe\sindex. -D 2019-08-28T02:09:47.441 +C Further\simprovements\sto\sparser\sspeed\sby\senlarging\slookup\stables\sto\seliminate\nthe\sneed\sto\sdo\srange\schecking\son\sthe\sindex\sprior\sto\slookup. +D 2019-08-28T11:31:11.959 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1756,8 +1756,8 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c 9ec344d29cbeb03fdc0fce668eedfb7495792170de933adf95cf8d6904a166ad F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 3b40d248784ada53b1210fb389ebfc149f3043eebcfbe477646430137071798b -F tool/lempar.c be669a41410f127485e7d52b0c50f5b449c745aa23e06940b9d1a9c9eda58633 +F tool/lemon.c c9848ef9694689d244a5097238ca1f83df85cc52c80ad149a4cf49595a0ee9c2 +F tool/lempar.c 5d1b20f2e9d7d6e3e1f5e1c11dba455c798bf0097634f5aef7c68a1ae7bc7d86 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1837,10 +1837,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 94085fb3e756bc984237b74b6e29c68462ad860870c64dcb5124feaeec387660 -R 622629c7f73f36222c78d3454b5494fa -T *branch * lemon-optimization -T *sym-lemon-optimization * -T -sym-trunk * +P bafd872398e58766e996963372c7acc03a1e20a6d39a3867ca45d3ea0ed2ac1d +R 958bf9b2f74079dc6a59cf74943d64b1 U drh -Z 2510adf038772aa7dc037a8992922686 +Z 3fa2b314bcffff2187af5dcc971ba8ee diff --git a/manifest.uuid b/manifest.uuid index 24efefb261..2c14daf263 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bafd872398e58766e996963372c7acc03a1e20a6d39a3867ca45d3ea0ed2ac1d \ No newline at end of file +47d3e091ae49eb7947af5abef9b5b96b16b86d349e51fe0677795649be6db473 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 3daa6d38cf..893e1d18b9 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4506,7 +4506,9 @@ void ReportTable( */ if( lemp->has_fallback ){ int mx = lemp->nterminal - 1; - while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } + /* 2019-08-28: Generate fallback entries for every token to avoid + ** having to do a range check on the index */ + /* while( mx>0 && lemp->symbols[mx]->fallback==0 ){ mx--; } */ lemp->tablesize += (mx+1)*szCodeType; for(i=0; i<=mx; i++){ struct symbol *p = lemp->symbols[i]; diff --git a/tool/lempar.c b/tool/lempar.c index baf6a217df..9d5b19f465 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -530,8 +530,9 @@ static YYACTIONTYPE yy_find_shift_action( if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", @@ -546,16 +547,8 @@ static YYACTIONTYPE yy_find_shift_action( #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 - ){ + assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); + if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", From 6ee3fa87fd7ca53713cffb9ff662a61f797b760d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2019 11:49:45 +0000 Subject: [PATCH 3/3] Fix an unreachable branch in sqlite3ParserFallback() FossilOrigin-Name: e059178b47109caee2c2211b2db6e594c014af636677118a64e10edf01ac017d --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 167921e6c3..23b6a28b23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\sparser\sspeed\sby\senlarging\slookup\stables\sto\seliminate\nthe\sneed\sto\sdo\srange\schecking\son\sthe\sindex\sprior\sto\slookup. -D 2019-08-28T11:31:11.959 +C Fix\san\sunreachable\sbranch\sin\ssqlite3ParserFallback() +D 2019-08-28T11:49:45.616 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1757,7 +1757,7 @@ F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c 9ec344d29cbeb03fdc0fce668eedfb7495792170de933adf95cf8d6904a166ad F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c c9848ef9694689d244a5097238ca1f83df85cc52c80ad149a4cf49595a0ee9c2 -F tool/lempar.c 5d1b20f2e9d7d6e3e1f5e1c11dba455c798bf0097634f5aef7c68a1ae7bc7d86 +F tool/lempar.c eb2841e2a7fd484cf44b1f526b06e7ab0f216d2f41818bf9485e8f38e3d1db19 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1837,7 +1837,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bafd872398e58766e996963372c7acc03a1e20a6d39a3867ca45d3ea0ed2ac1d -R 958bf9b2f74079dc6a59cf74943d64b1 +P 47d3e091ae49eb7947af5abef9b5b96b16b86d349e51fe0677795649be6db473 +R eed84f5af5dd0a6ce743afeb00c4b4e4 U drh -Z 3fa2b314bcffff2187af5dcc971ba8ee +Z 0c74ffec3ce3109c0e0f66d709525f70 diff --git a/manifest.uuid b/manifest.uuid index 2c14daf263..1d045fd59e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47d3e091ae49eb7947af5abef9b5b96b16b86d349e51fe0677795649be6db473 \ No newline at end of file +e059178b47109caee2c2211b2db6e594c014af636677118a64e10edf01ac017d \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index 9d5b19f465..f75ad51c39 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -1064,9 +1064,8 @@ void Parse( */ int ParseFallback(int iToken){ #ifdef YYFALLBACK - if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ - return yyFallback[iToken]; - } + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; #else (void)iToken; #endif