From e58f74f680f97f382db35215b22203b3b10abb30 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Dec 2017 17:06:41 +0000 Subject: [PATCH 01/15] Improved parser tracing output. FossilOrigin-Name: 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b --- manifest | 16 ++++++++++------ manifest.uuid | 2 +- tool/lempar.c | 36 +++++++++++++++++++++++++----------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index f187cc0290..07c293333d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader. -D 2017-12-23T14:39:36.160 +C Improved\sparser\stracing\soutput. +D 2017-12-24T17:06:41.289 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e -F tool/lempar.c 967ebf585cd09b11b89d255d213865109a9c4ff075680d22580a2826de288c89 +F tool/lempar.c a427c2375df118fa52e69174ffbbf9e26878096f0a109df6b77062d6032afe18 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9 -R 7fbfe3e61eca395fa5baf5f121a4d2ad +P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 +Q +c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32 +R 3f4f31cde41f94adafa8015552d0561b +T *branch * lemon-improvements +T *sym-lemon-improvements * +T -sym-trunk * U drh -Z d82d9c01768cefc5beb206b92e1398bf +Z 3f6d9f274761457d9d7dcc3ca6c0270c diff --git a/manifest.uuid b/manifest.uuid index 8454418cd5..bdf982cb88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 \ No newline at end of file +25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index da81ddd4bc..89c1c80f70 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -578,20 +578,21 @@ static void yyStackOverflow(yyParser *yypParser){ ** Print tracing information for a SHIFT action */ #ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState){ +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ if( yyTraceFILE ){ if( yyNewStateyytos->major], + fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], yyNewState); }else{ - fprintf(yyTraceFILE,"%sShift '%s'\n", - yyTracePrompt,yyTokenName[yypParser->yytos->major]); + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); } } } #else -# define yyTraceShift(X,Y) +# define yyTraceShift(X,Y,Z) #endif /* @@ -633,7 +634,7 @@ static void yy_shift( yytos->stateno = (YYACTIONTYPE)yyNewState; yytos->major = (YYCODETYPE)yyMajor; yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState); + yyTraceShift(yypParser, yyNewState, "Shift"); } /* The following table contains information about every rule that @@ -673,8 +674,14 @@ static void yy_reduce( #ifndef NDEBUG if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ yysize = yyRuleInfo[yyruleno].nrhs; - fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, - yyRuleName[yyruleno], yymsp[yysize].stateno); + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s], go to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], yymsp[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s].\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno]); + } } #endif /* NDEBUG */ @@ -737,7 +744,7 @@ static void yy_reduce( yypParser->yytos = yymsp; yymsp->stateno = (YYACTIONTYPE)yyact; yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact); + yyTraceShift(yypParser, yyact, "... then shift"); } } @@ -848,7 +855,14 @@ void Parse( #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); + int stateno = yypParser->yytos->stateno; + if( stateno < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],stateno); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],stateno-YY_MIN_REDUCE); + } } #endif From 512795dfeae9dd816eefda3eee3222aedcd6d2b5 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Dec 2017 18:56:28 +0000 Subject: [PATCH 02/15] Fix a NULL pointer dereference after a syntax error that can occur as a result of check-in [6b2ff26c25bb9da3] yesterday. This problem was discovered by the OSSFuzz. FossilOrigin-Name: d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 1 + test/colname.test | 10 ++++++++++ 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f187cc0290..1c622c1102 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\serror\shandling\slogic\sin\sthe\sextension\sloader. -D 2017-12-23T14:39:36.160 +C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz. +D 2017-12-24T18:56:28.786 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c ed567f088edbc305dad33a6b14e08f8216a3860f6bad1d180450d5a5414bf346 +F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test a7ecb8f1d6d8b30a6cf8fa84a2cd6f6e91cad8296376fabe485cf93cd5eb6229 +F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58 F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 05fee1a21ea398f1e4d6f1cf361657eff25ed6cd8f85ab398262dcfd30da57e9 -R 7fbfe3e61eca395fa5baf5f121a4d2ad +P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 +R 2e4222d820aa06549d33319e9e33c627 U drh -Z d82d9c01768cefc5beb206b92e1398bf +Z 67298e45eb2d4d992f0cdf5678deffdf diff --git a/manifest.uuid b/manifest.uuid index 8454418cd5..d58d2c0dd5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 \ No newline at end of file +d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba \ No newline at end of file diff --git a/src/build.c b/src/build.c index 01d8972415..390dae5d3b 100644 --- a/src/build.c +++ b/src/build.c @@ -1965,6 +1965,7 @@ void sqlite3EndTable( pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + if( pParse->nErr ) return; pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); if( pSelTab==0 ) return; assert( p->aCol==0 ); diff --git a/test/colname.test b/test/colname.test index 5a40286773..cacf91e4e3 100644 --- a/test/colname.test +++ b/test/colname.test @@ -398,6 +398,16 @@ do_execsql_test colname-9.320 { SELECT name FROM pragma_table_info('t2'); } {Bbb} +# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve) +# caused by check-in https://sqlite.org/src/info/6b2ff26c25 +# +# Prior to being fixed, the following CREATE TABLE was dereferencing +# a NULL pointer and segfaulting. +# +do_catchsql_test colname-9.400 { + CREATE TABLE t4 AS SELECT #0; +} {1 {near "#0": syntax error}} + # Make sure the quotation marks get removed from the column names # when constructing a new table from an aggregate SELECT. From 5c8241b8750ce919b4151761a1d721af58bfbf7b Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Dec 2017 23:38:10 +0000 Subject: [PATCH 03/15] In the LEMON-generated parser, rearrange the meanings of integer action codes so that reduce actions occur last. This means that the most common case (reduce actions) can be recognized with a single comparison operation, thus speeding up the main parser loop, slightly. FossilOrigin-Name: 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a --- manifest | 18 ++++----- manifest.uuid | 2 +- tool/lemon.c | 100 ++++++++++++++++++++++++++++++++------------------ tool/lempar.c | 16 ++++---- 4 files changed, 81 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index 07c293333d..d8e37c9644 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sparser\stracing\soutput. -D 2017-12-24T17:06:41.289 +C In\sthe\sLEMON-generated\sparser,\srearrange\sthe\smeanings\sof\sinteger\saction\scodes\nso\sthat\sreduce\sactions\soccur\slast.\s\sThis\smeans\sthat\sthe\smost\scommon\scase\n(reduce\sactions)\scan\sbe\srecognized\swith\sa\ssingle\scomparison\soperation,\sthus\nspeeding\sup\sthe\smain\sparser\sloop,\sslightly. +D 2017-12-24T23:38:10.370 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c e6056373044d55296d21f81467dba7632bbb81dc49af072b3f0e76338771497e -F tool/lempar.c a427c2375df118fa52e69174ffbbf9e26878096f0a109df6b77062d6032afe18 +F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804 +F tool/lempar.c c8dd4dcf0bca9d7c27c62f7df12882c30db749cd9bb83d6f71796b9fabb94f6c F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,11 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 -Q +c4951833c2b976223c2393d82fd2606068c71cd19612ca9df4e26debab980e32 -R 3f4f31cde41f94adafa8015552d0561b -T *branch * lemon-improvements -T *sym-lemon-improvements * -T -sym-trunk * +P 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b +R 02413a9b8686f5cb855fd9a72c28a347 U drh -Z 3f6d9f274761457d9d7dcc3ca6c0270c +Z a4798d5b361261f3f5c39967bad11abf diff --git a/manifest.uuid b/manifest.uuid index bdf982cb88..d4db86c744 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b \ No newline at end of file +7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index acc5450c99..33ef43d1b1 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -384,6 +384,12 @@ struct lemon { int nrule; /* Number of rules */ int nsymbol; /* Number of terminal and nonterminal symbols */ int nterminal; /* Number of terminal symbols */ + int minShiftReduce; /* Minimum shift-reduce action value */ + int errAction; /* Error action value */ + int accAction; /* Accept action value */ + int noAction; /* No-op action value */ + int minReduce; /* Minimum reduce action */ + int maxAction; /* Maximum action value of any kind */ struct symbol **symbols; /* Sorted array of pointers to symbols */ int errorcnt; /* Number of errors */ struct symbol *errsym; /* The error symbol */ @@ -3020,6 +3026,27 @@ PRIVATE FILE *file_open( return fp; } +/* Print the text of a rule +*/ +void rule_print(FILE *out, struct rule *rp){ + int i, j; + fprintf(out, "%s",rp->lhs->name); + /* if( rp->lhsalias ) fprintf(out,"(%s)",rp->lhsalias); */ + fprintf(out," ::="); + for(i=0; inrhs; i++){ + struct symbol *sp = rp->rhs[i]; + if( sp->type==MULTITERMINAL ){ + fprintf(out," %s", sp->subsym[0]->name); + for(j=1; jnsubsym; j++){ + fprintf(out,"|%s", sp->subsym[j]->name); + } + }else{ + fprintf(out," %s", sp->name); + } + /* if( rp->rhsalias[i] ) fprintf(out,"(%s)",rp->rhsalias[i]); */ + } +} + /* Duplicate the input file without comments and without actions ** on rules */ void Reprint(struct lemon *lemp) @@ -3047,21 +3074,7 @@ void Reprint(struct lemon *lemp) printf("\n"); } for(rp=lemp->rule; rp; rp=rp->next){ - printf("%s",rp->lhs->name); - /* if( rp->lhsalias ) printf("(%s)",rp->lhsalias); */ - printf(" ::="); - for(i=0; inrhs; i++){ - sp = rp->rhs[i]; - if( sp->type==MULTITERMINAL ){ - printf(" %s", sp->subsym[0]->name); - for(j=1; jnsubsym; j++){ - printf("|%s", sp->subsym[j]->name); - } - }else{ - printf(" %s", sp->name); - } - /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ - } + rule_print(stdout, rp); printf("."); if( rp->precsym ) printf(" [%s]",rp->precsym->name); /* if( rp->code ) printf("\n %s",rp->code); */ @@ -3321,16 +3334,19 @@ PRIVATE int compute_action(struct lemon *lemp, struct action *ap) switch( ap->type ){ case SHIFT: act = ap->x.stp->statenum; break; case SHIFTREDUCE: { - act = ap->x.rp->iRule + lemp->nstate; /* Since a SHIFT is inherient after a prior REDUCE, convert any ** SHIFTREDUCE action with a nonterminal on the LHS into a simple ** REDUCE action: */ - if( ap->sp->index>=lemp->nterminal ) act += lemp->nrule; + if( ap->sp->index>=lemp->nterminal ){ + act = lemp->minReduce + ap->x.rp->iRule; + }else{ + act = lemp->minShiftReduce + ap->x.rp->iRule; + } break; } - case REDUCE: act = ap->x.rp->iRule + lemp->nstate+lemp->nrule; break; - case ERROR: act = lemp->nstate + lemp->nrule*2; break; - case ACCEPT: act = lemp->nstate + lemp->nrule*2 + 1; break; + case REDUCE: act = lemp->minReduce + ap->x.rp->iRule; break; + case ERROR: act = lemp->errAction; break; + case ACCEPT: act = lemp->accAction; break; default: act = -1; break; } return act; @@ -4038,6 +4054,13 @@ void ReportTable( int mnNtOfst, mxNtOfst; struct axset *ax; + lemp->minShiftReduce = lemp->nstate; + lemp->errAction = lemp->minShiftReduce + lemp->nrule; + lemp->accAction = lemp->errAction + 1; + lemp->noAction = lemp->accAction + 1; + lemp->minReduce = lemp->noAction + 1; + lemp->maxAction = lemp->minReduce + lemp->nrule; + in = tplt_open(lemp); if( in==0 ) return; out = file_open(lemp,".c","wb"); @@ -4076,7 +4099,7 @@ void ReportTable( minimum_size_type(0, lemp->nsymbol+1, &szCodeType)); lineno++; fprintf(out,"#define YYNOCODE %d\n",lemp->nsymbol+1); lineno++; fprintf(out,"#define YYACTIONTYPE %s\n", - minimum_size_type(0,lemp->nstate+lemp->nrule*2+5,&szActionType)); lineno++; + minimum_size_type(0,lemp->maxAction,&szActionType)); lineno++; if( lemp->wildcard ){ fprintf(out,"#define YYWILDCARD %d\n", lemp->wildcard->index); lineno++; @@ -4201,15 +4224,16 @@ void ReportTable( fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; - fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",lemp->nstate); lineno++; - i = lemp->nstate + lemp->nrule; + i = lemp->minShiftReduce; + fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",i); lineno++; + i += lemp->nrule; fprintf(out,"#define YY_MAX_SHIFTREDUCE %d\n", i-1); lineno++; - fprintf(out,"#define YY_MIN_REDUCE %d\n", i); lineno++; - i = lemp->nstate + lemp->nrule*2; + fprintf(out,"#define YY_ERROR_ACTION %d\n", lemp->errAction); lineno++; + fprintf(out,"#define YY_ACCEPT_ACTION %d\n", lemp->accAction); lineno++; + fprintf(out,"#define YY_NO_ACTION %d\n", lemp->noAction); lineno++; + fprintf(out,"#define YY_MIN_REDUCE %d\n", lemp->minReduce); lineno++; + i = lemp->minReduce + lemp->nrule; fprintf(out,"#define YY_MAX_REDUCE %d\n", i-1); lineno++; - fprintf(out,"#define YY_ERROR_ACTION %d\n", i); lineno++; - fprintf(out,"#define YY_ACCEPT_ACTION %d\n", i+1); lineno++; - fprintf(out,"#define YY_NO_ACTION %d\n", i+2); lineno++; tplt_xfer(lemp->name,in,out,&lineno); /* Now output the action table and its associates: @@ -4231,7 +4255,7 @@ void ReportTable( fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; for(i=j=0; instate + lemp->nrule + 2; + if( action<0 ) action = lemp->noAction; if( j==0 ) fprintf(out," /* %5d */ ", i); fprintf(out, " %4d,", action); if( j==9 || i==n-1 ){ @@ -4320,7 +4344,11 @@ void ReportTable( for(i=j=0; isorted[i]; if( j==0 ) fprintf(out," /* %5d */ ", i); - fprintf(out, " %4d,", stp->iDfltReduce+lemp->nstate+lemp->nrule); + if( stp->iDfltReduce<0 ){ + fprintf(out, " %4d,", lemp->errAction); + }else{ + fprintf(out, " %4d,", stp->iDfltReduce + lemp->minReduce); + } if( j==9 || i==n-1 ){ fprintf(out, "\n"); lineno++; j = 0; @@ -4401,7 +4429,7 @@ void ReportTable( if( sp==0 || sp->type==TERMINAL || sp->index<=0 || sp->destructor!=0 ) continue; if( once ){ - fprintf(out, " /* Default NON-TERMINAL Destructor */\n"); lineno++; + fprintf(out, " /* Default NON-TERMINAL Destructor */\n");lineno++; once = 0; } fprintf(out," case %d: /* %s */\n", sp->index, sp->name); lineno++; @@ -4444,8 +4472,10 @@ void ReportTable( ** Note: This code depends on the fact that rules are number ** sequentually beginning with 0. */ - for(rp=lemp->rule; rp; rp=rp->next){ - fprintf(out," { %d, %d },\n",rp->lhs->index,-rp->nrhs); lineno++; + for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){ + fprintf(out," { %4d, %4d }, /* (%d) ",rp->lhs->index,-rp->nrhs,i); + rule_print(out, rp); + fprintf(out," */\n"); lineno++; } tplt_xfer(lemp->name,in,out,&lineno); @@ -4711,7 +4741,7 @@ void ResortStates(struct lemon *lemp) for(i=0; instate; i++){ stp = lemp->sorted[i]; stp->nTknAct = stp->nNtAct = 0; - stp->iDfltReduce = lemp->nrule; /* Init dflt action to "syntax error" */ + stp->iDfltReduce = -1; /* Init dflt action to "syntax error" */ stp->iTknOfst = NO_OFFSET; stp->iNtOfst = NO_OFFSET; for(ap=stp->ap; ap; ap=ap->next){ @@ -4723,7 +4753,7 @@ void ResortStates(struct lemon *lemp) stp->nNtAct++; }else{ assert( stp->autoReduce==0 || stp->pDfltReduce==ap->x.rp ); - stp->iDfltReduce = iAction - lemp->nstate - lemp->nrule; + stp->iDfltReduce = iAction; } } } diff --git a/tool/lempar.c b/tool/lempar.c index 89c1c80f70..7d25175ca7 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -75,11 +75,11 @@ ** YY_MAX_SHIFT Maximum value for shift actions ** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions ** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions ** YY_ERROR_ACTION The yy_action[] code for syntax error ** YY_ACCEPT_ACTION The yy_action[] code for accept ** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions */ #ifndef INTERFACE # define INTERFACE 1 @@ -115,9 +115,6 @@ ** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then ** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** ** N == YY_ERROR_ACTION A syntax error has occurred. ** ** N == YY_ACCEPT_ACTION The parser accepts its input. @@ -125,6 +122,9 @@ ** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** ** The action table is constructed as a single large table named yy_action[]. ** Given state S and lookahead X, the action is computed as either: ** @@ -868,14 +868,14 @@ void Parse( do{ yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact <= YY_MAX_SHIFTREDUCE ){ + if( yyact >= YY_MIN_REDUCE ){ + yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ yy_shift(yypParser,yyact,yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif yymajor = YYNOCODE; - }else if( yyact <= YY_MAX_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor,yyminor); }else{ assert( yyact == YY_ERROR_ACTION ); yyminorunion.yy0 = yyminor; From ef53a9f0af73a66d8844fb2b58c669213677530e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Dec 2017 00:10:05 +0000 Subject: [PATCH 04/15] In the LEMON-generated parser, avoid unnecessary tests for the acceptance state. FossilOrigin-Name: fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 21 ++++++++++----------- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index d8e37c9644..f8d8364cbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sLEMON-generated\sparser,\srearrange\sthe\smeanings\sof\sinteger\saction\scodes\nso\sthat\sreduce\sactions\soccur\slast.\s\sThis\smeans\sthat\sthe\smost\scommon\scase\n(reduce\sactions)\scan\sbe\srecognized\swith\sa\ssingle\scomparison\soperation,\sthus\nspeeding\sup\sthe\smain\sparser\sloop,\sslightly. -D 2017-12-24T23:38:10.370 +C In\sthe\sLEMON-generated\sparser,\savoid\sunnecessary\stests\sfor\sthe\sacceptance\nstate. +D 2017-12-25T00:10:05.418 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804 -F tool/lempar.c c8dd4dcf0bca9d7c27c62f7df12882c30db749cd9bb83d6f71796b9fabb94f6c +F tool/lempar.c 8062f219b4ce349853cb3ab3ebd3ab44466604235347457d703a9f4252e76dd5 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 25be5750545b7b0ed9e1a1baca31611b354519688f875aa1590b21bb6ff42f1b -R 02413a9b8686f5cb855fd9a72c28a347 +P 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a +R 904d9f58d13471f5c0ad3064ead9b812 U drh -Z a4798d5b361261f3f5c39967bad11abf +Z c00f50853c78fa733a13b33ab92ee708 diff --git a/manifest.uuid b/manifest.uuid index d4db86c744..e026b0ddc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a \ No newline at end of file +fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046 \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index 7d25175ca7..6c6ca77dde 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -472,7 +472,7 @@ static unsigned int yy_find_shift_action( int i; int stateno = pParser->yytos->stateno; - if( stateno>=YY_MIN_REDUCE ) return stateno; + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); do{ i = yy_shift_ofst[stateno]; @@ -736,16 +736,11 @@ static void yy_reduce( /* It is not possible for a REDUCE to be followed by an error */ assert( yyact!=YY_ERROR_ACTION ); - if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos += yysize; - yy_accept(yypParser); - }else{ - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - } + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); } /* @@ -876,6 +871,10 @@ void Parse( yypParser->yyerrcnt--; #endif yymajor = YYNOCODE; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; }else{ assert( yyact == YY_ERROR_ACTION ); yyminorunion.yy0 = yyminor; From 3a9d6c7156e33c63f03f12be8e2168c8d0aa7d2b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Dec 2017 04:15:38 +0000 Subject: [PATCH 05/15] Enhance LEMON so that it generates the action table in such a way that no range check is needed on the lookahead table to verify that the next input token is valid. This makes the lookahead table slightly larger (about 120 bytes) but helps the parser to run faster. FossilOrigin-Name: 7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814 --- manifest | 14 +++++------ manifest.uuid | 2 +- tool/lemon.c | 65 +++++++++++++++++++++++++++++++++++++-------------- tool/lempar.c | 16 ++++--------- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index f8d8364cbb..d9f3940ae0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sLEMON-generated\sparser,\savoid\sunnecessary\stests\sfor\sthe\sacceptance\nstate. -D 2017-12-25T00:10:05.418 +C Enhance\sLEMON\sso\sthat\sit\sgenerates\sthe\saction\stable\sin\ssuch\sa\sway\sthat\sno\nrange\scheck\sis\sneeded\son\sthe\slookahead\stable\sto\sverify\sthat\sthe\snext\sinput\ntoken\sis\svalid.\s\sThis\smakes\sthe\slookahead\stable\sslightly\slarger\s(about\s120\nbytes)\sbut\shelps\sthe\sparser\sto\srun\sfaster. +D 2017-12-25T04:15:38.668 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 7c6919d98e459c0f8a3673be64b03425553733dba01c12939b2fadc30e4e2804 -F tool/lempar.c 8062f219b4ce349853cb3ab3ebd3ab44466604235347457d703a9f4252e76dd5 +F tool/lemon.c c8d7ce4fe7a90f7fa6a5985452aa926fcf25376cf90095c9d06c432ab0bebdbc +F tool/lempar.c 427ee280f3c3781e82bbee21f428bc8ae18ab245d4f66d65da46b598ded81648 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7bfe7a360261ac7227840db49487c2f0fe338a2f1b868fcaada1e04a8d2b8f7a -R 904d9f58d13471f5c0ad3064ead9b812 +P fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046 +R f14cbf100834d359550a23c924f7ebaa U drh -Z c00f50853c78fa733a13b33ab92ee708 +Z bb1d7d0db816330a6f1e2b8011c2819b diff --git a/manifest.uuid b/manifest.uuid index e026b0ddc4..b69ed5a1cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046 \ No newline at end of file +7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 33ef43d1b1..96dc756d1b 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -413,6 +413,7 @@ struct lemon { char *tokenprefix; /* A prefix added to token names in the .h file */ int nconflict; /* Number of parsing conflicts */ int nactiontab; /* Number of entries in the yy_action[] table */ + int nlookaheadtab; /* Number of entries in yy_lookahead[] */ int tablesize; /* Total table size of all tables in bytes */ int basisflag; /* Print only basis configurations */ int has_fallback; /* True if any %fallback is seen in the grammar */ @@ -589,10 +590,12 @@ struct acttab { int mxLookahead; /* Maximum aLookahead[].lookahead */ int nLookahead; /* Used slots in aLookahead[] */ int nLookaheadAlloc; /* Slots allocated in aLookahead[] */ + int nterminal; /* Number of terminal symbols */ + int nsymbol; /* total number of symbols */ }; /* Return the number of entries in the yy_action table */ -#define acttab_size(X) ((X)->nAction) +#define acttab_lookahead_size(X) ((X)->nAction) /* The value for the N-th entry in yy_action */ #define acttab_yyaction(X,N) ((X)->aAction[N].action) @@ -608,13 +611,15 @@ void acttab_free(acttab *p){ } /* Allocate a new acttab structure */ -acttab *acttab_alloc(void){ +acttab *acttab_alloc(int nsymbol, int nterminal){ acttab *p = (acttab *) calloc( 1, sizeof(*p) ); if( p==0 ){ fprintf(stderr,"Unable to allocate memory for a new acttab."); exit(1); } memset(p, 0, sizeof(*p)); + p->nsymbol = nsymbol; + p->nterminal = nterminal; return p; } @@ -655,16 +660,24 @@ void acttab_action(acttab *p, int lookahead, int action){ ** to an empty set in preparation for a new round of acttab_action() calls. ** ** Return the offset into the action table of the new transaction. +** +** If the makeItSafe parameter is true, then the offset is chosen so that +** it is impossible to overread the yy_lookaside[] table regardless of +** the lookaside token. This is done for the terminal symbols, as they +** come from external inputs and can contain syntax errors. When makeItSafe +** is false, there is more flexibility in selecting offsets, resulting in +** a smaller table. For non-terminal symbols, which are never syntax errors, +** makeItSafe can be false. */ -int acttab_insert(acttab *p){ - int i, j, k, n; +int acttab_insert(acttab *p, int makeItSafe){ + int i, j, k, n, end; assert( p->nLookahead>0 ); /* Make sure we have enough space to hold the expanded action table ** in the worst case. The worst case occurs if the transaction set ** must be appended to the current action table */ - n = p->mxLookahead + 1; + n = p->nsymbol + 1; if( p->nAction + n >= p->nActionAlloc ){ int oldAlloc = p->nActionAlloc; p->nActionAlloc = p->nAction + n + p->nActionAlloc + 20; @@ -686,7 +699,8 @@ int acttab_insert(acttab *p){ ** ** i is the index in p->aAction[] where p->mnLookahead is inserted. */ - for(i=p->nAction-1; i>=0; i--){ + end = makeItSafe ? p->mnLookahead : 0; + for(i=p->nAction-1; i>=end; i--){ if( p->aAction[i].lookahead==p->mnLookahead ){ /* All lookaheads and actions in the aLookahead[] transaction ** must match against the candidate aAction[i] entry. */ @@ -716,12 +730,13 @@ int acttab_insert(acttab *p){ ** an empty offset in the aAction[] table in which we can add the ** aLookahead[] transaction. */ - if( i<0 ){ + if( inAction, which means the ** transaction will be appended. */ - for(i=0; inActionAlloc - p->mxLookahead; i++){ + i = makeItSafe ? p->mnLookahead : 0; + for(; inActionAlloc - p->mxLookahead; i++){ if( p->aAction[i].lookahead<0 ){ for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; @@ -739,11 +754,19 @@ int acttab_insert(acttab *p){ } } /* Insert transaction set at index i. */ +#if 0 + printf("Acttab:"); + for(j=0; jnLookahead; j++){ + printf(" %d", p->aLookahead[j].lookahead); + } + printf(" inserted at %d\n", i); +#endif for(j=0; jnLookahead; j++){ k = p->aLookahead[j].lookahead - p->mnLookahead + i; p->aAction[k] = p->aLookahead[j]; if( k>=p->nAction ) p->nAction = k+1; } + if( makeItSafe && i+p->nterminal>p->nAction ) p->nAction = i+p->nterminal; p->nLookahead = 0; /* Return the offset that is added to the lookahead in order to get the @@ -751,6 +774,16 @@ int acttab_insert(acttab *p){ return i - p->mnLookahead; } +/* +** Return the size of the action table without the trailing syntax error +** entries. +*/ +int acttab_action_size(acttab *p){ + int n = p->nAction; + while( n>0 && p->aAction[n-1].lookahead<0 ){ n--; } + return n; +} + /********************** From the file "build.c" *****************************/ /* ** Routines to construction the finite state machine for the LEMON @@ -1724,6 +1757,7 @@ int main(int argc, char **argv) stats_line("states", lem.nxstate); stats_line("conflicts", lem.nconflict); stats_line("action table entries", lem.nactiontab); + stats_line("lookahead table entries", lem.nlookaheadtab); stats_line("total table size (bytes)", lem.tablesize); } if( lem.nconflict > 0 ){ @@ -4167,7 +4201,7 @@ void ReportTable( ** of placing the largest action sets first */ for(i=0; inxstate*2; i++) ax[i].iOrder = i; qsort(ax, lemp->nxstate*2, sizeof(ax[0]), axset_compare); - pActtab = acttab_alloc(); + pActtab = acttab_alloc(lemp->nsymbol, lemp->nterminal); for(i=0; inxstate*2 && ax[i].nAction>0; i++){ stp = ax[i].stp; if( ax[i].isTkn ){ @@ -4178,7 +4212,7 @@ void ReportTable( if( action<0 ) continue; acttab_action(pActtab, ap->sp->index, action); } - stp->iTknOfst = acttab_insert(pActtab); + stp->iTknOfst = acttab_insert(pActtab, 1); if( stp->iTknOfstiTknOfst; if( stp->iTknOfst>mxTknOfst ) mxTknOfst = stp->iTknOfst; }else{ @@ -4190,7 +4224,7 @@ void ReportTable( if( action<0 ) continue; acttab_action(pActtab, ap->sp->index, action); } - stp->iNtOfst = acttab_insert(pActtab); + stp->iNtOfst = acttab_insert(pActtab, 0); if( stp->iNtOfstiNtOfst; if( stp->iNtOfst>mxNtOfst ) mxNtOfst = stp->iNtOfst; } @@ -4249,7 +4283,7 @@ void ReportTable( */ /* Output the yy_action table */ - lemp->nactiontab = n = acttab_size(pActtab); + lemp->nactiontab = n = acttab_action_size(pActtab); lemp->tablesize += n*szActionType; fprintf(out,"#define YY_ACTTAB_COUNT (%d)\n", n); lineno++; fprintf(out,"static const YYACTIONTYPE yy_action[] = {\n"); lineno++; @@ -4268,6 +4302,7 @@ void ReportTable( fprintf(out, "};\n"); lineno++; /* Output the yy_lookahead table */ + lemp->nlookaheadtab = n = acttab_lookahead_size(pActtab); lemp->tablesize += n*szCodeType; fprintf(out,"static const YYCODETYPE yy_lookahead[] = {\n"); lineno++; for(i=j=0; inxstate; while( n>0 && lemp->sorted[n-1]->iTknOfst==NO_OFFSET ) n--; - fprintf(out, "#define YY_SHIFT_USE_DFLT (%d)\n", lemp->nactiontab); lineno++; fprintf(out, "#define YY_SHIFT_COUNT (%d)\n", n-1); lineno++; fprintf(out, "#define YY_SHIFT_MIN (%d)\n", mnTknOfst); lineno++; fprintf(out, "#define YY_SHIFT_MAX (%d)\n", mxTknOfst); lineno++; @@ -4312,7 +4346,6 @@ void ReportTable( fprintf(out, "};\n"); lineno++; /* Output the yy_reduce_ofst[] table */ - fprintf(out, "#define YY_REDUCE_USE_DFLT (%d)\n", mnNtOfst-1); lineno++; n = lemp->nxstate; while( n>0 && lemp->sorted[n-1]->iNtOfst==NO_OFFSET ) n--; fprintf(out, "#define YY_REDUCE_COUNT (%d)\n", n-1); lineno++; @@ -4382,10 +4415,8 @@ void ReportTable( */ for(i=0; insymbol; i++){ lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name); - fprintf(out," %-15s",line); - if( (i&3)==3 ){ fprintf(out,"\n"); lineno++; } + fprintf(out," /* %4d */ \"%s\",\n",i, lemp->symbols[i]->name); lineno++; } - if( (i&3)!=0 ){ fprintf(out,"\n"); lineno++; } tplt_xfer(lemp->name,in,out,&lineno); /* Generate a table containing a text string that describes every diff --git a/tool/lempar.c b/tool/lempar.c index 6c6ca77dde..7ceaaa6207 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -131,19 +131,13 @@ ** (A) N = yy_action[ yy_shift_ofst[S] + X ] ** (B) N = yy_default[S] ** -** The (A) formula is preferred. The B formula is used instead if: -** (1) The yy_shift_ofst[S]+X value is out of range, or -** (2) yy_lookahead[yy_shift_ofst[S]+X] is not equal to X, or -** (3) yy_shift_ofst[S] equal YY_SHIFT_USE_DFLT. -** (Implementation note: YY_SHIFT_USE_DFLT is chosen so that -** YY_SHIFT_USE_DFLT+X will be out of range for all possible lookaheads X. -** Hence only tests (1) and (2) need to be evaluated.) +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. ** ** The formulas above are for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of -** YY_SHIFT_USE_DFLT. +** the yy_shift_ofst[] array. ** ** The following are the tables generated in this section: ** @@ -478,7 +472,8 @@ static unsigned int yy_find_shift_action( i = yy_shift_ofst[stateno]; assert( iLookAhead!=YYNOCODE ); i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + assert( i>=0 && i Date: Mon, 25 Dec 2017 13:43:54 +0000 Subject: [PATCH 06/15] Fix an assertion fault on a syntax error input caused by check-in [6b2ff26c25bb9da3]. Problem discovered by OSSFuzz. FossilOrigin-Name: 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 1 + test/colname.test | 11 ++++++++++- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1c622c1102..d9715194d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sNULL\spointer\sdereference\safter\sa\ssyntax\serror\sthat\scan\soccur\sas\sa\nresult\sof\scheck-in\s[6b2ff26c25bb9da3]\syesterday.\s\sThis\sproblem\swas\ndiscovered\sby\sthe\sOSSFuzz. -D 2017-12-24T18:56:28.786 +C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz. +D 2017-12-25T13:43:54.974 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -423,7 +423,7 @@ F src/btmutex.c 0e9ce2d56159b89b9bc8e197e023ee11e39ff8ca F src/btree.c b83a6b03f160528020bb965f0c3a40af5286cd4923c3870fd218177f03a120a7 F src/btree.h 32ef5d3f25dc70ef1ee9cecf84a023c21378f06a57cd701d2e866e141b150f09 F src/btreeInt.h 55b702efce17e5d1941865464227d3802cfc9c7c832fac81d4c94dced47a71fc -F src/build.c ab5bdf955c85bcd56acbf310a48bbd50b4b92079efa40d997a7e4246f8e03741 +F src/build.c 6ab114a9a4717622df3745d26f2c55e9113f356441a8f76fa2b1d7e1f2472c20 F src/callback.c fe677cb5f5abb02f7a772a62a98c2f516426081df68856e8f2d5f950929b966a F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 @@ -682,7 +682,7 @@ F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 F test/collateB.test 1e68906951b846570f29f20102ed91d29e634854ee47454d725f2151ecac0b95 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 -F test/colname.test 36da785927822ecd0de979459e27e9be63f458dd08d3edde41af3af37a337d58 +F test/colname.test 101aa39392a1f6883278f588836a3ab99178f8103f78032433400475cc05109f F test/conflict.test 029faa2d81a0d1cafb5f88614beb663d972c01db F test/conflict2.test bb0b94cf7196c64a3cbd815c66d3ee98c2fecd9c F test/conflict3.test a83db76a6c3503b2fa057c7bfb08c318d8a422202d8bc5b86226e078e5b49ff9 @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07c773148d8db185fa54991df09298b64f4fef28879e6c9395759265e8183977 -R 2e4222d820aa06549d33319e9e33c627 +P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba +R 22dc7c511444b2944fcb40dc7656150e U drh -Z 67298e45eb2d4d992f0cdf5678deffdf +Z df3951acae9acd7f479ba37833a7cfe6 diff --git a/manifest.uuid b/manifest.uuid index d58d2c0dd5..55584bef0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba \ No newline at end of file +90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 390dae5d3b..ae2a92c79b 100644 --- a/src/build.c +++ b/src/build.c @@ -1976,6 +1976,7 @@ void sqlite3EndTable( sqlite3DeleteTable(db, pSelTab); sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); sqlite3Select(pParse, pSelect, &dest); + if( pParse->nErr ) return; sqlite3VdbeEndCoroutine(v, regYield); sqlite3VdbeJumpHere(v, addrTop - 1); addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); diff --git a/test/colname.test b/test/colname.test index cacf91e4e3..358ea77e07 100644 --- a/test/colname.test +++ b/test/colname.test @@ -398,7 +398,7 @@ do_execsql_test colname-9.320 { SELECT name FROM pragma_table_info('t2'); } {Bbb} -# Issue detected by clusterfuzz on 2017-12-24 (Christmas Eve) +# Issue detected by OSSFuzz on 2017-12-24 (Christmas Eve) # caused by check-in https://sqlite.org/src/info/6b2ff26c25 # # Prior to being fixed, the following CREATE TABLE was dereferencing @@ -408,6 +408,15 @@ do_catchsql_test colname-9.400 { CREATE TABLE t4 AS SELECT #0; } {1 {near "#0": syntax error}} +# Issue detected by OSSFuzz on 2017-12-25 (Christmas Day) +# also caused by check-in https://sqlite.org/src/info/6b2ff26c25 +# +# Prior to being fixed, the following CREATE TABLE caused an +# assertion fault. +# +do_catchsql_test colname-9.410 { + CREATE TABLE t5 AS SELECT RAISE(abort,a); +} {1 {RAISE() may only be used within a trigger-program}} # Make sure the quotation marks get removed from the column names # when constructing a new table from an aggregate SELECT. From f6c37dbc8c9ae15689f91e1d9c1e43e7b8737120 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Dec 2017 14:30:44 +0000 Subject: [PATCH 07/15] Add options to speedtest1.c and speed-check.sh for testing performance of floating-point computatations. FossilOrigin-Name: ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 --- manifest | 17 +++++----- manifest.uuid | 2 +- test/speedtest1.c | 76 +++++++++++++++++++++++++++++++++++++++++++-- tool/speed-check.sh | 6 ++++ 4 files changed, 90 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d9715194d3..aa2114d805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassertion\sfault\son\sa\ssyntax\serror\sinput\scaused\sby\scheck-in\n[6b2ff26c25bb9da3].\s\sProblem\sdiscovered\sby\sOSSFuzz. -D 2017-12-25T13:43:54.974 +C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations. +D 2017-12-26T14:30:44.117 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1248,7 +1248,7 @@ F test/speed3.test 694affeb9100526007436334cf7d08f3d74b85ef F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c e44c5fccddcfe916c3bf7fe2f87dcc4b4fd66a0d923eb83515f311212670f267 +F test/speedtest1.c a5faf4cbe5769eee4b721b3875cb3f12520a9b99d9026b1063b47c39603375b8 F test/spellfix.test f9c1f431e2c096c8775fec032952320c0e4700db F test/spellfix2.test dfc8f519a3fc204cb2dfa8b4f29821ae90f6f8c3 F test/spellfix3.test 0f9efaaa502a0e0a09848028518a6fb096c8ad33 @@ -1647,7 +1647,7 @@ F tool/showstat4.c 0682ebea7abf4d3657f53c4a243f2e7eab48eab344ed36a94bb75dcd19a5c F tool/showwal.c ad9d768f96ca6199ad3a8c9562d679680bd032dd01204ea3e5ea6fb931d81847 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/spaceanal.tcl 4bfd19aad7eb3ce0372ef0255f58035e0bba4ff5e9acfd763a10c6fb365c8dec -F tool/speed-check.sh a97ae367e9172a706101901e7caef48f1a14fc8a49053b25e79f6a67296b3412 +F tool/speed-check.sh 9ae425da8819e54e780cf494fc6d8175dfb16e109ae3214a45a5c9bb2b74e2c4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c ecb6542862151c3e6509bbc00509b234562ae81e F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -1687,7 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d49afb8f9804e96662d1e3cadc4c6643908706d848a53d5ed019919c98f2ccba -R 22dc7c511444b2944fcb40dc7656150e -U drh -Z df3951acae9acd7f479ba37833a7cfe6 +P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c +Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17 +R c2447707ab323a591ab8e05be0f0cae9 +U dan +Z 1002781326ca68c114cfc46396055176 diff --git a/manifest.uuid b/manifest.uuid index 55584bef0a..aaa3417a9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c \ No newline at end of file +ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index b92801a208..9342da79bf 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -32,7 +32,7 @@ static const char zHelp[] = " --size N Relative test size. Default=100\n" " --stats Show statistics at the end\n" " --temp N N from 0 to 9. 0: no temp table. 9: all temp tables\n" - " --testset T Run test-set T (main, cte, rtree, orm, debug)\n" + " --testset T Run test-set T (main, cte, rtree, orm, fp, debug)\n" " --trace Turn on SQL tracing\n" " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" @@ -1120,7 +1120,77 @@ void testset_cte(void){ ); speedtest1_run(); speedtest1_end_test(); +} +/* +** Compute a pseudo-random floating point ascii number. +*/ +void speedtest1_random_ascii_fp(char *zFP){ + int x = speedtest1_random(); + int y = speedtest1_random(); + int z; + z = y%10; + if( z<0 ) z = -z; + y /= 10; + sqlite3_snprintf(100,zFP,"%d.%de%d",y,z,x%200); +} + +/* +** A testset for floating-point numbers. +*/ +void testset_fp(void){ + int n; + int i; + char zFP1[100]; + char zFP2[100]; + + n = g.szTest*5000; + speedtest1_begin_test(100, "Fill a table with %d FP values", n*2); + speedtest1_exec("BEGIN"); + speedtest1_exec("CREATE%s TABLE t1(a REAL %s, b REAL %s);", + isTemp(1), g.zNN, g.zNN); + speedtest1_prepare("INSERT INTO t1 VALUES(?1,?2); -- %d times", n); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + n = g.szTest/25 + 2; + speedtest1_begin_test(110, "%d range queries", n); + speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2"); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_end_test(); + + speedtest1_begin_test(120, "CREATE INDEX three times"); + speedtest1_exec("BEGIN;"); + speedtest1_exec("CREATE INDEX t1a ON t1(a);"); + speedtest1_exec("CREATE INDEX t1b ON t1(b);"); + speedtest1_exec("CREATE INDEX t1ab ON t1(a,b);"); + speedtest1_exec("COMMIT;"); + speedtest1_end_test(); + + n = g.szTest/3 + 2; + speedtest1_begin_test(130, "%d indexed range queries", n); + speedtest1_prepare("SELECT sum(b) FROM t1 WHERE a BETWEEN ?1 AND ?2"); + for(i=1; i<=n; i++){ + speedtest1_random_ascii_fp(zFP1); + speedtest1_random_ascii_fp(zFP2); + sqlite3_bind_text(g.pStmt, 1, zFP1, -1, SQLITE_STATIC); + sqlite3_bind_text(g.pStmt, 2, zFP2, -1, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_end_test(); } #ifdef SQLITE_ENABLE_RTREE @@ -1873,6 +1943,8 @@ int main(int argc, char **argv){ testset_orm(); }else if( strcmp(zTSet,"cte")==0 ){ testset_cte(); + }else if( strcmp(zTSet,"fp")==0 ){ + testset_fp(); }else if( strcmp(zTSet,"rtree")==0 ){ #ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); @@ -1881,7 +1953,7 @@ int main(int argc, char **argv){ "the R-Tree tests\n"); #endif }else{ - fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n", + fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree fp\n", zTSet); } speedtest1_final(); diff --git a/tool/speed-check.sh b/tool/speed-check.sh index 2cda5c8078..6cc3018981 100644 --- a/tool/speed-check.sh +++ b/tool/speed-check.sh @@ -117,6 +117,12 @@ while test "$1" != ""; do --orm) SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset orm" ;; + --cte) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset cte" + ;; + --fp) + SPEEDTEST_OPTS="$SPEEDTEST_OPTS --testset fp" + ;; -*) CC_OPTS="$CC_OPTS $1" ;; From c8f44d245c5034c8f96361026a49b748ac02a700 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Dec 2017 14:32:25 +0000 Subject: [PATCH 08/15] Fix crashes that could occur if SQL NULL values were passed to the built-in FTS5 snippet function. Edit: breaks amalgamation builds. FossilOrigin-Name: 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a --- ext/fts5/fts5_aux.c | 16 +++++++++++++--- ext/fts5/test/fts5af.test | 10 ++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c index 219ea6fff8..594b981dda 100644 --- a/ext/fts5/fts5_aux.c +++ b/ext/fts5/fts5_aux.c @@ -358,6 +358,16 @@ static int fts5SnippetScore( return rc; } +/* +** Return the value in pVal interpreted as utf-8 text. Except, if pVal +** contains a NULL value, return a pointer to a static string zero +** bytes in length instead of a NULL pointer. +*/ +static const char *fts5ValueToText(sqlite3_value *pVal){ + const char *zRet = (const char*)sqlite3_value_text(pVal); + return zRet ? zRet : ""; +} + /* ** Implementation of snippet() function. */ @@ -393,9 +403,9 @@ static void fts5SnippetFunction( nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); iCol = sqlite3_value_int(apVal[0]); - ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); - ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); - zEllips = (const char*)sqlite3_value_text(apVal[3]); + ctx.zOpen = fts5ValueToText(apVal[1]); + ctx.zClose = fts5ValueToText(apVal[2]); + zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); iBestCol = (iCol>=0 ? iCol : 0); diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test index fa4ebd2955..86c8f753fa 100644 --- a/ext/fts5/test/fts5af.test +++ b/ext/fts5/test/fts5af.test @@ -175,6 +175,16 @@ do_execsql_test 5.1 { SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x'); } {{[x] a a a a a...}} +do_execsql_test 5.2 { + SELECT snippet(p1, 0, '[', ']', NULL, 6) FROM p1('x'); +} {{[x] a a a a a}} +do_execsql_test 5.3 { + SELECT snippet(p1, 0, NULL, ']', '...', 6) FROM p1('x'); +} {{x] a a a a a...}} +do_execsql_test 5.4 { + SELECT snippet(p1, 0, '[', NULL, '...', 6) FROM p1('x'); +} {{[x a a a a a...}} + } ;# foreach_detail_mode finish_test diff --git a/manifest b/manifest index aa2114d805..f3f08454e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\soptions\sto\sspeedtest1.c\sand\sspeed-check.sh\sfor\stesting\sperformance\sof\nfloating-point\scomputatations. -D 2017-12-26T14:30:44.117 +C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds. +D 2017-12-26T14:32:25.807 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -106,7 +106,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 F ext/fts5/fts5Int.h eda28e3a0a5d87c412e8355fe35da875b04cb389908c8eb0d867ad662adbc491 -F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 +F ext/fts5/fts5_aux.c ca666a3bbe07c5a3bbe9fffaea19c935a1efaf337333e28bad7bdd1971ffd093 F ext/fts5/fts5_buffer.c 1dd1ec0446b3acfc2d7d407eb894762a461613e2695273f48e449bfd13e973ff F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 F ext/fts5/fts5_expr.c 01048018d21524e2c302b063ff5c3cdcf546e03297215e577205d85b47499deb @@ -129,7 +129,7 @@ F ext/fts5/test/fts5ab.test 9205c839332c908aaad2b01ab8670ece8b161e8f2ec8a9fabf18 F ext/fts5/test/fts5ac.test a7aa7e1fefc6e1918aa4d3111d5c44a09177168e962c5fd2cca9620de8a7ed6d F ext/fts5/test/fts5ad.test e8cf959dfcd57c8e46d6f5f25665686f3b6627130a9a981371dafdf6482790de F ext/fts5/test/fts5ae.test 1142d16d9cc193894dc13cc8f9c7a8a21411ac61b5567a878514df6f9f0d7bb7 -F ext/fts5/test/fts5af.test aa635947bda31ac87fbe99483eef4d9a8571f58ad89c75dfb63312a35688eceb +F ext/fts5/test/fts5af.test 724247405b13f8f06cc6ce464dc4f152dc5dd4e86b12c2099685d8f19747bf7b F ext/fts5/test/fts5ag.test 7816f25a0707578f08145ab539fc0ca025f8951e788b28a6a18a06b2099469dd F ext/fts5/test/fts5ah.test 27b5a33bfd0363ca8a4dc659e6e2a5df3dea1c3c5b04bc51ca6aeb1277bd9b21 F ext/fts5/test/fts5ai.test d837c42249c0d8ad1a2912270e22cf2f303790a611f85c0be3a58e42a3696e3d @@ -1687,8 +1687,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 90d6e4f10d3055a776d24854c442a2a68e726af8fc382cdb6241a834082e4c4c -Q +b3c6105181d22c8a53f5bf662e3d182bba962ee8afd1ad972ebd10094b20fc17 -R c2447707ab323a591ab8e05be0f0cae9 +P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 +Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d +R 742f62796704fae3ae9dc15618c72290 U dan -Z 1002781326ca68c114cfc46396055176 +Z ecb93df9cf8c6f4bd1c8f7a6ca97d753 diff --git a/manifest.uuid b/manifest.uuid index aaa3417a9c..9c54144fee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 \ No newline at end of file +6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a \ No newline at end of file From 02a43f6defda835b5baaaee274923f1b80d31ceb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Dec 2017 14:46:20 +0000 Subject: [PATCH 09/15] Faster and smaller implementation of sqlite3AtoF() based on a suggestion from Cezary H. Noweta. FossilOrigin-Name: fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/util.c | 26 ++++++++++++++++++++------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f3f08454e9..3d3fdb11e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scrashes\sthat\scould\soccur\sif\sSQL\sNULL\svalues\swere\spassed\sto\sthe\sbuilt-in\nFTS5\ssnippet\sfunction.\sEdit:\sbreaks\samalgamation\sbuilds. -D 2017-12-26T14:32:25.807 +C Faster\sand\ssmaller\simplementation\sof\ssqlite3AtoF()\sbased\son\sa\ssuggestion\nfrom\sCezary\sH.\sNoweta. +D 2017-12-26T14:46:20.800 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -545,7 +545,7 @@ F src/treeview.c eae35972ff44f67064de2eaf35f04afe94e7aea3271a8b3bcebb3f954880fec F src/trigger.c 775053eecf6b73062e243404b56f5064446254d5cce17d8704d5cdffd72a546a F src/update.c 961bd1265d4d1e5cd65c9a54fa5122fb7aefcb003fcf2de0c092fceb7e58972c F src/utf.c 810fbfebe12359f10bc2a011520a6e10879ab2a163bcb26c74768eab82ea62a5 -F src/util.c d01fa6f45bfad3b65fb2490513aa2e0676412c61b4b094340b513cf72c3704a4 +F src/util.c 7315e97a8dc2c8e19ca64196c652cf0a65d13fd0a211b2cec082062372dc6261 F src/vacuum.c 90839322fd5f00df9617eb21b68beda9b6e2a2937576b0d65985e4aeb1c53739 F src/vdbe.c 3393b508d9ad084ffce232a7c53e375ef5ac99b50b685c5131fcdfce97a9d534 F src/vdbe.h d50cadf12bcf9fb99117ef392ce1ea283aa429270481426b6e8b0280c101fd97 @@ -1687,8 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ebfea8728fec955b1d74b1d0a3de498fd1a32e8b39572a8fdab606ed87b169b4 -Q +553a3ad32498ddda920216cd44a376a439a58fbb326d2d3800528867db1ffa9d -R 742f62796704fae3ae9dc15618c72290 -U dan -Z ecb93df9cf8c6f4bd1c8f7a6ca97d753 +P 6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a +R c3240df89b3aa2504611bdd745085460 +U drh +Z 3dc20c6e2c72c6078d443025934f152b diff --git a/manifest.uuid b/manifest.uuid index 9c54144fee..bdfe92339c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a790b67a0a5c698526db16ea262b13ecdd1b6ca74e80bdccfcad88ddbdc933a \ No newline at end of file +fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa \ No newline at end of file diff --git a/src/util.c b/src/util.c index a4dbe8fdaf..75de4b3b30 100644 --- a/src/util.c +++ b/src/util.c @@ -320,6 +320,24 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } +/* +** Compute 10 to the E-th power. Examples: E==1 results in 10. +** E==2 results in 100. E==50 results in 1.0e50. +** +** This routine only works for values of E between 1 and 341. +*/ +static LONGDOUBLE_TYPE sqlite3Pow10(int E){ + LONGDOUBLE_TYPE x = 10.0; + LONGDOUBLE_TYPE r = 1.0; + while(1){ + if( E & 1 ) r *= x; + E >>= 1; + if( E==0 ) break; + x *= x; + } + return r; +} + /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. @@ -475,11 +493,10 @@ do_atof_calc: if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ result = (double)s; }else{ - LONGDOUBLE_TYPE scale = 1.0; /* attempt to handle extremely small/large numbers better */ if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } + LONGDOUBLE_TYPE scale = sqlite3Pow10(e-308); if( esign<0 ){ result = s / scale; result /= 1.0e+308; @@ -499,10 +516,7 @@ do_atof_calc: } } }else{ - /* 1.0e+22 is the largest power of 10 than can be - ** represented exactly. */ - while( e%22 ) { scale *= 1.0e+1; e -= 1; } - while( e>0 ) { scale *= 1.0e+22; e -= 22; } + LONGDOUBLE_TYPE scale = sqlite3Pow10(e); if( esign<0 ){ result = s / scale; }else{ From 0d9de99c5cbc065fa0243f1a23ab79081e05fea4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Dec 2017 18:04:23 +0000 Subject: [PATCH 10/15] Add support for measuring and reporting coverage of the parser state machine using the SQLITE_TESTCTRL_PARSER_COVERAGE test-control. FossilOrigin-Name: 1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/main.c | 16 ++++++++++++++++ src/shell.c.in | 11 +++++++++++ src/sqlite.h.in | 3 ++- src/sqliteInt.h | 3 +++ tool/lemon.c | 1 + tool/lempar.c | 37 +++++++++++++++++++++++++++++++++++-- 8 files changed, 80 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d9f3940ae0..7f1e56cfbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sLEMON\sso\sthat\sit\sgenerates\sthe\saction\stable\sin\ssuch\sa\sway\sthat\sno\nrange\scheck\sis\sneeded\son\sthe\slookahead\stable\sto\sverify\sthat\sthe\snext\sinput\ntoken\sis\svalid.\s\sThis\smakes\sthe\slookahead\stable\sslightly\slarger\s(about\s120\nbytes)\sbut\shelps\sthe\sparser\sto\srun\sfaster. -D 2017-12-25T04:15:38.668 +C Add\ssupport\sfor\smeasuring\sand\sreporting\scoverage\sof\sthe\sparser\sstate\smachine\nusing\sthe\sSQLITE_TESTCTRL_PARSER_COVERAGE\stest-control. +D 2017-12-26T18:04:23.951 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -443,7 +443,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c cb67cc56ef2ddd13e6944b2c0dd08a920bcd9503230adef8b9928d338097c722 F src/legacy.c 134ab3e3fae00a0f67a5187981d6935b24b337bcf0f4b3e5c9fa5763da95bf4e F src/loadext.c 55bcc3c741059a1056859e8adaf133aa179e22be12215c0936b2f354ef71209b -F src/main.c 7ce55fa3c0bf669944de309ebab1655ed06ec67869adb0372c7a1062e461c448 +F src/main.c 690c4134f944cbd5b71d59dd6e61ce4131f6a50ab774f38108e57d07d79cf876 F src/malloc.c a02c9e69bc76bee0f639416b947a946412890b606301454727feadcb313536d6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de @@ -479,11 +479,11 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c bbee7e31d369a18a2f4836644769882e9c5d40ef4a3af911db06410b65cb3730 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 8b22abe193e4d8243befa2038e4ae2405802fed1c446e5e502d11f652e09ba74 -F src/shell.c.in 339169a3d1307b5566ebe9ce15832d03439206106724c78cc3d9125a7b851795 -F src/sqlite.h.in 2126192945019d4cdce335cb236b440a05ec75c93e4cd94c9c6d6e7fcc654cc4 +F src/shell.c.in f3ec8f90dd698ea98781a90642c91eacbc24f4e55bb551c7b2762000d3ef55dc +F src/sqlite.h.in b4dc75265ed04b98e2184011a7dd0054ce2137ff84867a6be8b4f3bdfbc03d30 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h c02d628cca67f3889c689d82d25c3eb45e2c155db08e4c6089b5840d64687d34 -F src/sqliteInt.h 003b78433baae4e5c997f99f2f9cf98d90754f256baeacb32f8189569a48251f +F src/sqliteInt.h 26bf7cc7aaa6a6eb3d3c0581f9ef5523da01babb8de1d796793d5165fea95958 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b F src/status.c 9737ed017279a9e0c5da748701c3c7bf1e8ae0dae459aad20dd64fcff97a7e35 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34 @@ -1608,8 +1608,8 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c c8d7ce4fe7a90f7fa6a5985452aa926fcf25376cf90095c9d06c432ab0bebdbc -F tool/lempar.c 427ee280f3c3781e82bbee21f428bc8ae18ab245d4f66d65da46b598ded81648 +F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a +F tool/lempar.c 2a688bf343f21c25908cf082d0d707f45bc4416eea8550787809da4025c3d7c6 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P fdbb35c54f2b6cb65d04ac295f207ff3e69360e0558348c77eb5e62691807046 -R f14cbf100834d359550a23c924f7ebaa +P 7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814 +R e80238b7e2c8eabf5efe2bfae85580fc U drh -Z bb1d7d0db816330a6f1e2b8011c2819b +Z d5a5596d3c74652b6ba118722fb4ea75 diff --git a/manifest.uuid b/manifest.uuid index b69ed5a1cc..dce691ea27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7eb0198d0102e97e4b7ad9e359d95985e55e09c510ea4b360265ac8feb9ed814 \ No newline at end of file +1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 32ce1889f6..3c8035c120 100644 --- a/src/main.c +++ b/src/main.c @@ -3911,6 +3911,22 @@ int sqlite3_test_control(int op, ...){ sqlite3_mutex_leave(db->mutex); break; } + +#if defined(YYCOVERAGE) + /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) + ** + ** This test control (only available when SQLite is compiled with + ** -DYYCOVERAGE) writes a report onto "out" that shows all + ** state/lookahead combinations in the parser state machine + ** which are never exercised. If any state is missed, make the + ** return code SQLITE_ERROR. + */ + case SQLITE_TESTCTRL_PARSER_COVERAGE: { + FILE *out = va_arg(ap, FILE*); + if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; + break; + } +#endif /* defined(YYCOVERAGE) */ } va_end(ap); #endif /* SQLITE_UNTESTABLE */ diff --git a/src/shell.c.in b/src/shell.c.in index 13b1fcde39..155ca708a5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6108,6 +6108,9 @@ static int do_meta_command(char *zLine, ShellState *p){ { "localtime_fault", SQLITE_TESTCTRL_LOCALTIME_FAULT,"BOOLEAN" }, { "never_corrupt", SQLITE_TESTCTRL_NEVER_CORRUPT, "BOOLEAN" }, { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS, "DISABLE-MASK" }, +#ifdef YYCOVERAGE + { "parser_coverage", SQLITE_TESTCTRL_PARSER_COVERAGE, "" }, +#endif { "pending_byte", SQLITE_TESTCTRL_PENDING_BYTE, "OFFSET " }, { "prng_reset", SQLITE_TESTCTRL_PRNG_RESET, "" }, { "prng_restore", SQLITE_TESTCTRL_PRNG_RESTORE, "" }, @@ -6233,6 +6236,14 @@ static int do_meta_command(char *zLine, ShellState *p){ isOk = 3; } break; + +#ifdef YYCOVERAGE + case SQLITE_TESTCTRL_PARSER_COVERAGE: + if( nArg==2 ){ + sqlite3_test_control(testctrl, p->out); + isOk = 3; + } +#endif } } if( isOk==0 && iCtrl>=0 ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f161eea6f2..7e5fccdae6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7042,7 +7042,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 -#define SQLITE_TESTCTRL_LAST 25 /* Largest TESTCTRL */ +#define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +#define SQLITE_TESTCTRL_LAST 26 /* Largest TESTCTRL */ /* ** CAPI3REF: SQLite Runtime Status diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b0c4711b03..074d363d48 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -4348,6 +4348,9 @@ void sqlite3Put4byte(u8*, u32); #ifdef SQLITE_DEBUG void sqlite3ParserTrace(FILE*, char *); #endif +#if defined(YYCOVERAGE) + int sqlite3ParserCoverage(FILE*); +#endif /* ** If the SQLITE_ENABLE IOTRACE exists then the global variable diff --git a/tool/lemon.c b/tool/lemon.c index 96dc756d1b..111082c72a 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -4257,6 +4257,7 @@ void ReportTable( ** been computed */ fprintf(out,"#define YYNSTATE %d\n",lemp->nxstate); lineno++; fprintf(out,"#define YYNRULE %d\n",lemp->nrule); lineno++; + fprintf(out,"#define YYNTOKEN %d\n",lemp->nterminal); lineno++; fprintf(out,"#define YY_MAX_SHIFT %d\n",lemp->nxstate-1); lineno++; i = lemp->minShiftReduce; fprintf(out,"#define YY_MIN_SHIFTREDUCE %d\n",i); lineno++; diff --git a/tool/lempar.c b/tool/lempar.c index 7ceaaa6207..29dc15d03d 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -72,6 +72,7 @@ ** defined, then do no error processing. ** YYNSTATE the combined number of states. ** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols ** YY_MAX_SHIFT Maximum value for shift actions ** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions ** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions @@ -253,13 +254,13 @@ void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ } #endif /* NDEBUG */ -#ifndef NDEBUG +#if defined(YYCOVERAGE) || !defined(NDEBUG) /* For tracing shifts, the names of all terminals and nonterminals ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { %% }; -#endif /* NDEBUG */ +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. @@ -455,6 +456,35 @@ int ParseStackPeak(void *p){ } #endif +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** has not previously been seen by the parser. Return the number of +** missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +int ParseCoverage(FILE *out){ + int i, j; + int nMissed = 0; + for(i=0; iYY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif do{ i = yy_shift_ofst[stateno]; assert( iLookAhead!=YYNOCODE ); From 22716cbb0baa9b07231b033247a7cae0f1ae0c3a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Dec 2017 18:32:06 +0000 Subject: [PATCH 11/15] Change the lemon-parser coverage report format to report all state/lookahead pairs and indicate on each whether it is hit or missed. FossilOrigin-Name: 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9adf601335..ffec829b50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sbug\sfixes\sfrom\strunk. -D 2017-12-26T18:14:53.001 +C Change\sthe\slemon-parser\scoverage\sreport\sformat\sto\sreport\sall\sstate/lookahead\npairs\sand\sindicate\son\seach\swhether\sit\sis\shit\sor\smissed. +D 2017-12-26T18:32:06.041 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a -F tool/lempar.c 2a688bf343f21c25908cf082d0d707f45bc4416eea8550787809da4025c3d7c6 +F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1253a872dbf48656d4efd588ab61223a5ac550d9b2b932249d6ba585276ba573 fd2e0e7a770c2ce9355068aad1024c3d2861c104fd3be304a91c55ca742155fa -R 2e2b89187d0becf4c2715a566c5e3287 +P d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98 +R 930f1a264ff138489be35924f879ef78 U drh -Z ade3bde65575a1ed1ca48cdbaee1adc1 +Z 2620b1a658dd06210670b65491310e1c diff --git a/manifest.uuid b/manifest.uuid index e738275177..97230d7170 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98 \ No newline at end of file +86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700 \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index 29dc15d03d..5404fa5f5f 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -476,9 +476,11 @@ int ParseCoverage(FILE *out){ int nMissed = 0; for(i=0; i Date: Wed, 27 Dec 2017 15:21:16 +0000 Subject: [PATCH 12/15] In LEMON, fix an off-by-one error that can make the lookahead table one byte too smal. FossilOrigin-Name: 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ffec829b50..784faa63b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\slemon-parser\scoverage\sreport\sformat\sto\sreport\sall\sstate/lookahead\npairs\sand\sindicate\son\seach\swhether\sit\sis\shit\sor\smissed. -D 2017-12-26T18:32:06.041 +C In\sLEMON,\sfix\san\soff-by-one\serror\sthat\scan\smake\sthe\slookahead\stable\sone\nbyte\stoo\ssmal. +D 2017-12-27T15:21:16.478 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1608,7 +1608,7 @@ F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c d92031adf63377ff5055441c9d26a41a03378e8cac8dbcc1cd6ef7190f51aa6a +F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d76e12066fa6950f877cbe33b1892e2b0afa948978815d5b9a90cde1fff8ec98 -R 930f1a264ff138489be35924f879ef78 +P 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700 +R 4a4c12385039bdde527999ce093c45db U drh -Z 2620b1a658dd06210670b65491310e1c +Z dbb131aa448f925318d1af4566b5714c diff --git a/manifest.uuid b/manifest.uuid index 97230d7170..d4b1dea234 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700 \ No newline at end of file +93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 111082c72a..96bbed7473 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -766,7 +766,7 @@ int acttab_insert(acttab *p, int makeItSafe){ p->aAction[k] = p->aLookahead[j]; if( k>=p->nAction ) p->nAction = k+1; } - if( makeItSafe && i+p->nterminal>p->nAction ) p->nAction = i+p->nterminal; + if( makeItSafe && i+p->nterminal>=p->nAction ) p->nAction = i+p->nterminal+1; p->nLookahead = 0; /* Return the offset that is added to the lookahead in order to get the From 7e7b753158f6c7fa0442f6609179df2512da2633 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Dec 2017 16:13:22 +0000 Subject: [PATCH 13/15] In the lemon-generated parser, do not report the End-of-input character and the wildcard character as missed coverage. FossilOrigin-Name: 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 784faa63b2..74d89bf960 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sLEMON,\sfix\san\soff-by-one\serror\sthat\scan\smake\sthe\slookahead\stable\sone\nbyte\stoo\ssmal. -D 2017-12-27T15:21:16.478 +C In\sthe\slemon-generated\sparser,\sdo\snot\sreport\sthe\sEnd-of-input\scharacter\sand\nthe\swildcard\scharacter\sas\smissed\scoverage. +D 2017-12-27T16:13:22.261 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 -F tool/lempar.c 64e760f7cc21ea7e73513b062a86a3c478dfd1ed41fe9837a6f49958cdb61e03 +F tool/lempar.c 48ca9d9f280762da24d667f423c5421e09a6839d08a7d6a516d009db22974deb F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 86e30fc284c740b55f75884b67988fe837b28878f586f6ec8850ecf80164e700 -R 4a4c12385039bdde527999ce093c45db +P 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e +R 930e0d52d43acba766af53475563b556 U drh -Z dbb131aa448f925318d1af4566b5714c +Z 594a8d4b6a5a5a1f42055bf091b10e50 diff --git a/manifest.uuid b/manifest.uuid index d4b1dea234..5d9f174839 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e \ No newline at end of file +3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1 \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index 5404fa5f5f..e5dd50cfb7 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -475,7 +475,8 @@ int ParseCoverage(FILE *out){ int i, j; int nMissed = 0; for(i=0; i Date: Wed, 27 Dec 2017 17:14:50 +0000 Subject: [PATCH 14/15] Change the coverage measurement logic in the lemon-generated parser so that it only checks for coverage of state/lookahead pairs that are valid syntax. It turns out that some states are unreachable if the lookahead is not valid syntax, because the states are only reachable through a shift following a reduce, and the reduce does not happen if the lookahead is a syntax error. FossilOrigin-Name: 9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 26 ++++++++++++++++---------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 74d89bf960..cb95612f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\slemon-generated\sparser,\sdo\snot\sreport\sthe\sEnd-of-input\scharacter\sand\nthe\swildcard\scharacter\sas\smissed\scoverage. -D 2017-12-27T16:13:22.261 +C Change\sthe\scoverage\smeasurement\slogic\sin\sthe\slemon-generated\sparser\sso\sthat\nit\sonly\schecks\sfor\scoverage\sof\sstate/lookahead\spairs\sthat\sare\svalid\ssyntax.\nIt\sturns\sout\sthat\ssome\sstates\sare\sunreachable\sif\sthe\slookahead\sis\snot\svalid\nsyntax,\sbecause\sthe\sstates\sare\sonly\sreachable\sthrough\sa\sshift\sfollowing\sa\nreduce,\sand\sthe\sreduce\sdoes\snot\shappen\sif\sthe\slookahead\sis\sa\ssyntax\serror. +D 2017-12-27T17:14:50.938 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 -F tool/lempar.c 48ca9d9f280762da24d667f423c5421e09a6839d08a7d6a516d009db22974deb +F tool/lempar.c 1b1279a362b7045b4e3214301665c12e32689e2c9caf33b35587669bac6b22fa F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 93792bc58a2eccc7e07b14307388350bb376db32c5055b79a44e4fa8ff91d58e -R 930e0d52d43acba766af53475563b556 +P 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1 +R 43f297c7f87ea228ee76b1c746f9c4a6 U drh -Z 594a8d4b6a5a5a1f42055bf091b10e50 +Z 38efd78b8cbae26fc786eb4139191684 diff --git a/manifest.uuid b/manifest.uuid index 5d9f174839..8cb561dc1a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1 \ No newline at end of file +9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index e5dd50cfb7..97fd421334 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -467,20 +467,25 @@ static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; /* ** Write into out a description of every state/lookahead combination that -** has not previously been seen by the parser. Return the number of -** missed state/lookahead combinations. +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. */ #if defined(YYCOVERAGE) int ParseCoverage(FILE *out){ - int i, j; + int stateno, iLookAhead, i; int nMissed = 0; - for(i=0; i=0 && i+YYNTOKEN<=sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - assert( i>=0 && i Date: Wed, 27 Dec 2017 17:36:58 +0000 Subject: [PATCH 15/15] The previous check-in had an error in the coverage reporting logic. FossilOrigin-Name: ec9b19eb652e81cd3c8a5595eb39aac3aca4abe72136c4b65c9cc257594a9c92 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lempar.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index cb95612f6f..2ca894e566 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\scoverage\smeasurement\slogic\sin\sthe\slemon-generated\sparser\sso\sthat\nit\sonly\schecks\sfor\scoverage\sof\sstate/lookahead\spairs\sthat\sare\svalid\ssyntax.\nIt\sturns\sout\sthat\ssome\sstates\sare\sunreachable\sif\sthe\slookahead\sis\snot\svalid\nsyntax,\sbecause\sthe\sstates\sare\sonly\sreachable\sthrough\sa\sshift\sfollowing\sa\nreduce,\sand\sthe\sreduce\sdoes\snot\shappen\sif\sthe\slookahead\sis\sa\ssyntax\serror. -D 2017-12-27T17:14:50.938 +C The\sprevious\scheck-in\shad\san\serror\sin\sthe\scoverage\sreporting\slogic. +D 2017-12-27T17:36:58.482 F Makefile.in ceb40bfcb30ebba8e1202b34c56ff7e13e112f9809e2381d99be32c2726058f5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 6480671f7c129e61208d69492b3c71ce4310d49fceac83cfb17f1c081e242b69 @@ -1609,7 +1609,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f F tool/lemon.c 7f7735326ca9c3b48327b241063cee52d35d44e20ebe1b3624a81658052a4d39 -F tool/lempar.c 1b1279a362b7045b4e3214301665c12e32689e2c9caf33b35587669bac6b22fa +F tool/lempar.c dddd4f592b8bad36aec4500d456c5db5fe42fefc4ee384913880439d8917f87a F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca @@ -1687,7 +1687,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3fe964873da16c0e0b1c4f1945f965d4137df7a307acd6a3eb6585ffbaa2afd1 -R 43f297c7f87ea228ee76b1c746f9c4a6 +P 9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b +R 0e64cfa85d322f92630985625ab9cde3 U drh -Z 38efd78b8cbae26fc786eb4139191684 +Z afc4d6b44fe600f4cffd568a58de353c diff --git a/manifest.uuid b/manifest.uuid index 8cb561dc1a..3498a34465 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9dce46508772bd0f9e940c4d44933154044bb58c1b3511dd0143287bf795dd6b \ No newline at end of file +ec9b19eb652e81cd3c8a5595eb39aac3aca4abe72136c4b65c9cc257594a9c92 \ No newline at end of file diff --git a/tool/lempar.c b/tool/lempar.c index 97fd421334..9164eb0c1e 100644 --- a/tool/lempar.c +++ b/tool/lempar.c @@ -480,7 +480,7 @@ int ParseCoverage(FILE *out){ for(stateno=0; stateno