From 61f92cd045319f19e12125d2f49bb3d521c0934c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Jan 2014 03:06:18 +0000 Subject: [PATCH] In LEMON, fix a bug in the text formatter introduced by the previous commit. Also add the new "%token_class" directive for defining symbolic names that stand any one of a collection of tokens. FossilOrigin-Name: da7890ca6b1d8e511377a469047120220e8c3b2d --- manifest | 12 ++--- manifest.uuid | 2 +- tool/lemon.c | 118 ++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 97 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index edfa17a7d1..b56fe09692 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\ssprintf(),\sstrcpy()\sor\sstrcat()\sin\sthe\simplementation\sof\sthe\nlemon\sparser\sgenerator\stool,\sto\savoid\scompiler\swarnings\sin\sOpenBSD. -D 2014-01-10T23:21:00.243 +C In\sLEMON,\sfix\sa\sbug\sin\sthe\stext\sformatter\sintroduced\sby\sthe\sprevious\ncommit.\s\sAlso\sadd\sthe\snew\s"%token_class"\sdirective\sfor\sdefining\ssymbolic\nnames\sthat\sstand\sany\sone\sof\sa\scollection\sof\stokens. +D 2014-01-11T03:06:18.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 04f60c891f6c07643221b068bbc2621be46c039a +F tool/lemon.c 4a3d4a579c5dff6a42785e97d1f2b59789f3b8dd F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7f1e7ae313c7625ef2623d78883dce776eecca30 -R 153a0201d8a4a180b2a188d3abd169a6 +P e43c522dde01e134f1adc94f534d2b3eda74afc2 +R 498b2eca2a4d9bdf3c22bba6f4386191 U drh -Z c49dfd6bd62bb08d757d02ffce1c49c4 +Z 303f39885598d1865b7eb401105ea9ea diff --git a/manifest.uuid b/manifest.uuid index fdbe371621..2d1234c95c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e43c522dde01e134f1adc94f534d2b3eda74afc2 \ No newline at end of file +da7890ca6b1d8e511377a469047120220e8c3b2d \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 191ab0d88b..25787abf8d 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -71,12 +71,15 @@ static void lemon_addtext( char *zBuf, /* The buffer to which text is added */ int *pnUsed, /* Slots of the buffer used so far */ const char *zIn, /* Text to add */ - int nIn /* Bytes of text to add. -1 to use strlen() */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ ){ if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + while( iWidth>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth--; } if( nIn==0 ) return; memcpy(&zBuf[*pnUsed], zIn, nIn); *pnUsed += nIn; + while( (-iWidth)>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth++; } zBuf[*pnUsed] = 0; } static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ @@ -87,15 +90,22 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ str[0] = 0; for(i=j=0; (c = zFormat[i])!=0; i++){ if( c=='%' ){ - lemon_addtext(str, &nUsed, &zFormat[j], i-j); + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); c = zFormat[++i]; + if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){ + if( c=='-' ) i++; + while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + if( c=='-' ) iWidth = -iWidth; + c = zFormat[i]; + } if( c=='d' ){ int v = va_arg(ap, int); if( v<0 ){ - lemon_addtext(str, &nUsed, "-", 1); + lemon_addtext(str, &nUsed, "-", 1, iWidth); v = -v; }else if( v==0 ){ - lemon_addtext(str, &nUsed, "0", 1); + lemon_addtext(str, &nUsed, "0", 1, iWidth); } k = 0; while( v>0 ){ @@ -103,17 +113,17 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ zTemp[sizeof(zTemp)-k] = (v%10) + '0'; v /= 10; } - lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k); + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); }else if( c=='s' ){ z = va_arg(ap, const char*); - lemon_addtext(str, &nUsed, z, -1); + lemon_addtext(str, &nUsed, z, -1, iWidth); }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ i += 2; k = va_arg(ap, int); z = va_arg(ap, const char*); - lemon_addtext(str, &nUsed, z, k); + lemon_addtext(str, &nUsed, z, k, iWidth); }else if( c=='%' ){ - lemon_addtext(str, &nUsed, "%", 1); + lemon_addtext(str, &nUsed, "%", 1, 0); }else{ fprintf(stderr, "illegal format\n"); exit(1); @@ -121,7 +131,7 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ j = i+1; } } - lemon_addtext(str, &nUsed, &zFormat[j], i-j); + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); return nUsed; } static int lemon_sprintf(char *str, const char *format, ...){ @@ -1538,12 +1548,15 @@ int main(int argc, char **argv) } /* Count and index the symbols of the grammar */ - lem.nsymbol = Symbol_count(); Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); lem.symbols = Symbol_arrayof(); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; - qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; + for(i=0; iindex = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; iindex = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; for(i=1; isupper(lem.symbols[i]->name[0]); i++); lem.nterminal = i; @@ -2031,7 +2044,9 @@ enum e_state { WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, - WAITING_FOR_WILDCARD_ID + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN }; struct pstate { char *filename; /* Name of the input file */ @@ -2041,6 +2056,7 @@ struct pstate { struct lemon *gp; /* Global state vector */ enum e_state state; /* The state of the parser */ struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ struct symbol *lhs; /* Left-hand side of current rule */ const char *lhsalias; /* Alias for the LHS */ int nrhs; /* Number of right-hand side symbols seen */ @@ -2345,6 +2361,8 @@ to follow the previous rule."); psp->state = WAITING_FOR_FALLBACK_ID; }else if( strcmp(x,"wildcard")==0 ){ psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Unknown declaration keyword: \"%%%s\".",x); @@ -2513,6 +2531,40 @@ to follow the previous rule."); } } break; + case WAITING_FOR_CLASS_ID: + if( !islower(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: ", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !isupper(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; case RESYNC_AFTER_RULE_ERROR: /* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; ** break; */ @@ -2867,11 +2919,13 @@ void Reprint(struct lemon *lemp) printf(" ::="); for(i=0; inrhs; i++){ sp = rp->rhs[i]; - printf(" %s", sp->name); if( sp->type==MULTITERMINAL ){ + printf(" %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ printf("|%s", sp->subsym[j]->name); } + }else{ + printf(" %s", sp->name); } /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ } @@ -2893,11 +2947,13 @@ void ConfigPrint(FILE *fp, struct config *cfp) if( i==cfp->dot ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; - fprintf(fp," %s", sp->name); if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ fprintf(fp,"|%s",sp->subsym[j]->name); } + }else{ + fprintf(fp," %s", sp->name); } } } @@ -3644,9 +3700,11 @@ static void writeRuleText(FILE *out, struct rule *rp){ fprintf(out,"%s ::=", rp->lhs->name); for(j=0; jnrhs; j++){ struct symbol *sp = rp->rhs[j]; - fprintf(out," %s", sp->name); - if( sp->type==MULTITERMINAL ){ + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ int k; + fprintf(out," %s", sp->subsym[0]->name); for(k=1; knsubsym; k++){ fprintf(out,"|%s",sp->subsym[k]->name); } @@ -4114,7 +4172,8 @@ void ReportHeader(struct lemon *lemp) if( in ){ int nextChar; for(i=1; interminal && fgets(line,LINESIZE,in); i++){ - lemon_sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); if( strcmp(line,pattern) ) break; } nextChar = fgetc(in); @@ -4127,7 +4186,7 @@ void ReportHeader(struct lemon *lemp) out = file_open(lemp,".h","wb"); if( out ){ for(i=1; interminal; i++){ - fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); } fclose(out); } @@ -4497,11 +4556,15 @@ struct symbol *Symbol_new(const char *x) return sp; } -/* Compare two symbols for working purposes +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. ** ** Symbols that begin with upper case letters (terminals or tokens) ** must sort before symbols that begin with lower case letters -** (non-terminals). Other than that, the order does not matter. +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. ** ** We find experimentally that leaving the symbols in their original ** order (the order they appeared in the grammar file) gives the @@ -4509,12 +4572,11 @@ struct symbol *Symbol_new(const char *x) */ int Symbolcmpp(const void *_a, const void *_b) { - const struct symbol **a = (const struct symbol **) _a; - const struct symbol **b = (const struct symbol **) _b; - int i1 = (**a).index + 10000000*((**a).name[0]>'Z'); - int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); - assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); - return i1-i2; + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; } /* There is one instance of the following structure for each