mirrors
/
sqlite
mirror of https://github.com/sqlite/sqlite
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Performance enhancement to the tokenizer. 

FossilOrigin-Name: 6ea2df86c95c226052f528424e9bee367a2e765a
This commit is contained in:
drh 2015-11-10 03:30:51 +00:00
parent 07b09a9420
commit 54bb56d82a
4 changed files with 30 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
manifest
View File

@ -1,5 +1,5 @@
C Remove\san\sunused\snon-terminal\sfrom\sthe\sgrammar. C Performance\senhancement\sto\sthe\stokenizer.
D 2015-11-10T00:02:49.412 D 2015-11-10T03:30:51.926
F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1 F Makefile.in d828db6afa6c1fa060d01e33e4674408df1942a1
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc e928e68168df69b353300ac87c10105206653a03 F Makefile.msc e928e68168df69b353300ac87c10105206653a03
@ -395,7 +395,7 @@ F src/test_vfs.c 3b65d42e18b262805716bd96178c81da8f2d9283
F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0 F src/threads.c bbfb74450643cb5372a43ad4f6cffd7e9dfcecb0
F src/tokenize.c c1006aa773da5725ef55e8d48f69c11d7141d011 F src/tokenize.c 5606871a377f390af7040ec3c12e0d183512d785
F src/treeview.c 78842e90c1f71269e7a73a1d4221b6fe360bab66 F src/treeview.c 78842e90c1f71269e7a73a1d4221b6fe360bab66
F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f F src/trigger.c 322f23aad694e8f31d384dcfa386d52a48d3c52f
F src/update.c 40e51cd0883cb5bfd6abb7d8a7cd8aa47fab2945 F src/update.c 40e51cd0883cb5bfd6abb7d8a7cd8aa47fab2945
@ -1360,7 +1360,7 @@ F tool/lempar.c 02cd882bd3144a5e25c86c652af49a6bc07baeae
F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
F tool/mkautoconfamal.sh 4bdf61548a143e5977bd86ab93d68b694d10c8fa F tool/mkautoconfamal.sh 4bdf61548a143e5977bd86ab93d68b694d10c8fa
F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkkeywordhash.c 37f9c2e62c31384b697ede8825e2d344e98db22c
F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b F tool/mkopcodec.tcl edde8adc42621b5e598127f8cdc6d52cfe21f52b
F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e F tool/mkopcodeh.tcl e04177031532b7aa9379ded50e820231ac4abd6e
F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
@ -1402,7 +1402,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P c4a7e93fca622fd11a6e16161fbd2f39c2575f00 P 3c37c522883ea9f2eec4f0ba5c5141912c003425
R 718dc1350ed033f1faf4636e4683af75 R 40750d96ad92c103526c6bd35096918c
U drh U drh
Z ed4dab40ca0663dda7ca06a05db43ba5 Z 8ead9b7802f9e64a677105e8e10b5cbd

2
manifest.uuid
View File

@ -1 +1 @@
3c37c522883ea9f2eec4f0ba5c5141912c003425 6ea2df86c95c226052f528424e9bee367a2e765a

4
src/tokenize.c
View File

@ -369,8 +369,8 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
break; break;
} }
for(i=1; IdChar(z[i]); i++){} for(i=1; IdChar(z[i]); i++){}
*tokenType = keywordCode((char*)z, i); *tokenType = TK_ID;
return i; return keywordCode((char*)z, i, tokenType);
} }
} }
*tokenType = TK_ILLEGAL; *tokenType = TK_ILLEGAL;

48
tool/mkkeywordhash.c
View File

@ -277,27 +277,8 @@ static Keyword aKeywordTable[] = {
/* Number of keywords */ /* Number of keywords */
static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0])); static int nKeyword = (sizeof(aKeywordTable)/sizeof(aKeywordTable[0]));
/* An array to map all upper-case characters into their corresponding /* Map all alphabetic characters into the same case */
** lower-case character. #define charMap(X) (0x20|(X))
*/
const unsigned char sqlite3UpperToLower[] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103,
104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,
122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107,
108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,
126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,
144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,
162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,
180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,
198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,
216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,
234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,
252,253,254,255
};
#define UpperToLower sqlite3UpperToLower
/* /*
** Comparision function for two Keyword records ** Comparision function for two Keyword records
@ -347,7 +328,7 @@ static Keyword *findById(int id){
** output. ** output.
*/ */
int main(int argc, char **argv){ int main(int argc, char **argv){
int i, j, k, h; int i, j, k, h, m;
int bestSize, bestCount; int bestSize, bestCount;
int count; int count;
int nChar; int nChar;
@ -372,8 +353,8 @@ int main(int argc, char **argv){
assert( p->len<sizeof(p->zOrigName) ); assert( p->len<sizeof(p->zOrigName) );
memcpy(p->zOrigName, p->zName, p->len+1); memcpy(p->zOrigName, p->zName, p->len+1);
totalLen += p->len; totalLen += p->len;
p->hash = (UpperToLower[(int)p->zName[0]]*4) ^ p->hash = (charMap(p->zName[0])*4) ^
(UpperToLower[(int)p->zName[p->len-1]]*3) ^ p->len; (charMap(p->zName[p->len-1])*3) ^ (p->len*1);
p->id = i+1; p->id = i+1;
} }
@ -481,7 +462,7 @@ int main(int argc, char **argv){
/* Begin generating code */ /* Begin generating code */
printf("%s", zHdr); printf("%s", zHdr);
printf("/* Hash score: %d */\n", bestCount); printf("/* Hash score: %d */\n", bestCount);
printf("static int keywordCode(const char *z, int n){\n"); printf("static int keywordCode(const char *z, int n, int *pType){\n");
printf(" /* zText[] encodes %d bytes of keywords in %d bytes */\n", printf(" /* zText[] encodes %d bytes of keywords in %d bytes */\n",
totalLen + nKeyword, nChar+1 ); totalLen + nKeyword, nChar+1 );
for(i=j=k=0; i<nKeyword; i++){ for(i=j=k=0; i<nKeyword; i++){
@ -585,10 +566,9 @@ int main(int argc, char **argv){
printf("%s };\n", j==0 ? "" : "\n"); printf("%s };\n", j==0 ? "" : "\n");
printf(" int h, i;\n"); printf(" int h, i;\n");
printf(" if( n<2 ) return TK_ID;\n"); printf(" if( n>=2 ){\n");
printf(" h = ((charMap(z[0])*4) ^\n" printf(" h = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) %% %d;\n",
" (charMap(z[n-1])*3) ^\n" bestSize);
" n) %% %d;\n", bestSize);
printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n"); printf(" for(i=((int)aHash[h])-1; i>=0; i=((int)aNext[i])-1){\n");
printf(" if( aLen[i]==n &&" printf(" if( aLen[i]==n &&"
" sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n"); " sqlite3StrNICmp(&zText[aOffset[i]],z,n)==0 ){\n");
@ -596,13 +576,17 @@ int main(int argc, char **argv){
printf(" testcase( i==%d ); /* %s */\n", printf(" testcase( i==%d ); /* %s */\n",
i, aKeywordTable[i].zOrigName); i, aKeywordTable[i].zOrigName);
} }
printf(" return aCode[i];\n"); printf(" *pType = aCode[i];\n");
printf(" break;\n");
printf(" }\n"); printf(" }\n");
printf(" }\n"); printf(" }\n");
printf(" return TK_ID;\n"); printf(" }\n");
printf(" return n;\n");
printf("}\n"); printf("}\n");
printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n"); printf("int sqlite3KeywordCode(const unsigned char *z, int n){\n");
printf(" return keywordCode((char*)z, n);\n"); printf(" int id = TK_ID;\n");
printf(" keywordCode((char*)z, n, &id);\n");
printf(" return id;\n");
printf("}\n"); printf("}\n");
printf("#define SQLITE_N_KEYWORD %d\n", nKeyword); printf("#define SQLITE_N_KEYWORD %d\n", nKeyword);