diff --git a/ext/fts1/fts1_tokenizer1.c b/ext/fts1/fts1_tokenizer1.c index f3e35b1e46..416c9b66ad 100644 --- a/ext/fts1/fts1_tokenizer1.c +++ b/ext/fts1/fts1_tokenizer1.c @@ -29,61 +29,56 @@ #include "fts1_tokenizer.h" -/* Duplicate a string; the caller must free() the returned string. - * (We don't use strdup() since it's not part of the standard C library and - * may not be available everywhere.) */ -/* TODO(shess) Copied from fulltext.c, consider util.c for such -** things. */ -static char *string_dup(const char *s){ - char *str = malloc(strlen(s) + 1); - strcpy(str, s); - return str; -} - typedef struct simple_tokenizer { sqlite3_tokenizer base; - const char *zDelim; /* token delimiters */ + char delim[128]; /* flag ASCII delimiters */ } simple_tokenizer; typedef struct simple_tokenizer_cursor { sqlite3_tokenizer_cursor base; const char *pInput; /* input we are tokenizing */ int nBytes; /* size of the input */ - const char *pCurrent; /* current position in pInput */ + int iOffset; /* current position in pInput */ int iToken; /* index of next token to be returned */ - char *zToken; /* storage for current token */ - int nTokenBytes; /* actual size of current token */ + char *pToken; /* storage for current token */ int nTokenAllocated; /* space allocated to zToken buffer */ } simple_tokenizer_cursor; static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */ +static int isDelim(simple_tokenizer *t, unsigned char c){ + return c<0x80 && t->delim[c]; +} + static int simpleCreate( int argc, const char **argv, sqlite3_tokenizer **ppTokenizer ){ simple_tokenizer *t; - t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer)); + t = (simple_tokenizer *) calloc(sizeof(simple_tokenizer), 1); /* TODO(shess) Delimiters need to remain the same from run to run, ** else we need to reindex. One solution would be a meta-table to ** track such information in the database, then we'd only want this ** information on the initial create. */ if( argc>1 ){ - t->zDelim = string_dup(argv[1]); - } else { - /* Build a string excluding alphanumeric ASCII characters */ - char zDelim[0x80]; /* nul-terminated, so nul not a member */ - int i, j; - for(i=1, j=0; i<0x80; i++){ - if( !isalnum(i) ){ - zDelim[j++] = i; + int i, n = strlen(argv[1]); + for(i=0; i=0x80 ){ + free(t); + return SQLITE_ERROR; } + t->delim[ch] = 1; + } + } else { + /* Mark non-alphanumeric ASCII characters as delimiters */ + int i; + for(i=1; i<0x80; i++){ + t->delim[i] = !isalnum(i); } - zDelim[j++] = '\0'; - assert( j<=sizeof(zDelim) ); - t->zDelim = string_dup(zDelim); } *ppTokenizer = &t->base; @@ -91,11 +86,7 @@ static int simpleCreate( } static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ - simple_tokenizer *t = (simple_tokenizer *) pTokenizer; - - free((void *) t->zDelim); - free(t); - + free(pTokenizer); return SQLITE_OK; } @@ -109,10 +100,9 @@ static int simpleOpen( c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor)); c->pInput = pInput; c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes; - c->pCurrent = c->pInput; /* start tokenizing at the beginning */ + c->iOffset = 0; /* start tokenizing at the beginning */ c->iToken = 0; - c->zToken = NULL; /* no space allocated, yet. */ - c->nTokenBytes = 0; + c->pToken = NULL; /* no space allocated, yet. */ c->nTokenAllocated = 0; *ppCursor = &c->base; @@ -121,12 +111,8 @@ static int simpleOpen( static int simpleClose(sqlite3_tokenizer_cursor *pCursor){ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; - - if( NULL!=c->zToken ){ - free(c->zToken); - } + free(c->pToken); free(c); - return SQLITE_OK; } @@ -137,35 +123,42 @@ static int simpleNext( ){ simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor; simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer; - int ii; + unsigned char *p = (unsigned char *)c->pInput; - while( c->pCurrent-c->pInputnBytes ){ - int n = (int) strcspn(c->pCurrent, t->zDelim); - if( n>0 ){ - if( n+1>c->nTokenAllocated ){ - c->zToken = realloc(c->zToken, n+1); + while( c->iOffsetnBytes ){ + int iStartOffset; + + /* Scan past delimiter characters */ + while( c->iOffsetnBytes && isDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + /* Count non-delimiter characters. */ + iStartOffset = c->iOffset; + while( c->iOffsetnBytes && !isDelim(t, p[c->iOffset]) ){ + c->iOffset++; + } + + if( c->iOffset>iStartOffset ){ + int i, n = c->iOffset-iStartOffset; + if( n>c->nTokenAllocated ){ + c->pToken = realloc(c->pToken, n); } - for(ii=0; iipCurrent[ii]; - c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch; + unsigned char ch = p[iStartOffset+i]; + c->pToken[i] = ch<0x80 ? tolower(ch) : ch; } - c->zToken[n] = '\0'; - *ppToken = c->zToken; + *ppToken = c->pToken; *pnBytes = n; - *piStartOffset = (int) (c->pCurrent-c->pInput); - *piEndOffset = *piStartOffset+n; + *piStartOffset = iStartOffset; + *piEndOffset = c->iOffset; *piPosition = c->iToken++; - c->pCurrent += n + 1; return SQLITE_OK; } - c->pCurrent += n + 1; - /* TODO(shess) could strspn() to skip delimiters en masse. Needs - ** to happen in two places, though, which is annoying. - */ } return SQLITE_DONE; } diff --git a/manifest b/manifest index 2a47e37ca9..0edebffefd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sFTS1\smodule\sso\sthat\sits\sname\sis\s"fts1"\sinstead\sof\s"fulltext",\nso\sthat\sall\ssymbols\swith\sexternal\slinkage\sbegin\swith\s"sqlite3Fts1",\sand\nso\sthat\sall\sfilenames\sbegin\swith\s"fts1".\s(CVS\s3377) -D 2006-08-31T15:07:14 +C Make\stokenizer\snot\srely\son\snul-terminated\stext.\s\sInstead\sof\susing\nstrcspn()\sand\sa\snul-terminated\sdelimiter\slist,\sI\sjust\sflagged\ndelimiters\sin\san\sarray\sand\swrote\sthings\sinline.\s\sSubmitting\sthis\sfor\nreview\sseparately\sbecause\sit's\spretty\sstandalone.\s(CVS\s3378) +D 2006-09-01T00:05:17 F Makefile.in 8e7f9ecebab2c6e0f3db20ff129a8f9405ab64f8 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -26,7 +26,7 @@ F ext/fts1/fts1.h fe8e8f38dd6d2d2645b9b0d6972e80985249575f F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114 F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089 F ext/fts1/fts1_tokenizer.h 887580709d32e44ca4ee3c7107c06ee444eb44c2 -F ext/fts1/fts1_tokenizer1.c 13789776e1afc4a4db04f2700eba24f9c5e77a90 +F ext/fts1/fts1_tokenizer1.c 03a9af15f252cf0ee3d518be8c2f48b3eb1d31a1 F ext/fts1/fulltext.c d935e600d87bc86b7d64f55c7520ea41d6034c5c F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d @@ -395,7 +395,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P 6c77c2d5e15e9d3efed3e274bc93cd5a4868f574 -R 3fbd41e2a6fe983095c0be06e9b96477 -U drh -Z 4687716ffdce69db90a5a92c651e7fec +P e1891f0dc58e5498a8845d8b9b5b092d7f9c7003 +R acb5259f7be3fb128befffe51d68519d +U shess +Z 5622cbcf68194bca1ea9e4f2a7e4d027 diff --git a/manifest.uuid b/manifest.uuid index 949092b33f..0fdd7d8361 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1891f0dc58e5498a8845d8b9b5b092d7f9c7003 \ No newline at end of file +2631ceaeefaca3aa837e3b439399f13c51456914 \ No newline at end of file