Make tokenizer not rely on nul-terminated text. Instead of using
strcspn() and a nul-terminated delimiter list, I just flagged delimiters in an array and wrote things inline. Submitting this for review separately because it's pretty standalone. (CVS 3378) FossilOrigin-Name: 2631ceaeefaca3aa837e3b439399f13c51456914
This commit is contained in:
parent
5db455e7b5
commit
c0beb14f23
@ -29,61 +29,56 @@
|
||||
|
||||
#include "fts1_tokenizer.h"
|
||||
|
||||
/* Duplicate a string; the caller must free() the returned string.
|
||||
* (We don't use strdup() since it's not part of the standard C library and
|
||||
* may not be available everywhere.) */
|
||||
/* TODO(shess) Copied from fulltext.c, consider util.c for such
|
||||
** things. */
|
||||
static char *string_dup(const char *s){
|
||||
char *str = malloc(strlen(s) + 1);
|
||||
strcpy(str, s);
|
||||
return str;
|
||||
}
|
||||
|
||||
typedef struct simple_tokenizer {
|
||||
sqlite3_tokenizer base;
|
||||
const char *zDelim; /* token delimiters */
|
||||
char delim[128]; /* flag ASCII delimiters */
|
||||
} simple_tokenizer;
|
||||
|
||||
typedef struct simple_tokenizer_cursor {
|
||||
sqlite3_tokenizer_cursor base;
|
||||
const char *pInput; /* input we are tokenizing */
|
||||
int nBytes; /* size of the input */
|
||||
const char *pCurrent; /* current position in pInput */
|
||||
int iOffset; /* current position in pInput */
|
||||
int iToken; /* index of next token to be returned */
|
||||
char *zToken; /* storage for current token */
|
||||
int nTokenBytes; /* actual size of current token */
|
||||
char *pToken; /* storage for current token */
|
||||
int nTokenAllocated; /* space allocated to zToken buffer */
|
||||
} simple_tokenizer_cursor;
|
||||
|
||||
static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
|
||||
|
||||
static int isDelim(simple_tokenizer *t, unsigned char c){
|
||||
return c<0x80 && t->delim[c];
|
||||
}
|
||||
|
||||
static int simpleCreate(
|
||||
int argc, const char **argv,
|
||||
sqlite3_tokenizer **ppTokenizer
|
||||
){
|
||||
simple_tokenizer *t;
|
||||
|
||||
t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
|
||||
t = (simple_tokenizer *) calloc(sizeof(simple_tokenizer), 1);
|
||||
/* TODO(shess) Delimiters need to remain the same from run to run,
|
||||
** else we need to reindex. One solution would be a meta-table to
|
||||
** track such information in the database, then we'd only want this
|
||||
** information on the initial create.
|
||||
*/
|
||||
if( argc>1 ){
|
||||
t->zDelim = string_dup(argv[1]);
|
||||
} else {
|
||||
/* Build a string excluding alphanumeric ASCII characters */
|
||||
char zDelim[0x80]; /* nul-terminated, so nul not a member */
|
||||
int i, j;
|
||||
for(i=1, j=0; i<0x80; i++){
|
||||
if( !isalnum(i) ){
|
||||
zDelim[j++] = i;
|
||||
int i, n = strlen(argv[1]);
|
||||
for(i=0; i<n; i++){
|
||||
unsigned char ch = argv[1][i];
|
||||
/* We explicitly don't support UTF-8 delimiters for now. */
|
||||
if( ch>=0x80 ){
|
||||
free(t);
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
t->delim[ch] = 1;
|
||||
}
|
||||
} else {
|
||||
/* Mark non-alphanumeric ASCII characters as delimiters */
|
||||
int i;
|
||||
for(i=1; i<0x80; i++){
|
||||
t->delim[i] = !isalnum(i);
|
||||
}
|
||||
zDelim[j++] = '\0';
|
||||
assert( j<=sizeof(zDelim) );
|
||||
t->zDelim = string_dup(zDelim);
|
||||
}
|
||||
|
||||
*ppTokenizer = &t->base;
|
||||
@ -91,11 +86,7 @@ static int simpleCreate(
|
||||
}
|
||||
|
||||
static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
|
||||
simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
|
||||
|
||||
free((void *) t->zDelim);
|
||||
free(t);
|
||||
|
||||
free(pTokenizer);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
@ -109,10 +100,9 @@ static int simpleOpen(
|
||||
c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
|
||||
c->pInput = pInput;
|
||||
c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
|
||||
c->pCurrent = c->pInput; /* start tokenizing at the beginning */
|
||||
c->iOffset = 0; /* start tokenizing at the beginning */
|
||||
c->iToken = 0;
|
||||
c->zToken = NULL; /* no space allocated, yet. */
|
||||
c->nTokenBytes = 0;
|
||||
c->pToken = NULL; /* no space allocated, yet. */
|
||||
c->nTokenAllocated = 0;
|
||||
|
||||
*ppCursor = &c->base;
|
||||
@ -121,12 +111,8 @@ static int simpleOpen(
|
||||
|
||||
static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
|
||||
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
|
||||
|
||||
if( NULL!=c->zToken ){
|
||||
free(c->zToken);
|
||||
}
|
||||
free(c->pToken);
|
||||
free(c);
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
@ -137,35 +123,42 @@ static int simpleNext(
|
||||
){
|
||||
simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
|
||||
simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
|
||||
int ii;
|
||||
unsigned char *p = (unsigned char *)c->pInput;
|
||||
|
||||
while( c->pCurrent-c->pInput<c->nBytes ){
|
||||
int n = (int) strcspn(c->pCurrent, t->zDelim);
|
||||
if( n>0 ){
|
||||
if( n+1>c->nTokenAllocated ){
|
||||
c->zToken = realloc(c->zToken, n+1);
|
||||
while( c->iOffset<c->nBytes ){
|
||||
int iStartOffset;
|
||||
|
||||
/* Scan past delimiter characters */
|
||||
while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
|
||||
c->iOffset++;
|
||||
}
|
||||
|
||||
/* Count non-delimiter characters. */
|
||||
iStartOffset = c->iOffset;
|
||||
while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
|
||||
c->iOffset++;
|
||||
}
|
||||
|
||||
if( c->iOffset>iStartOffset ){
|
||||
int i, n = c->iOffset-iStartOffset;
|
||||
if( n>c->nTokenAllocated ){
|
||||
c->pToken = realloc(c->pToken, n);
|
||||
}
|
||||
for(ii=0; ii<n; ii++){
|
||||
for(i=0; i<n; i++){
|
||||
/* TODO(shess) This needs expansion to handle UTF-8
|
||||
** case-insensitivity.
|
||||
*/
|
||||
char ch = c->pCurrent[ii];
|
||||
c->zToken[ii] = (unsigned char)ch<0x80 ? tolower(ch) : ch;
|
||||
unsigned char ch = p[iStartOffset+i];
|
||||
c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
|
||||
}
|
||||
c->zToken[n] = '\0';
|
||||
*ppToken = c->zToken;
|
||||
*ppToken = c->pToken;
|
||||
*pnBytes = n;
|
||||
*piStartOffset = (int) (c->pCurrent-c->pInput);
|
||||
*piEndOffset = *piStartOffset+n;
|
||||
*piStartOffset = iStartOffset;
|
||||
*piEndOffset = c->iOffset;
|
||||
*piPosition = c->iToken++;
|
||||
c->pCurrent += n + 1;
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
c->pCurrent += n + 1;
|
||||
/* TODO(shess) could strspn() to skip delimiters en masse. Needs
|
||||
** to happen in two places, though, which is annoying.
|
||||
*/
|
||||
}
|
||||
return SQLITE_DONE;
|
||||
}
|
||||
|
14
manifest
14
manifest
@ -1,5 +1,5 @@
|
||||
C Refactor\sthe\sFTS1\smodule\sso\sthat\sits\sname\sis\s"fts1"\sinstead\sof\s"fulltext",\nso\sthat\sall\ssymbols\swith\sexternal\slinkage\sbegin\swith\s"sqlite3Fts1",\sand\nso\sthat\sall\sfilenames\sbegin\swith\s"fts1".\s(CVS\s3377)
|
||||
D 2006-08-31T15:07:14
|
||||
C Make\stokenizer\snot\srely\son\snul-terminated\stext.\s\sInstead\sof\susing\nstrcspn()\sand\sa\snul-terminated\sdelimiter\slist,\sI\sjust\sflagged\ndelimiters\sin\san\sarray\sand\swrote\sthings\sinline.\s\sSubmitting\sthis\sfor\nreview\sseparately\sbecause\sit's\spretty\sstandalone.\s(CVS\s3378)
|
||||
D 2006-09-01T00:05:17
|
||||
F Makefile.in 8e7f9ecebab2c6e0f3db20ff129a8f9405ab64f8
|
||||
F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
|
||||
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
|
||||
@ -26,7 +26,7 @@ F ext/fts1/fts1.h fe8e8f38dd6d2d2645b9b0d6972e80985249575f
|
||||
F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114
|
||||
F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089
|
||||
F ext/fts1/fts1_tokenizer.h 887580709d32e44ca4ee3c7107c06ee444eb44c2
|
||||
F ext/fts1/fts1_tokenizer1.c 13789776e1afc4a4db04f2700eba24f9c5e77a90
|
||||
F ext/fts1/fts1_tokenizer1.c 03a9af15f252cf0ee3d518be8c2f48b3eb1d31a1
|
||||
F ext/fts1/fulltext.c d935e600d87bc86b7d64f55c7520ea41d6034c5c
|
||||
F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
|
||||
F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d
|
||||
@ -395,7 +395,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
|
||||
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
|
||||
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
|
||||
F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
|
||||
P 6c77c2d5e15e9d3efed3e274bc93cd5a4868f574
|
||||
R 3fbd41e2a6fe983095c0be06e9b96477
|
||||
U drh
|
||||
Z 4687716ffdce69db90a5a92c651e7fec
|
||||
P e1891f0dc58e5498a8845d8b9b5b092d7f9c7003
|
||||
R acb5259f7be3fb128befffe51d68519d
|
||||
U shess
|
||||
Z 5622cbcf68194bca1ea9e4f2a7e4d027
|
||||
|
@ -1 +1 @@
|
||||
e1891f0dc58e5498a8845d8b9b5b092d7f9c7003
|
||||
2631ceaeefaca3aa837e3b439399f13c51456914
|
Loading…
Reference in New Issue
Block a user