Be a bit more susicious of invalid results from the tokenizer. (CVS 4514)

FossilOrigin-Name: deb8f56d3adea0025d28b8effabec7c7b7fe3026

ext/fts3/fts3.c

@@ -4120,17 +4120,18 @@ static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
   if( rc!=SQLITE_OK ) return rc;
 
   pCursor->pTokenizer = pTokenizer;
-  while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor,
-                                               &pToken, &nTokenBytes,
-                                               &iStartOffset, &iEndOffset,
-                                               &iPosition) ){
+  while( SQLITE_OK==(rc=pTokenizer->pModule->xNext(pCursor,
+                                                   &pToken, &nTokenBytes,
+                                                   &iStartOffset, &iEndOffset,
+                                                   &iPosition)) ){
     DLCollector *p;
     int nData;                   /* Size of doclist before our update. */
 
-    /* Positions can't be negative; we use -1 as a terminator internally. */
-    if( iPosition<0 ){
-      pTokenizer->pModule->xClose(pCursor);
-      return SQLITE_ERROR;
+    /* Positions can't be negative; we use -1 as a terminator
+     * internally.  Token can't be NULL or empty. */
+    if( iPosition<0 || pToken == NULL || nTokenBytes == 0 ){
+      rc = SQLITE_ERROR;
+      break;
     }
 
     p = fts3HashFind(&v->pendingTerms, pToken, nTokenBytes);
@@ -4159,6 +4160,7 @@ static int buildTerms(fulltext_vtab *v, sqlite_int64 iDocid,
   ** not durable.  *ponder*
   */
   pTokenizer->pModule->xClose(pCursor);
+  if( SQLITE_DONE == rc ) return SQLITE_OK;
   return rc;
 }
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sos2CurrentTime()\sto\sgive\scorrect\sresults\sfor\snegative\stimezones.\s(CVS\s4513)
-D 2007-10-24T23:04:33
+C Be\sa\sbit\smore\ssusicious\sof\sinvalid\sresults\sfrom\sthe\stokenizer.\s(CVS\s4514)
+D 2007-10-24T23:24:22
 F Makefile.in 30c7e3ba426ddb253b8ef037d1873425da6009a8
 F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -49,7 +49,7 @@ F ext/fts2/fts2_tokenizer1.c 8a545c232bdffafd117c4eeaf59789691909f26a
 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.tokenizers a97c9a55b3422f6cb04af9de9296fe2447ea4a78
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 577d6714694212ca46cd6870242fcecdba3ea19b
+F ext/fts3/fts3.c 80b5eb9081232712e204614734bc30a8dbbd2140
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3_hash.c 1c2dc969a5b485848fb804c0ac41a046f18a09c9
 F ext/fts3/fts3_hash.h 004b759e1602ff16dfa02fea3ca1c77336ad6798
@@ -584,7 +584,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
-P 8a233845f61080cca24da3672444f3e5b985da67
-R cfec9c80ac2d201e383258e59bbc24ea
-U pweilbacher
-Z da7147504f76c9a8c8ddff06423f9939
+P a3b82548cb51431569aafacf38e3391a8a1b697b
+R 9840b2a2e4a7e399d42f951e7101b4e6
+U shess
+Z c626a985c57b5c6f7a3cfe6568c0f1ce

manifest.uuid

@@ -1 +1 @@
-a3b82548cb51431569aafacf38e3391a8a1b697b
+deb8f56d3adea0025d28b8effabec7c7b7fe3026