Avoid passing NULL to the xOpen method of an FTS3/4 tokenizer.

FossilOrigin-Name: 0dd09fc034c127718366d3a3183e367d2f9fd82d

ext/fts3/fts3_write.c

@@ -542,6 +542,14 @@ static int fts3PendingTermsAdd(
 
   assert( pTokenizer && pModule );
 
+  /* If the user has inserted a NULL value, this function may be called with
+  ** zText==0. In this case, add zero token entries to the hash table and 
+  ** return early. */
+  if( zText==0 ){
+    *pnWord = 0;
+    return SQLITE_OK;
+  }
+
   rc = pModule->xOpen(pTokenizer, zText, -1, &pCsr);
   if( rc!=SQLITE_OK ){
     return rc;
@@ -632,11 +640,9 @@ static int fts3InsertTerms(Fts3Table *p, sqlite3_value **apVal, u32 *aSz){
   int i;                          /* Iterator variable */
   for(i=2; i<p->nColumn+2; i++){
     const char *zText = (const char *)sqlite3_value_text(apVal[i]);
-    if( zText ){
-      int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
-      if( rc!=SQLITE_OK ){
-        return rc;
-      }
+    int rc = fts3PendingTermsAdd(p, zText, i-2, &aSz[i-2]);
+    if( rc!=SQLITE_OK ){
+      return rc;
     }
     aSz[p->nColumn] += sqlite3_value_bytes(apVal[i]);
   }

manifest

@@ -1,5 +1,5 @@
-C Change\sthe\sprototype\sfor\sthe\sopen()\ssystem\scall\sto\sagree\swith\sPosix.\s\sThough\na\sfaulty\sfunction\sprototype\sin\sa\spointer\scast\sis\sa\sseemingly\sinnocuous\serror,\s\nthe\scorrect\sprototype\sis\snecessary\sfor\spthreads\sto\swork\scorrectly\son\sNetBSD.
-D 2011-04-17T17:09:58.565
+C Avoid\spassing\sNULL\sto\sthe\sxOpen\smethod\sof\san\sFTS3/4\stokenizer.
+D 2011-04-19T06:43:22.111
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 7a4d9524721d40ef9ee26f93f9bd6a51dba106f2
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -74,7 +74,7 @@ F ext/fts3/fts3_snippet.c e857c6a89d81d3b89df59f3b44b35c68d8ed5c62
 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
-F ext/fts3/fts3_write.c 813495ed106eb9461044e3c0374f4db69b37eb09
+F ext/fts3/fts3_write.c c0af09a04021926d7d84094fa950defc9213416d
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -450,7 +450,7 @@ F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f
 F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
-F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
+F test/fts3atoken.test bbb9e63a915f3df0e35d06e0add932b5bf2d54a9
 F test/fts3aux1.test 719c35cbbcc04dde8e5a54a6f69851a0af9ed1f2
 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@@ -929,7 +929,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 9d6c6129fd8be42c164f14bb90011b252c1f2c46
-R 4ae7f108651e5730cd8c416e7800e26e
-U drh
-Z 939cf52c691885a86594e39d90974750
+P 3e135748f1efacb52b414b3ac3f4ae2c08bcd8fb
+R a6226e702419163cd5db7bb5053e7329
+U dan
+Z 2b55d93cd349a17e33ad50a5d9dd90a0

manifest.uuid

@@ -1 +1 @@
-3e135748f1efacb52b414b3ac3f4ae2c08bcd8fb
+0dd09fc034c127718366d3a3183e367d2f9fd82d

test/fts3atoken.test

@@ -24,6 +24,8 @@ ifcapable !fts3 {
   return
 }
 
+set ::testprefix fts3token
+
 proc escape_string {str} {
   set out ""
   foreach char [split $str ""] {
@@ -167,8 +169,18 @@ ifcapable icu {
   do_icu_test fts3token-4.8 en_US  $input $output
 }
 
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE x1 USING fts3(name,TOKENIZE icu en_US);
+  insert into x1 (name) values (NULL);
+  insert into x1 (name) values (NULL);
+  delete from x1;
+}
+
 do_test fts3token-internal {
   execsql { SELECT fts3_tokenizer_internal_test() }
 } {ok}
 
+
 finish_test
+
+