Improve fts5 tests.

FossilOrigin-Name: c1f07a3aa98eac87e2747527d15e5e5562221ceb

ext/fts5/fts5.c

@@ -529,46 +529,48 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){
 ** on the xClose method of the virtual table interface.
 */
 static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){
-  Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
+  if( pCursor ){
-  Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
+    Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab);
-  Fts5Cursor **pp;
+    Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
-  Fts5Auxdata *pData;
+    Fts5Cursor **pp;
-  Fts5Auxdata *pNext;
+    Fts5Auxdata *pData;
+    Fts5Auxdata *pNext;
 
-  fts5CsrNewrow(pCsr);
+    fts5CsrNewrow(pCsr);
-  if( pCsr->pStmt ){
+    if( pCsr->pStmt ){
-    int eStmt = fts5StmtType(pCsr->idxNum);
+      int eStmt = fts5StmtType(pCsr->idxNum);
-    sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
+      sqlite3Fts5StorageStmtRelease(pTab->pStorage, eStmt, pCsr->pStmt);
-  }
+    }
-  if( pCsr->pSorter ){
+    if( pCsr->pSorter ){
-    Fts5Sorter *pSorter = pCsr->pSorter;
+      Fts5Sorter *pSorter = pCsr->pSorter;
-    sqlite3_finalize(pSorter->pStmt);
+      sqlite3_finalize(pSorter->pStmt);
-    sqlite3_free(pSorter);
+      sqlite3_free(pSorter);
-  }
+    }
-  
-  if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
-    sqlite3Fts5ExprFree(pCsr->pExpr);
-  }
 
-  for(pData=pCsr->pAuxdata; pData; pData=pNext){
+    if( pCsr->idxNum!=FTS5_PLAN_SOURCE ){
-    pNext = pData->pNext;
+      sqlite3Fts5ExprFree(pCsr->pExpr);
-    if( pData->xDelete ) pData->xDelete(pData->pPtr);
+    }
-    sqlite3_free(pData);
+
+    for(pData=pCsr->pAuxdata; pData; pData=pNext){
+      pNext = pData->pNext;
+      if( pData->xDelete ) pData->xDelete(pData->pPtr);
+      sqlite3_free(pData);
+    }
+
+    /* Remove the cursor from the Fts5Global.pCsr list */
+    for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
+    *pp = pCsr->pNext;
+
+    sqlite3_finalize(pCsr->pRankArgStmt);
+    sqlite3_free(pCsr->apRankArg);
+
+    sqlite3_free(pCsr->zSpecial);
+    if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
+      sqlite3_free(pCsr->zRank);
+      sqlite3_free(pCsr->zRankArgs);
+    }
+    sqlite3_free(pCsr);
   }
-
-  /* Remove the cursor from the Fts5Global.pCsr list */
-  for(pp=&pTab->pGlobal->pCsr; (*pp)!=pCsr; pp=&(*pp)->pNext);
-  *pp = pCsr->pNext;
-
-  sqlite3_finalize(pCsr->pRankArgStmt);
-  sqlite3_free(pCsr->apRankArg);
-
-  sqlite3_free(pCsr->zSpecial);
-  if( CsrFlagTest(pCsr, FTS5CSR_FREE_ZRANK) ){
-    sqlite3_free(pCsr->zRank);
-    sqlite3_free(pCsr->zRankArgs);
-  }
-  sqlite3_free(pCsr);
   return SQLITE_OK;
 }
 
@@ -893,7 +895,11 @@ static int fts5CursorParseRank(
     char *zRank = 0;
     char *zRankArgs = 0;
 
-    rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
+    if( z==0 ){
+      if( sqlite3_value_type(pRank)==SQLITE_NULL ) rc = SQLITE_ERROR;
+    }else{
+      rc = sqlite3Fts5ConfigParseRank(z, &zRank, &zRankArgs);
+    }
     if( rc==SQLITE_OK ){
       pCsr->zRank = zRank;
       pCsr->zRankArgs = zRankArgs;
@@ -1207,16 +1213,16 @@ static int fts5UpdateMethod(
   **   3. Values for each of the nCol matchable columns.
   **   4. Values for the two hidden columns (<tablename> and "rank").
   */
-  assert( nArg==1 || nArg==(2 + pConfig->nCol + 2) );
 
   eType0 = sqlite3_value_type(apVal[0]);
   eConflict = sqlite3_vtab_on_conflict(pConfig->db);
 
   assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL );
   assert( pVtab->zErrMsg==0 );
+  assert( (nArg==1 && eType0==SQLITE_INTEGER) || nArg==(2+pConfig->nCol+2) );
 
   fts5TripCursors(pTab);
-  if( rc==SQLITE_OK && eType0==SQLITE_INTEGER ){
+  if( eType0==SQLITE_INTEGER ){
     if( fts5IsContentless(pTab) ){
       pTab->base.zErrMsg = sqlite3_mprintf(
           "cannot %s contentless fts5 table: %s", 
@@ -1227,7 +1233,8 @@ static int fts5UpdateMethod(
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel);
     }
-  }else if( nArg>1 ){
+  }else{
+    assert( nArg>1 );
     sqlite3_value *pCmd = apVal[2 + pConfig->nCol];
     if( SQLITE_NULL!=sqlite3_value_type(pCmd) ){
       const char *z = (const char*)sqlite3_value_text(pCmd);
@@ -1471,6 +1478,7 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
   if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){
     i64 iRowid = fts5CursorRowid(pCsr);
     rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize);
+    CsrFlagClear(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
   }
   if( iCol<0 ){
     int i;
@@ -1879,6 +1887,23 @@ static int fts5CreateTokenizer(
   return rc;
 }
 
+static Fts5TokenizerModule *fts5LocateTokenizer(
+  Fts5Global *pGlobal, 
+  const char *zName
+){
+  Fts5TokenizerModule *pMod = 0;
+
+  if( zName==0 ){
+    pMod = pGlobal->pDfltTok;
+  }else{
+    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
+      if( sqlite3_stricmp(zName, pMod->zName)==0 ) break;
+    }
+  }
+
+  return pMod;
+}
+
 /*
 ** Find a tokenizer. This is the implementation of the 
 ** fts5_api.xFindTokenizer() method.
@@ -1889,21 +1914,13 @@ static int fts5FindTokenizer(
   void **ppUserData,
   fts5_tokenizer *pTokenizer      /* Populate this object */
 ){
-  Fts5Global *pGlobal = (Fts5Global*)pApi;
   int rc = SQLITE_OK;
-  Fts5TokenizerModule *pTok;
+  Fts5TokenizerModule *pMod;
 
-  if( zName==0 ){
+  pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName);
-    pTok = pGlobal->pDfltTok;
+  if( pMod ){
-  }else{
+    *pTokenizer = pMod->x;
-    for(pTok=pGlobal->pTok; pTok; pTok=pTok->pNext){
+    *ppUserData = pMod->pUserData;
-      if( sqlite3_stricmp(zName, pTok->zName)==0 ) break;
-    }
-  }
-
-  if( pTok ){
-    *pTokenizer = pTok->x;
-    *ppUserData = pTok->pUserData;
   }else{
     memset(pTokenizer, 0, sizeof(fts5_tokenizer));
     rc = SQLITE_ERROR;
@@ -1917,24 +1934,23 @@ int sqlite3Fts5GetTokenizer(
   const char **azArg,
   int nArg,
   Fts5Tokenizer **ppTok,
-  fts5_tokenizer **ppTokApi
+  fts5_tokenizer **ppTokApi,
+  char **pzErr
 ){
-  Fts5TokenizerModule *pMod = 0;
+  Fts5TokenizerModule *pMod;
   int rc = SQLITE_OK;
 
-  if( nArg==0 ){
+  pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]);
-    pMod = pGlobal->pDfltTok;
-  }else{
-    for(pMod=pGlobal->pTok; pMod; pMod=pMod->pNext){
-      if( sqlite3_stricmp(azArg[0], pMod->zName)==0 ) break;
-    }
-  }
-
   if( pMod==0 ){
+    assert( nArg>0 );
     rc = SQLITE_ERROR;
+    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
   }else{
     rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
     *ppTokApi = &pMod->x;
+    if( rc!=SQLITE_OK && pzErr ){
+      *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+    }
   }
 
   if( rc!=SQLITE_OK ){

ext/fts5/fts5Int.h

@@ -61,7 +61,8 @@ int sqlite3Fts5GetTokenizer(
   const char **azArg,
   int nArg,
   Fts5Tokenizer**,
-  fts5_tokenizer**
+  fts5_tokenizer**,
+  char **pzErr
 );
 
 /*

ext/fts5/fts5_aux.c

@@ -230,11 +230,12 @@ static void fts5HighlightFunction(
 
     if( rc==SQLITE_OK ){
       sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
     }
     sqlite3_free(ctx.zOut);
   }
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+  }
 }
 /*
 ** End of highlight() implementation.

ext/fts5/fts5_config.c

@@ -330,11 +330,9 @@ static int fts5ConfigParseSpecial(
           rc = SQLITE_ERROR;
         }else{
           rc = sqlite3Fts5GetTokenizer(pGlobal, 
-              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi
+              (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi,
+              pzErr
           );
-          if( rc!=SQLITE_OK ){
-            *pzErr = sqlite3_mprintf("error in tokenizer constructor");
-          }
         }
       }
     }
@@ -387,7 +385,7 @@ static int fts5ConfigParseSpecial(
 static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
   assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
   return sqlite3Fts5GetTokenizer(
-      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi
+      pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
   );
 }
 
@@ -563,7 +561,7 @@ int sqlite3Fts5ConfigParse(
         rc = SQLITE_ERROR;
       }else{
         if( bOption ){
-          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo, pzErr);
+          rc = fts5ConfigParseSpecial(pGlobal, pRet, zOne, zTwo?zTwo:"", pzErr);
         }else{
           rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
           zOne = 0;

ext/fts5/fts5_expr.c

@@ -280,15 +280,15 @@ int sqlite3Fts5ExprPhraseExpr(
   Fts5ExprPhrase *pCopy;
 
   pOrig = pExpr->apExprPhrase[iPhrase];
+  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
+      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
+  );
   pNew = (Fts5Expr*)fts5ExprMalloc(&rc, sizeof(Fts5Expr));
   apPhrase = (Fts5ExprPhrase**)fts5ExprMalloc(&rc, sizeof(Fts5ExprPhrase*));
   pNode = (Fts5ExprNode*)fts5ExprMalloc(&rc, sizeof(Fts5ExprNode));
   pNear = (Fts5ExprNearset*)fts5ExprMalloc(&rc, 
       sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
   );
-  pCopy = (Fts5ExprPhrase*)fts5ExprMalloc(&rc, 
-      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
-  );
 
   for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
     pCopy->aTerm[i].zTerm = fts5ExprStrdup(&rc, pOrig->aTerm[i].zTerm);

ext/fts5/fts5_tcl.c

@@ -36,6 +36,9 @@ int sqlite3_fts5_may_be_corrupt = 0;
 ** can extract the sqlite3* pointer from an existing Tcl SQLite
 ** connection.
 */
+
+extern const char *sqlite3ErrName(int);
+
 struct SqliteDb {
   sqlite3 *db;
 };
@@ -390,7 +393,7 @@ static int xF5tApi(
 #undef CASE
 
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in api call", 0);
+    Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
     return TCL_ERROR;
   }
 
@@ -727,8 +730,6 @@ static int f5tTokenizerTokenize(
   return rc;
 }
 
-extern const char *sqlite3ErrName(int);
-
 /*
 ** sqlite3_fts5_token TEXT START END POS
 */

ext/fts5/fts5_tokenize.c

@@ -529,11 +529,16 @@ static int fts5PorterCreate(
   int rc = SQLITE_OK;
   PorterTokenizer *pRet;
   void *pUserdata = 0;
+  const char *zBase = "unicode61";
+
+  if( nArg>0 ){
+    zBase = azArg[0];
+  }
 
   pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
   if( pRet ){
     memset(pRet, 0, sizeof(PorterTokenizer));
-    rc = pApi->xFindTokenizer(pApi, "unicode61", &pUserdata, &pRet->tokenizer);
+    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
   }else{
     rc = SQLITE_NOMEM;
   }

ext/fts5/test/fts5aa.test

@@ -297,7 +297,7 @@ do_catchsql_test 12.2 {
 } {1 {unknown special query: stuff}}
 
 do_test 12.3 {
-  set res [db one { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
+  set res [db eval { SELECT t2 FROM t2 WHERE t2 MATCH '* reads ' }]
   string is integer $res
 } {1}
 

ext/fts5/test/fts5al.test

@@ -268,6 +268,14 @@ do_execsql_test 4.3.3 {
   {a three} 0 {a one} 1 {a four} 1 {a two} 2 {a five} 2 
 }
 
+do_catchsql_test 4.4.3 {
+  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH 'xyz(3)' 
+} {1 {no such function: xyz}}
+do_catchsql_test 4.4.4 {
+  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
+} {1 {parse error in rank function: }}
+
+
 
 finish_test
 

ext/fts5/test/fts5aux.test

@@ -0,0 +1,57 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the auxiliary function APIs.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5aux
+
+proc inst {cmd i} { 
+  $cmd xInst $i
+}
+sqlite3_fts5_create_function db inst inst
+
+proc colsize {cmd i} { 
+  $cmd xColumnSize $i
+}
+sqlite3_fts5_create_function db colsize colsize
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE f1 USING fts5(a, b);
+  INSERT INTO f1 VALUES('one two', 'two one zero');
+}
+
+do_catchsql_test 1.1 {
+  SELECT inst(f1, -1) FROM f1 WHERE f1 MATCH 'two';
+} {1 SQLITE_RANGE}
+do_catchsql_test 1.2 {
+  SELECT inst(f1, 0) FROM f1 WHERE f1 MATCH 'two';
+} {0 {{0 0 1}}}
+do_catchsql_test 1.3 {
+  SELECT inst(f1, 1) FROM f1 WHERE f1 MATCH 'two';
+} {0 {{0 1 0}}}
+do_catchsql_test 1.4 {
+  SELECT inst(f1, 2) FROM f1 WHERE f1 MATCH 'two';
+} {1 SQLITE_RANGE}
+
+do_catchsql_test 2.1 {
+  SELECT colsize(f1, 2) FROM f1 WHERE f1 MATCH 'two';
+} {1 SQLITE_RANGE}
+
+do_execsql_test 2.2 {
+  SELECT colsize(f1, 0), colsize(f1, 1) FROM f1 WHERE f1 MATCH 'zero';
+} {2 3}
+
+
+
+finish_test
+

ext/fts5/test/fts5content.test

@@ -215,6 +215,24 @@ do_execsql_test 4.7 {
 do_execsql_test 4.8 { SELECT rowid FROM t2 WHERE t2 MATCH 'b'} {}
 do_execsql_test 4.9 { SELECT rowid FROM t2 WHERE t2 MATCH 'y'} {-40}
 
+#-------------------------------------------------------------------------
+# Test that if the 'rowid' field of a 'delete' is not an integer, no
+# changes are made to the FTS index.
+#
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t5 USING fts5(a, b, content=);
+  INSERT INTO t5(rowid, a, b) VALUES(-1, 'one',   'two');
+  INSERT INTO t5(rowid, a, b) VALUES( 0, 'three', 'four');
+  INSERT INTO t5(rowid, a, b) VALUES( 1, 'five',  'six');
+}
+
+set ::checksum [execsql {SELECT md5sum(id, block) FROM t5_data}]
+
+do_execsql_test 5.1 {
+  INSERT INTO t5(t5, rowid, a, b) VALUES('delete', NULL, 'three', 'four');
+  SELECT md5sum(id, block) FROM t5_data;
+} $::checksum
+
 
 finish_test
 

ext/fts5/test/fts5corrupt.test

@@ -70,7 +70,24 @@ do_test 2.1 {
 } {}
 
 #--------------------------------------------------------------------
+# A mundane test - missing row in the %_content table.
 #
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(x);
+  INSERT INTO t3 VALUES('one o');
+  INSERT INTO t3 VALUES('two e');
+  INSERT INTO t3 VALUES('three o');
+  INSERT INTO t3 VALUES('four e');
+  INSERT INTO t3 VALUES('five o');
+}
+do_execsql_test 3.1 {
+  SELECT * FROM t3 WHERE t3 MATCH 'o'
+} {{one o} {three o} {five o}}
+
+do_catchsql_test 3.1 {
+  DELETE FROM t3_content WHERE rowid = 3;
+  SELECT * FROM t3 WHERE t3 MATCH 'o';
+} {1 {database disk image is malformed}}
 
 finish_test
 

ext/fts5/test/fts5doclist.test

@@ -0,0 +1,41 @@
+# 2015 April 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This test is focused on edge cases in the doclist format.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5doclist
+
+
+#-------------------------------------------------------------------------
+# Create a table with 1000 columns. Then add some large documents to it.
+# All text is in the right most column of the table.
+#
+do_test 1.0 {
+  set cols [list]
+  for {set i 0} {$i < 900} {incr i} { lappend cols "x$i" }
+  execsql "CREATE VIRTUAL TABLE ccc USING fts5([join $cols ,])"
+} {}
+
+db func rnddoc fts5_rnddoc 
+do_execsql_test 1.1 {
+  WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
+  INSERT INTO ccc(x899) SELECT rnddoc(500) FROM ii;
+}
+
+do_execsql_test 1.2 {
+  INSERT INTO ccc(ccc) VALUES('integrity-check');
+}
+
+
+finish_test
+

ext/fts5/test/fts5fault4.test

@@ -22,6 +22,8 @@ ifcapable !fts5 {
   return
 }
 
+if 1 {
+
 #-------------------------------------------------------------------------
 # An OOM while dropping an fts5 table.
 #
@@ -40,6 +42,154 @@ do_faultsim_test 1 -faults oom-* -prep {
   faultsim_test_result [list 0 {}]
 }
 
+#-------------------------------------------------------------------------
+# An OOM within an "ORDER BY rank" query.
+#
+db func rnddoc fts5_rnddoc 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE xx USING fts5(x);
+  INSERT INTO xx VALUES ('abc ' || rnddoc(10));
+  INSERT INTO xx VALUES ('abc abc' || rnddoc(9));
+  INSERT INTO xx VALUES ('abc abc abc' || rnddoc(8));
+} {}
+faultsim_save_and_close
+
+do_faultsim_test 2 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM xx }
+} -body {
+  execsql { SELECT rowid FROM xx WHERE xx MATCH 'abc' ORDER BY rank }
+} -test {
+  faultsim_test_result [list 0 {3 2 1}]
+}
+
+#-------------------------------------------------------------------------
+# An OOM while "reseeking" an FTS cursor.
+#
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE jj USING fts5(j);
+  INSERT INTO jj(rowid, j) VALUES(101, 'm t w t f s s');
+  INSERT INTO jj(rowid, j) VALUES(202, 't w t f s');
+  INSERT INTO jj(rowid, j) VALUES(303, 'w t f');
+  INSERT INTO jj(rowid, j) VALUES(404, 't');
+}
+faultsim_save_and_close
+
+do_faultsim_test 3 -faults oom-* -prep {
+  faultsim_restore_and_reopen
+  execsql { SELECT * FROM jj }
+} -body {
+  set res [list]
+  db eval { SELECT rowid FROM jj WHERE jj MATCH 't' } {
+    lappend res $rowid
+    if {$rowid==303} {
+      execsql { DELETE FROM jj WHERE rowid=404 }
+    }
+  }
+  set res
+} -test {
+  faultsim_test_result [list 0 {101 202 303}]
+}
+
+#-------------------------------------------------------------------------
+# An OOM within a special "*reads" query.
+#
+reset_db
+db func rnddoc fts5_rnddoc
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
+
+  WITH ii(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10 )
+  INSERT INTO x1 SELECT rnddoc(5) FROM ii;
+}
+
+set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
+
+do_faultsim_test 4 -faults oom-* -body {
+  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
+} -test {
+  faultsim_test_result {0 {0 {} 3}}
+}
+
+#-------------------------------------------------------------------------
+# An OOM within a query that uses a custom rank function.
+#
+reset_db
+do_execsql_test 5.0 {
+  PRAGMA encoding='utf16';
+  CREATE VIRTUAL TABLE x2 USING fts5(x);
+  INSERT INTO x2(rowid, x) VALUES(10, 'a b c'); -- 3
+  INSERT INTO x2(rowid, x) VALUES(20, 'a b c'); -- 6
+  INSERT INTO x2(rowid, x) VALUES(30, 'a b c'); -- 2
+  INSERT INTO x2(rowid, x) VALUES(40, 'a b c'); -- 5
+  INSERT INTO x2(rowid, x) VALUES(50, 'a b c'); -- 1
+}
+
+proc rowidmod {cmd mod} { 
+  set row [$cmd xRowid]
+  expr {$row % $mod}
+}
+sqlite3_fts5_create_function db rowidmod rowidmod
+
+do_faultsim_test 5.1 -faults oom-* -body {
+  db eval {
+    SELECT rowid || '-' || rank FROM x2 WHERE x2 MATCH 'b' AND 
+    rank MATCH "rowidmod('7')" ORDER BY rank
+  }
+} -test {
+  faultsim_test_result {0 {50-1 30-2 10-3 40-5 20-6}}
+}
+
+proc rowidprefix {cmd prefix} { 
+  set row [$cmd xRowid]
+  set {} "${row}-${prefix}"
+}
+sqlite3_fts5_create_function db rowidprefix rowidprefix
+
+set str [string repeat abcdefghijklmnopqrstuvwxyz 10]
+do_faultsim_test 5.2 -faults oom-* -body {
+  db eval "
+    SELECT rank, x FROM x2 WHERE x2 MATCH 'b' AND 
+    rank MATCH 'rowidprefix(''$::str'')'
+    LIMIT 1
+  "
+} -test {
+  faultsim_test_result "0 {10-$::str {a b c}}"
+}
+
+}
+
+
+#-------------------------------------------------------------------------
+# OOM errors within auxiliary functions.
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE x3 USING fts5(xxx);
+  INSERT INTO x3 VALUES('a b c d c b a');
+}
+
+do_faultsim_test 6.1 -faults oom-t* -body {
+  db eval { SELECT highlight(x3, 0, '*', '*') FROM x3 WHERE x3 MATCH 'c' }
+} -test {
+  faultsim_test_result {0 {{a b *c* d *c* b a}}}
+}
+
+proc firstinst {cmd} { 
+  foreach {p c o} [$cmd xInst 0] {}
+  expr $c*100 + $o
+}
+sqlite3_fts5_create_function db firstinst firstinst
+
+do_faultsim_test 6.2 -faults oom-t* -body {
+  db eval { SELECT firstinst(x3) FROM x3 WHERE x3 MATCH 'c' }
+} -test {
+  faultsim_test_result {0 2} {1 SQLITE_NOMEM}
+}
+
+
+
 
 finish_test
 

ext/fts5/test/fts5plan.test

@@ -0,0 +1,61 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file focuses on testing the planner (xBestIndex function).
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5plan
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x, y);
+  CREATE VIRTUAL TABLE f1 USING fts5(ff);
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1, f1 WHERE f1 MATCH t1.x
+} {
+  0 0 0 {SCAN TABLE t1} 
+  0 1 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
+}
+
+do_eqp_test 1.2 {
+  SELECT * FROM t1, f1 WHERE f1 > t1.x
+} {
+  0 0 1 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 1 0 {SCAN TABLE t1} 
+}
+
+do_eqp_test 1.3 {
+  SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
+} {
+  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 2:}
+  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+}
+
+do_eqp_test 1.4 {
+  SELECT * FROM f1 ORDER BY rank
+} {
+  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+  0 0 0 {USE TEMP B-TREE FOR ORDER BY}
+}
+
+do_eqp_test 1.5 {
+  SELECT * FROM f1 WHERE rank MATCH ?
+} {
+  0 0 0 {SCAN TABLE f1 VIRTUAL TABLE INDEX 1:}
+}
+
+
+
+
+finish_test
+

ext/fts5/test/fts5rank.test

@@ -0,0 +1,39 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file focuses on testing queries that use the "rank" column.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5rank
+
+
+#-------------------------------------------------------------------------
+# "ORDER BY rank" + highlight() + large poslists.
+#
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE xyz USING fts5(z);
+}
+do_test 1.1 {
+  set doc [string trim [string repeat "x y " 500]]
+  execsql { INSERT INTO xyz VALUES($doc) }
+} {}
+do_execsql_test 1.2 {
+  SELECT highlight(xyz, 0, '[', ']') FROM xyz WHERE xyz MATCH 'x' ORDER BY rank
+} [list [string map {x [x]} $doc]]
+
+do_execsql_test 1.3 {
+  SELECT highlight(xyz, 0, '[', ']') FROM xyz
+  WHERE xyz MATCH 'x AND y' ORDER BY rank
+} [list [string map {x [x] y [y]} $doc]]
+
+finish_test
+

ext/fts5/test/fts5rebuild.test

@@ -46,5 +46,16 @@ do_execsql_test 1.7 {
   INSERT INTO f1(f1) VALUES('integrity-check');
 } {}
 
+
+#-------------------------------------------------------------------------
+# Check that 'rebuild' may not be used with a contentless table.
+#
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE nc USING fts5(doc, content=);
+}
+
+do_catchsql_test 2.2 {
+  INSERT INTO nc(nc) VALUES('rebuild');
+} {1 {'rebuild' may not be used with a contentless fts5 table}}
 finish_test
 

ext/fts5/test/fts5restart.test

@@ -19,6 +19,10 @@ do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE f1 USING fts5(ff);
 }
 
+#-------------------------------------------------------------------------
+# Run the 'optimize' command. Check that it does not disturb ongoing
+# full-text queries.
+#
 do_test 1.1 {
   for {set i 1} {$i < 1000} {incr i} {
     execsql { INSERT INTO f1 VALUES('a b c d e') }
@@ -30,7 +34,6 @@ do_execsql_test 1.2 {
   SELECT rowid FROM f1 WHERE f1 MATCH 'c';
 } $lRowid
 
-breakpoint
 do_test 1.3 {
   set res [list]
   db eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
@@ -42,6 +45,101 @@ do_test 1.3 {
   set res
 } $lRowid
 
+do_test 1.4.1 {
+  sqlite3 db2 test.db
+  set res [list]
+  db2 eval { SELECT rowid FROM f1 WHERE f1 MATCH 'c' } {
+    if {$rowid == 100} {
+      set cres [catchsql { INSERT INTO f1(f1) VALUES('optimize') }]
+    }
+    lappend res $rowid
+  }
+  set res
+} $lRowid
+
+do_test 1.4.2 {
+  db2 close
+  set cres
+} {1 {database is locked}}
+
+#-------------------------------------------------------------------------
+# Open a couple of cursors. Then close them in the same order.
+#
+do_test 2.1 {
+  set ::s1 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'b'" -1 X]
+  set ::s2 [sqlite3_prepare db "SELECT rowid FROM f1 WHERE f1 MATCH 'c'" -1 X]
+
+  sqlite3_step $::s1
+} {SQLITE_ROW}
+do_test 2.2 {
+  sqlite3_step $::s2
+} {SQLITE_ROW}
+
+do_test 2.1 {
+  sqlite3_finalize $::s1
+  sqlite3_finalize $::s2
+} {SQLITE_OK}
+
+#-------------------------------------------------------------------------
+# Copy data between two FTS5 tables.
+#
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE f2 USING fts5(gg);
+  INSERT INTO f2 SELECT ff FROM f1 WHERE f1 MATCH 'b+c+d';
+}
+do_execsql_test 3.2 {
+  SELECT rowid FROM f2 WHERE f2 MATCH 'a+b+c+d+e'
+} $lRowid
+
+#-------------------------------------------------------------------------
+# Remove the row that an FTS5 cursor is currently pointing to. And 
+# various other similar things. Check that this does not disturb 
+# ongoing scans.
+#
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE n4 USING fts5(n);
+  INSERT INTO n4(rowid, n) VALUES(100, '1 2 3 4 5');
+  INSERT INTO n4(rowid, n) VALUES(200, '1 2 3 4');
+  INSERT INTO n4(rowid, n) VALUES(300, '2 3 4');
+  INSERT INTO n4(rowid, n) VALUES(400, '2 3');
+  INSERT INTO n4(rowid, n) VALUES(500, '3');
+}
+
+do_test 4.1 {
+  set res [list]
+  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' } {
+    if {$rowid==300} {
+      execsql { DELETE FROM n4 WHERE rowid=300 }
+    }
+    lappend res $rowid
+  }
+  set res
+} {100 200 300 400 500}
+
+do_test 4.2 {
+  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
+  set res [list]
+  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
+    if {$rowid==300} {
+      execsql { DELETE FROM n4 WHERE rowid=300 }
+    }
+    lappend res $rowid
+  }
+  set res
+} {500 400 300 200 100}
+
+do_test 4.3 {
+  execsql { INSERT INTO n4(rowid, n) VALUES(300, '2 3 4') }
+  set res [list]
+  db eval { SELECT rowid FROM n4 WHERE n4 MATCH '3' ORDER BY rowid DESC} {
+    if {$rowid==300} {
+      execsql { DELETE FROM n4  }
+    }
+    lappend res $rowid
+  }
+  set res
+} {500 400 300}
+
 
 
 finish_test

ext/fts5/test/fts5tokenizer.test

@@ -37,6 +37,14 @@ do_execsql_test 1.4 {
   DROP TABLE ft1;
 }
 
+do_catchsql_test 1.5 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'nosuch');
+} {1 {no such tokenizer: nosuch}}
+
+do_catchsql_test 1.6 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter nosuch');
+} {1 {error in tokenizer constructor}}
+
 do_execsql_test 2.0 {
   CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
   INSERT INTO ft1 VALUES('embedded databases');

manifest

@@ -1,5 +1,5 @@
-C Fix\san\sfts5\sbug\sin\shandling\swrites\swhile\sthere\sare\sactive\scursors.
+C Improve\sfts5\stests.
-D 2015-04-28T20:24:50.023
+D 2015-04-29T20:54:08.849
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,23 +104,23 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
 F ext/fts3/unicode/mkunicode.tcl 159c1194da0bc72f51b3c2eb71022568006dc5ad
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
-F ext/fts5/fts5.c 45e82b584bb3fce22c82d8521bccaa2a6e9202fe
+F ext/fts5/fts5.c 932284a253cc9bb32caf047879669720680be2c3
 F ext/fts5/fts5.h 24a2cc35b5e76eec57b37ba48c12d9d2cb522b3a
-F ext/fts5/fts5Int.h f573fe6c50471f1d66682fce282da801009c54e1
+F ext/fts5/fts5Int.h 2e0a1a6b77e1e014b7e9b1479ca686ff79930457
-F ext/fts5/fts5_aux.c fcea18b1a2a3f95a498b52aba2983557d7678a22
+F ext/fts5/fts5_aux.c d53f00f31ad615ca4f139dd8751f9041afa00971
 F ext/fts5/fts5_buffer.c 8c8cfe7f09ca2767ab53ea883f9a0af0edb6bbae
-F ext/fts5/fts5_config.c f344ffa24d2add70fd5bde2b73c44846ad7a06bd
+F ext/fts5/fts5_config.c ecbbd5163758a958106867051892e0dfecf68b5c
-F ext/fts5/fts5_expr.c e647a2626af5c80a6325532b23eea6d1eb252a78
+F ext/fts5/fts5_expr.c 663c75dfdb1bfd8809d696357d7b55f507815098
 F ext/fts5/fts5_hash.c 29d8b0668727863cc1f1efa65efe4dd78635b016
 F ext/fts5/fts5_index.c de588982b0237b1605d6c37afd115b34c95c3da1
 F ext/fts5/fts5_storage.c ef60fc9dcc4e274f9589165e26833173c273ae18
-F ext/fts5/fts5_tcl.c af1d37fa93bcabc926aa4e89500adedbbe84a520
+F ext/fts5/fts5_tcl.c 19ab8cfa642950648968dcf25075d6d969900524
-F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
+F ext/fts5/fts5_tokenize.c 830eae0d35a5a5a90af34df65da3427f46d942fc
 F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
-F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c
+F ext/fts5/test/fts5aa.test e17657bd749cb3982745ec503ce22372dee7340d
 F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
 F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e
 F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d
@@ -131,30 +131,34 @@ F ext/fts5/test/fts5ah.test d74cf8b7de5b8424f732acef69fe12122a12f2bf
 F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
 F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
 F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
-F ext/fts5/test/fts5al.test 6a5717faaf7f1e0e866360022d284903f3a4eede
+F ext/fts5/test/fts5al.test e6bddd2c11c0d1e3ae189ee51081899d2f4ea570
+F ext/fts5/test/fts5aux.test 1e475d928a3d1decf74167394db20330d7beeb0f
 F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
-F ext/fts5/test/fts5content.test c4f5b0fe1bc7523bb6706591d05d1194a0aec452
+F ext/fts5/test/fts5content.test 532e15b541254410adc7bfb51f94631cfe82de8f
-F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
+F ext/fts5/test/fts5corrupt.test 138aecc75c36c3dac9259c7f57c5bc3d009255f8
 F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0
 F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
+F ext/fts5/test/fts5doclist.test 635b80ac785627841a59c583bac702b55d49fdc5
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d
 F ext/fts5/test/fts5fault2.test 26c3d70648f691e2cc9391e14bbc11a973656383
 F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
-F ext/fts5/test/fts5fault4.test e860e0cf7e56f2f87330023be1f1ced44128d5c8
+F ext/fts5/test/fts5fault4.test aea710bbf5680ed41afb9d3313c297d429f9feac
 F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
 F ext/fts5/test/fts5hash.test adb7b0442cc1c77c507f07e16d11490486e75dfa
 F ext/fts5/test/fts5merge.test 453a0717881aa7784885217b2040f3f275caff03
 F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
+F ext/fts5/test/fts5plan.test 89783f70dab89ff936ed6f21d88959b49c853a47
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
 F ext/fts5/test/fts5prefix.test 1287803c3df0e43f536196256fb9e0e6baccb4f1
-F ext/fts5/test/fts5rebuild.test ee6792715c6c528cc188e7869d67c3c655889ddb
+F ext/fts5/test/fts5rank.test f59a6b20ec8e08cb130d833dcece59cf9cd92890
-F ext/fts5/test/fts5restart.test 4934233dd2633665198d563c79f1015e3a2b9518
+F ext/fts5/test/fts5rebuild.test 77c6613aa048f38b4a12ddfacb2e6e1342e1b066
+F ext/fts5/test/fts5restart.test cd58a5fb552ac10db549482698e503f82693bcd0
 F ext/fts5/test/fts5rowid.test a1b2a6d76648c734c1aab11ee1a619067e8d90e6
-F ext/fts5/test/fts5tokenizer.test 7a6ee24db908c09a0dc1eba634ffa17afcc05d86
+F ext/fts5/test/fts5tokenizer.test bbcde2a7473dcaa9a1fc6809aa8965acb7b846ff
 F ext/fts5/test/fts5unicode.test 79b3e34eb29ce4929628aa514a40cb467fdabe4d
 F ext/fts5/test/fts5unicode2.test 64a5267fd6082fcb46439892ebd0cbaa5c38acee
 F ext/fts5/test/fts5unindexed.test f388605341a476b6ab622b4c267cd168f59a5944
@@ -1311,7 +1315,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 8e8136f2dc08082c2984462719d9cba0f212c92a
+P 07f70955392697556ca2951c9b6c3a5204cd5ec0
-R 65017a9d42efb9cfacfe6e269e92cdbd
+R 07c0e8a626b67779b0890a98070e21c4
 U dan
-Z 91e05f6f4d01e29ed38305d1f49be921
+Z 5236003ea42b936c8f2a3578dc6f6593

manifest.uuid

@@ -1 +1 @@
-07f70955392697556ca2951c9b6c3a5204cd5ec0
+c1f07a3aa98eac87e2747527d15e5e5562221ceb