Add tests for fts5.

FossilOrigin-Name: e748651c940eae2389fe826cf5c25f1166a5e611

ext/fts5/fts5_index.c

@@ -256,6 +256,11 @@
 */
 #define FTS5_SEGMENT_MAX_HEIGHT ((1 << FTS5_DATA_HEIGHT_B)-1)
 
+/*
+** Maximum segments permitted in a single index 
+*/
+#define FTS5_MAX_SEGMENT 2000
+
 /*
 ** The rowid for the doclist index associated with leaf page pgno of segment
 ** segid in index idx.
@@ -365,6 +370,7 @@ struct Fts5StructureLevel {
 };
 struct Fts5Structure {
   u64 nWriteCounter;              /* Total leaves written to level 0 */
+  int nSegment;                   /* Total segments in this structure */
   int nLevel;                     /* Number of levels in this index */
   Fts5StructureLevel aLevel[0];   /* Array of nLevel level objects */
 };
@@ -725,6 +731,7 @@ static int fts5BufferCompareBlob(
   return (res==0 ? (pLeft->n - nRight) : res);
 }
 
+
 /*
 ** Compare the contents of the two buffers using memcmp(). If one buffer
 ** is a prefix of the other, it is considered the lesser.
@@ -740,6 +747,17 @@ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){
   return (res==0 ? (pLeft->n - pRight->n) : res);
 }
 
+#ifdef SQLITE_DEBUG
+static int fts5BlobCompare(
+  const u8 *pLeft, int nLeft, 
+  const u8 *pRight, int nRight
+){
+  int nCmp = MIN(nLeft, nRight);
+  int res = memcmp(pLeft, pRight, nCmp);
+  return (res==0 ? (nLeft - nRight) : res);
+}
+#endif
+
 
 /*
 ** Close the read-only blob handle, if it is open.
@@ -1042,6 +1060,7 @@ static int fts5StructureDecode(
 
   if( pRet ){
     pRet->nLevel = nLevel;
+    pRet->nSegment = nSegment;
     i += sqlite3GetVarint(&pData[i], &pRet->nWriteCounter);
 
     for(iLvl=0; rc==SQLITE_OK && iLvl<nLevel; iLvl++){
@@ -1166,8 +1185,10 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p, int iIdx){
 }
 
 /*
-** Return the total number of segments in index structure pStruct.
+** Return the total number of segments in index structure pStruct. This
+** function is only ever used as part of assert() conditions.
 */
+#ifdef SQLITE_DEBUG
 static int fts5StructureCountSegments(Fts5Structure *pStruct){
   int nSegment = 0;               /* Total number of segments */
   if( pStruct ){
@@ -1179,6 +1200,7 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){
 
   return nSegment;
 }
+#endif
 
 /*
 ** Serialize and store the "structure" record for index iIdx.
@@ -1188,12 +1210,11 @@ static int fts5StructureCountSegments(Fts5Structure *pStruct){
 */
 static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
   if( p->rc==SQLITE_OK ){
-    int nSegment;                 /* Total number of segments */
     Fts5Buffer buf;               /* Buffer to serialize record into */
     int iLvl;                     /* Used to iterate through levels */
     int iCookie;                  /* Cookie value to store */
 
-    nSegment = fts5StructureCountSegments(pStruct);
+    assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
     memset(&buf, 0, sizeof(Fts5Buffer));
 
     /* Append the current configuration cookie */
@@ -1202,7 +1223,7 @@ static void fts5StructureWrite(Fts5Index *p, int iIdx, Fts5Structure *pStruct){
     fts5BufferAppend32(&p->rc, &buf, iCookie);
 
     fts5BufferAppendVarint(&p->rc, &buf, pStruct->nLevel);
-    fts5BufferAppendVarint(&p->rc, &buf, nSegment);
+    fts5BufferAppendVarint(&p->rc, &buf, pStruct->nSegment);
     fts5BufferAppendVarint(&p->rc, &buf, (i64)pStruct->nWriteCounter);
 
     for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
@@ -1583,6 +1604,7 @@ static void fts5SegIterNextPage(
 static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
   int nSz;
   int n = fts5GetVarint32(p, nSz);
+  assert_nc( nSz>=0 );
   *pnSz = nSz/2;
   *pbDel = nSz & 0x0001;
   return n;
@@ -1601,9 +1623,13 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){
 */
 static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
   if( p->rc==SQLITE_OK ){
-    const u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
     int iOff = pIter->iLeafOffset;  /* Offset to read at */
-    pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
+    if( iOff>=pIter->pLeaf->n ){
+      p->rc = FTS5_CORRUPT;
+    }else{
+      const u8 *a = &pIter->pLeaf->p[iOff];
+      pIter->iLeafOffset += fts5GetPoslistSize(a, &pIter->nPos, &pIter->bDel);
+    }
   }
 }
 
@@ -2028,18 +2054,16 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
   if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
     int iOff = pIter->iLeafOffset + pIter->nPos;
     while( iOff<pLeaf->n ){
+      int bDummy;
+      int nPos;
       i64 iDelta;
 
       /* iOff is currently the offset of the start of position list data */
       iOff += getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
       if( iDelta==0 ) return;
-
+      assert_nc( iOff<pLeaf->n );
-      if( iOff<pLeaf->n ){
+      iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
-        int bDummy;
+      iOff += nPos;
-        int nPos;
-        iOff += fts5GetPoslistSize(&pLeaf->p[iOff], &nPos, &bDummy);
-        iOff += nPos;
-      }
     }
   }
 
@@ -2346,11 +2370,9 @@ static void fts5SegIterGotoPage(
   int iLeafPgno
 ){
   assert( iLeafPgno>pIter->iLeafPgno );
-  if( p->rc==SQLITE_OK ){
+  pIter->iLeafPgno = iLeafPgno-1;
-    pIter->iLeafPgno = iLeafPgno-1;
+  fts5SegIterNextPage(p, pIter);
-    fts5SegIterNextPage(p, pIter);
+  assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
-    assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
-  }
 
   if( p->rc==SQLITE_OK ){
     int iOff;
@@ -2413,7 +2435,7 @@ static void fts5SegIterNextFrom(
     }
   }
 
-  while( 1 ){
+  while( p->rc==SQLITE_OK ){
     if( bMove ) fts5SegIterNext(p, pIter, 0);
     if( pIter->pLeaf==0 ) break;
     if( bRev==0 && pIter->iRowid>=iMatch ) break;
@@ -2502,6 +2524,7 @@ static void fts5MultiIterNext(
       int iFirst = pIter->aFirst[1].iFirst;
       int bNewTerm = 0;
       Fts5SegIter *pSeg = &pIter->aSeg[iFirst];
+      assert( p->rc==SQLITE_OK );
       if( bUseFrom && pSeg->pDlidx ){
         fts5SegIterNextFrom(p, pSeg, iFrom);
       }else{
@@ -2542,8 +2565,8 @@ static void fts5MultiIterNew(
   int nSegment,                   /* Number of segments to merge (iLevel>=0) */
   Fts5MultiSegIter **ppOut        /* New object */
 ){
-  int nSeg;                       /* Number of segments merged */
+  int nSeg;                       /* Number of segment-iters in use */
-  int nSlot;                      /* Power of two >= nSeg */
+  int nSlot = 0;                  /* Power of two >= nSeg */
   int iIter = 0;                  /* */
   int iSeg;                       /* Used to iterate through segments */
   Fts5StructureLevel *pLvl;
@@ -2552,13 +2575,17 @@ static void fts5MultiIterNew(
   assert( (pTerm==0 && nTerm==0) || iLevel<0 );
 
   /* Allocate space for the new multi-seg-iterator. */
-  if( iLevel<0 ){
+  if( p->rc==SQLITE_OK ){
-    nSeg = fts5StructureCountSegments(pStruct);
+    if( iLevel<0 ){
-    nSeg += (p->apHash ? 1 : 0);
+      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
-  }else{
+      nSeg = pStruct->nSegment;
-    nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
+      nSeg += (p->apHash ? 1 : 0);
+    }else{
+      nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
+    }
+    for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
   }
-  for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
+
   *ppOut = pNew = fts5IdxMalloc(p, 
       sizeof(Fts5MultiSegIter) +          /* pNew */
       sizeof(Fts5SegIter) * nSlot +       /* pNew->aSeg[] */
@@ -2732,101 +2759,41 @@ static void fts5ChunkIterRelease(Fts5ChunkIter *pIter){
   pIter->pLeaf = 0;
 }
 
-/*
-** Read and return the next 32-bit varint from the position-list iterator 
-** passed as the second argument.
-**
-** If an error occurs, zero is returned an an error code left in 
-** Fts5Index.rc. If an error has already occurred when this function is
-** called, it is a no-op.
-*/
-static int fts5PosIterReadVarint(Fts5Index *p, Fts5PosIter *pIter){
-  int iVal = 0;
-  if( p->rc==SQLITE_OK ){
-    if( pIter->iOff>=pIter->chunk.n ){
-      fts5ChunkIterNext(p, &pIter->chunk);
-      if( fts5ChunkIterEof(p, &pIter->chunk) ) return 0;
-      pIter->iOff = 0;
-    }
-    pIter->iOff += fts5GetVarint32(&pIter->chunk.p[pIter->iOff], iVal);
-  }
-  return iVal;
-}
 
 /*
-** Advance the position list iterator to the next entry.
+** Allocate a new segment-id for the structure pStruct. The new segment
-*/
+** id must be between 1 and 65335 inclusive, and must not be used by 
-static void fts5PosIterNext(Fts5Index *p, Fts5PosIter *pIter){
+** any currently existing segment. If a free segment id cannot be found,
-  int iVal;
+** SQLITE_FULL is returned.
-  assert( fts5ChunkIterEof(p, &pIter->chunk)==0 );
-  iVal = fts5PosIterReadVarint(p, pIter);
-  if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){
-    if( iVal==1 ){
-      pIter->iCol = fts5PosIterReadVarint(p, pIter);
-      pIter->iPos = fts5PosIterReadVarint(p, pIter) - 2;
-    }else{
-      pIter->iPos += (iVal - 2);
-    }
-  }
-}
-
-/*
-** Initialize the Fts5PosIter object passed as the final argument to iterate
-** through the position-list associated with the index entry that iterator 
-** pMulti currently points to.
-*/
-static void fts5PosIterInit(
-  Fts5Index *p,                   /* FTS5 backend object */
-  Fts5MultiSegIter *pMulti,       /* Multi-seg iterator to read pos-list from */
-  Fts5PosIter *pIter              /* Initialize this object */
-){
-  if( p->rc==SQLITE_OK ){
-    Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
-    memset(pIter, 0, sizeof(*pIter));
-    fts5ChunkIterInit(p, pSeg, &pIter->chunk);
-    if( fts5ChunkIterEof(p, &pIter->chunk)==0 ){
-      fts5PosIterNext(p, pIter);
-    }
-  }
-}
-
-/*
-** Return true if the position iterator passed as the second argument is
-** at EOF. Or if an error has already occurred. Otherwise, return false.
-*/
-static int fts5PosIterEof(Fts5Index *p, Fts5PosIter *pIter){
-  return (p->rc || pIter->chunk.pLeaf==0);
-}
-
-/*
-** Allocate a new segment-id for the structure pStruct.
 **
 ** If an error has already occurred, this function is a no-op. 0 is 
 ** returned in this case.
 */
 static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){
   int i;
-  if( p->rc!=SQLITE_OK ) return 0;
+  u32 iSegid = 0;
 
-  for(i=0; i<100; i++){
+  if( p->rc==SQLITE_OK ){
-    int iSegid;
+    if( pStruct->nSegment>=FTS5_MAX_SEGMENT ){
-    sqlite3_randomness(sizeof(int), (void*)&iSegid);
+      p->rc = SQLITE_FULL;
-    iSegid = iSegid & ((1 << FTS5_DATA_ID_B)-1);
+    }else{
-    if( iSegid ){
+      while( iSegid==0 ){
-      int iLvl, iSeg;
+        int iLvl, iSeg;
-      for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+        sqlite3_randomness(sizeof(u32), (void*)&iSegid);
-        for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+        iSegid = (iSegid % ((1 << FTS5_DATA_ID_B) - 2)) + 1;
-          if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
+        assert( iSegid>0 && iSegid<=65535 );
-            iSegid = 0;
+        for(iLvl=0; iLvl<pStruct->nLevel; iLvl++){
+          for(iSeg=0; iSeg<pStruct->aLevel[iLvl].nSeg; iSeg++){
+            if( iSegid==pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ){
+              iSegid = 0;
+            }
           }
         }
       }
     }
-    if( iSegid ) return iSegid;
   }
 
-  p->rc = SQLITE_ERROR;
+  return (int)iSegid;
-  return 0;
 }
 
 /*
@@ -2838,7 +2805,7 @@ static void fts5IndexDiscardData(Fts5Index *p){
     Fts5Config *pConfig = p->pConfig;
     int i;
     for(i=0; i<=pConfig->nPrefix; i++){
-      if( p->apHash[i] ) sqlite3Fts5HashClear(p->apHash[i]);
+      sqlite3Fts5HashClear(p->apHash[i]);
     }
     p->nPendingData = 0;
   }
@@ -2853,7 +2820,8 @@ static int fts5PrefixCompress(
   int nNew, const u8 *pNew
 ){
   int i;
-  for(i=0; i<nNew && i<nOld; i++){
+  assert( fts5BlobCompare(pOld, nOld, pNew, nNew)<0 );
+  for(i=0; i<nOld; i++){
     if( pOld[i]!=pNew[i] ) break;
   }
   return i;
@@ -2861,14 +2829,14 @@ static int fts5PrefixCompress(
 
 /*
 ** If an "nEmpty" record must be written to the b-tree before the next
-** term, write it now.
+** term, write it now. 
 */
 static void fts5WriteBtreeNEmpty(Fts5Index *p, Fts5SegWriter *pWriter){
   if( pWriter->nEmpty ){
     int bFlag = 0;
     Fts5PageWriter *pPg;
     pPg = &pWriter->aWriter[1];
-    if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE && pWriter->cdlidx.n ){
+    if( pWriter->nEmpty>=FTS5_MIN_DLIDX_SIZE ){
       i64 iKey = FTS5_DOCLIST_IDX_ROWID(
           pWriter->iIdx, pWriter->iSegid, 
           pWriter->aWriter[0].pgno - 1 - pWriter->nEmpty
@@ -3040,8 +3008,8 @@ static void fts5WriteAppendTerm(
   int nPrefix;                    /* Bytes of prefix compression for term */
   Fts5PageWriter *pPage = &pWriter->aWriter[0];
 
-  assert( pPage==0 || pPage->buf.n==0 || pPage->buf.n>4 );
+  assert( pPage->buf.n==0 || pPage->buf.n>4 );
-  if( pPage && pPage->buf.n==0 ){
+  if( pPage->buf.n==0 ){
     /* Zero the first term and first docid fields */
     static const u8 zero[] = { 0x00, 0x00, 0x00, 0x00 };
     fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
@@ -3137,6 +3105,7 @@ static void fts5WriteAppendRowid(
   }
 }
 
+#if 0
 static void fts5WriteAppendPoslistInt(
   Fts5Index *p, 
   Fts5SegWriter *pWriter,
@@ -3150,6 +3119,7 @@ static void fts5WriteAppendPoslistInt(
     }
   }
 }
+#endif
 
 static void fts5WriteAppendPoslistData(
   Fts5Index *p, 
@@ -3389,6 +3359,7 @@ static void fts5IndexMergeLevel(
     pLvlOut->nSeg++;
     pSeg->pgnoFirst = 1;
     pSeg->iSegid = iSegid;
+    pStruct->nSegment++;
 
     /* Read input from all segments in the input level */
     nInput = pLvl->nSeg;
@@ -3461,10 +3432,12 @@ fflush(stdout);
       int nMove = (pLvl->nSeg - nInput) * sizeof(Fts5StructureSegment);
       memmove(pLvl->aSeg, &pLvl->aSeg[nInput], nMove);
     }
+    pStruct->nSegment -= nInput;
     pLvl->nSeg -= nInput;
     pLvl->nMerge = 0;
     if( pSeg->pgnoLast==0 ){
       pLvlOut->nSeg--;
+      pStruct->nSegment--;
     }
   }else{
     assert( pSeg->nHeight>0 && pSeg->pgnoLast>0 );
@@ -3768,6 +3741,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
       pSeg->nHeight = nHeight;
       pSeg->pgnoFirst = 1;
       pSeg->pgnoLast = pgnoLast;
+      pStruct->nSegment++;
     }
     fts5StructurePromote(p, 0, pStruct);
   }
@@ -3809,7 +3783,8 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
     Fts5Structure *pNew = 0;
     int nSeg = 0;
     if( pStruct ){
-      nSeg = fts5StructureCountSegments(pStruct);
+      assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
+      nSeg = pStruct->nSegment;
       if( nSeg>1 ){
         int nByte = sizeof(Fts5Structure);
         nByte += (pStruct->nLevel+1) * sizeof(Fts5StructureLevel);
@@ -3832,7 +3807,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
             iSegOut++;
           }
         }
-        pLvl->nSeg = nSeg;
+        pNew->nSegment = pLvl->nSeg = nSeg;
       }else{
         sqlite3_free(pNew);
         pNew = 0;
@@ -4146,7 +4121,11 @@ static void fts5MultiIterPoslist(
     Fts5ChunkIter iter;
     Fts5SegIter *pSeg = &pMulti->aSeg[ pMulti->aFirst[1].iFirst ];
     assert( fts5MultiIterEof(p, pMulti)==0 );
+    static int nCall = 0;
+    nCall++;
+
     fts5ChunkIterInit(p, pSeg, &iter);
+
     if( fts5ChunkIterEof(p, &iter)==0 ){
       if( bSz ){
         /* WRITEPOSLISTSIZE */
@@ -4434,6 +4413,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   u64 cksum2 = 0;                 /* Checksum based on contents of indexes */
   u64 cksum3 = 0;                 /* Checksum based on contents of indexes */
   Fts5Buffer term = {0,0,0};      /* Buffer used to hold most recent term */
+  Fts5Buffer poslist = {0,0,0};   /* Buffer used to hold a poslist */
 
   /* Check that the internal nodes of each segment match the leaves */
   for(iIdx=0; p->rc==SQLITE_OK && iIdx<=pConfig->nPrefix; iIdx++){
@@ -4470,18 +4450,18 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
         fts5MultiIterEof(p, pIter)==0;
         fts5MultiIterNext(p, pIter, 0, 0)
     ){
-      Fts5PosIter sPos;           /* Used to iterate through position list */
       int n;                      /* Size of term in bytes */
+      i64 iPos = 0;               /* Position read from poslist */
+      int iOff = 0;               /* Offset within poslist */
       i64 iRowid = fts5MultiIterRowid(pIter);
       char *z = (char*)fts5MultiIterTerm(pIter, &n);
 
-      /* Update cksum2 with the entries associated with the current term
+      poslist.n = 0;
-      ** and rowid.  */
+      fts5MultiIterPoslist(p, pIter, 0, &poslist);
-      for(fts5PosIterInit(p, pIter, &sPos);
+      while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){
-          fts5PosIterEof(p, &sPos)==0;
+        int iCol = FTS5_POS2COLUMN(iPos);
-          fts5PosIterNext(p, &sPos)
+        int iTokOff = FTS5_POS2OFFSET(iPos);
-      ){
+        cksum2 ^= fts5IndexEntryCksum(iRowid, iCol, iTokOff, z, n);
-        cksum2 ^= fts5IndexEntryCksum(iRowid, sPos.iCol, sPos.iPos, z, n);
       }
 
       /* If this is a new term, query for it. Update cksum3 with the results. */
@@ -4526,6 +4506,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
   if( p->rc==SQLITE_OK && cksum!=cksum3 ) p->rc = FTS5_CORRUPT;
 
   fts5BufferFree(&term);
+  fts5BufferFree(&poslist);
   return fts5IndexReturn(p);
 }
 

ext/fts5/test/fts5aa.test

@@ -47,11 +47,9 @@ do_execsql_test 2.0 {
 do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
 }
-do_execsql_test 2.2 {
+do_test 2.2 {
-  SELECT fts5_decode(id, block) FROM t1_data WHERE id==10
+  execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
-} {
+} {/{{structure idx=0} {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}
-  {{structure idx=0} {lvl=0 nMerge=0 {id=27723 h=1 leaves=1..1}}}
-}
 do_execsql_test 2.3 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 }

ext/fts5/test/fts5ab.test

@@ -263,5 +263,27 @@ do_execsql_test 6.3 {
   SELECT rowid FROM s3 WHERE s3 MATCH 'a'
 } {1 2}
 
+do_test 6.4 {
+  db close
+  sqlite3 db test.db
+  execsql {
+    BEGIN;
+      INSERT INTO s3(s3) VALUES('optimize');
+    ROLLBACK;
+  }
+} {}
+
+#-------------------------------------------------------------------------
+#
+set doc [string repeat "a b c " 500]
+breakpoint
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
+  INSERT INTO x1 VALUES($doc);
+}
+
+
+
 finish_test
 

ext/fts5/test/fts5corrupt2.test

@@ -28,7 +28,6 @@ do_execsql_test 1.0 {
   WITH ii(i) AS (SELECT 1 UNION SELECT i+1 FROM ii WHERE i<100)
   INSERT INTO t1 SELECT rnddoc(10) FROM ii;
 }
-
 set mask [expr 31 << 31]
 
 # Test 1:
@@ -51,7 +50,6 @@ foreach {tno stmt} {
   1 { DELETE FROM t1_data WHERE rowid=$rowid }
   2 { UPDATE t1_data SET block=14 WHERE rowid=$rowid }
 } {
-  break
   set tn 0
   foreach rowid [db eval {SELECT rowid FROM t1_data WHERE rowid>10}] {
     incr tn
@@ -77,6 +75,8 @@ foreach {tno stmt} {
   }
 }
 
+# Using the same database as the 1.* tests.
+#
 # Run N-1 tests, where N is the number of bytes in the rightmost leaf page
 # of the fts index. For test $i, truncate the rightmost leafpage to $i
 # bytes. Then test both the integrity-check detects the corruption.
@@ -112,5 +112,87 @@ for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
   } {}
 }
 
+#-------------------------------------------------------------------------
+# Test that corruption in leaf page headers is detected by queries that use
+# doclist-indexes.
+#
+set doc "A B C D E F G H I J "
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE x3 USING fts5(tt);
+  INSERT INTO x3(x3, rank) VALUES('pgsz', 32);
+  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<1000) 
+  INSERT INTO x3 
+  SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
+}
+
+foreach {tn hdr} {
+  1 "\00\00\00\00"
+  2 "\FF\FF\FF\FF"
+} {
+  set tn2 0
+  set nCorrupt 0
+  foreach rowid [db eval {SELECT rowid FROM x3_data WHERE rowid>10}] {
+    if {$rowid & $mask} continue
+    incr tn2
+    do_test 3.$tn.$tn2 {
+      execsql BEGIN
+
+      set fd [db incrblob main x3_data block $rowid]
+      fconfigure $fd -encoding binary -translation binary
+      puts -nonewline $fd $hdr
+      close $fd
+
+      set res [catchsql {SELECT rowid FROM x3 WHERE x3 MATCH 'x AND a'}]
+      if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
+      set {} 1
+    } {1}
+
+    execsql ROLLBACK
+  }
+
+  do_test 3.$tn.x { expr $nCorrupt>0 } 1
+}
+
+#--------------------------------------------------------------------
+#
+set doc "A B C D E F G H I J "
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x4 USING fts5(tt);
+  INSERT INTO x4(x4, rank) VALUES('pgsz', 32);
+  WITH ii(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM ii WHERE i<10) 
+  INSERT INTO x4 
+  SELECT ($doc || CASE WHEN (i%50)==0 THEN 'X' ELSE 'Y' END) FROM ii;
+}
+
+foreach {tn nCut} {
+  1 1
+  2 10
+} {
+  set tn2 0
+  set nCorrupt 0
+  foreach rowid [db eval {SELECT rowid FROM x4_data WHERE rowid>10}] {
+    if {$rowid & $mask} continue
+    incr tn2
+    do_test 4.$tn.$tn2 {
+      execsql {
+        BEGIN;
+          UPDATE x4_data SET block = substr(block, 1, length(block)-$nCut) 
+          WHERE id = $rowid;
+      }
+
+      set res [catchsql {
+        SELECT rowid FROM x4 WHERE x4 MATCH 'a' ORDER BY 1 DESC
+      }]
+      if {$res == "1 {database disk image is malformed}"} {incr nCorrupt}
+      set {} 1
+    } {1}
+
+    execsql ROLLBACK
+  }
+
+  do_test 4.$tn.x { expr $nCorrupt>0 } 1
+}
+
+
 finish_test
 

ext/fts5/test/fts5fault2.test

@@ -51,5 +51,28 @@ do_faultsim_test 1.2 -faults oom-* -prep {
   faultsim_test_result {0 {1000 900 800 700 600 500 400 300 200 100}}
 }
 
+#-------------------------------------------------------------------------
+# OOM within a query that accesses the in-memory hash table. 
+#
+reset_db 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE "a b c" USING fts5(a, b, c);
+  INSERT INTO "a b c" VALUES('one two', 'x x x', 'three four');
+  INSERT INTO "a b c" VALUES('nine ten', 'y y y', 'two two');
+}
+
+do_faultsim_test 2.1 -faults oom-trans* -prep {
+  execsql {
+    BEGIN;
+      INSERT INTO "a b c" VALUES('one one', 'z z z', 'nine ten');
+  }
+} -body {
+  execsql { SELECT rowid FROM "a b c" WHERE "a b c" MATCH 'one' }
+} -test {
+  faultsim_test_result {0 {1 3}}
+  catchsql { ROLLBACK }
+}
+
+
 finish_test
 

ext/fts5/test/fts5full.test

@@ -0,0 +1,37 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test that SQLITE_FULL is returned if the FTS5 table cannot find a free 
+# segid to use. In practice this can only really happen when automerge and
+# crisismerge are both disabled.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5full
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x8 USING fts5(i);
+  INSERT INTO x8(x8, rank) VALUES('automerge', 0);
+  INSERT INTO x8(x8, rank) VALUES('crisismerge', 100000);
+}
+
+db func rnddoc fts5_rnddoc
+do_test 1.1 {
+  list [catch {
+    for {set i 0} {$i < 2500} {incr i} {
+      execsql { INSERT INTO x8 VALUES( rnddoc(5) ); }
+    }
+  } msg] $msg
+} {1 {database or disk is full}}
+
+
+finish_test
+

manifest

@@ -1,5 +1,5 @@
-C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
+C Add\stests\sfor\sfts5.
-D 2015-04-24T20:18:21.844
+D 2015-04-25T18:56:48.351
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 31b38b9da2e4b36f54a013bd71a5c3f6e45ca78f
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
 F ext/fts5/fts5_config.c 43fcf838d3a3390d1245e3d5e651fa5cc1df575b
 F ext/fts5/fts5_expr.c 05da381ab26031243266069302c6eb4094b2c5dd
 F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
-F ext/fts5/fts5_index.c 39810b25a017f2626ac72b3e44afe9b534e5d5db
+F ext/fts5/fts5_index.c c87369d11271847df9f033f0df148e7f004a88a2
 F ext/fts5/fts5_storage.c b3a4cbbcd197fe587789398e51a631f92fc9196c
 F ext/fts5/fts5_tcl.c 10bf0eb678d34c1bfdcfaf653d2e6dd92afa8b38
 F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
@@ -120,8 +120,8 @@ F ext/fts5/fts5_unicode2.c f74f53316377068812a1fa5a37819e6b8124631d
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
-F ext/fts5/test/fts5aa.test 91f22b3cc7b372a2903c828e907a1e52f1177b8a
+F ext/fts5/test/fts5aa.test 87b2e01084c3e2a960ae1500dd5f0729dac2166c
-F ext/fts5/test/fts5ab.test 5da2e92a8047860b9e22b6fd3990549639d631b1
+F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
 F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e
 F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d
 F ext/fts5/test/fts5ae.test 9175201baf8c885fc1cbb2da11a0c61fd11224db
@@ -136,12 +136,13 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
 F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
 F ext/fts5/test/fts5corrupt.test 9e8524281aa322c522c1d6e2b347e24e060c2727
-F ext/fts5/test/fts5corrupt2.test 3be48d8a30d30e3ae819f04e957c45d091bfbb85
+F ext/fts5/test/fts5corrupt2.test 494111fd4f2dab36499cf97718eaba1f7c11e9d0
 F ext/fts5/test/fts5dlidx.test 748a84ceb74a4154725096a26dfa854260b0182f
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test ed71717a479bef32d05f02d9c48691011d160d4d
-F ext/fts5/test/fts5fault2.test 000ecebf28b8f2cd520f44c70962625ee11d65ac
+F ext/fts5/test/fts5fault2.test f478fa94e39a6911189f9e052a3b93ab4cd275fa
+F ext/fts5/test/fts5full.test 0924bdca5416a242103239ace79c6f5aa34bab8d
 F ext/fts5/test/fts5near.test 3f9f64e16cac82725d03d4e04c661090f0b3b947
 F ext/fts5/test/fts5optimize.test 0028c90a7817d3e576d1148fc8dff17d89054e54
 F ext/fts5/test/fts5porter.test 50322599823cb8080a99f0ec0c39f7d0c12bcb5e
@@ -1304,7 +1305,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 86309961344f4076ddcf55d730d3600ec3b6e45c dc88fe7e641c52d27fba8c753cee590db87388c5
+P 1c78d8920fb59da3cb97dd2eb09b3e08dfd14259
-R e0fc7874fca6c46f36b4c18c43a12035
+R 6572bfe0dee7d9becde3a8bb0a8d33a2
 U dan
-Z 339a1949f9a5bc9b03407831c5b374c4
+Z e7bb2196ce75c57e787365bfa027743a

manifest.uuid

@@ -1 +1 @@
-1c78d8920fb59da3cb97dd2eb09b3e08dfd14259
+e748651c940eae2389fe826cf5c25f1166a5e611