Fix some problems with transactions that both read and write an fts5 table.

FossilOrigin-Name: 0e225b15357765f132c3364b222f9931a608a5b2

ext/fts5/fts5Int.h

@@ -228,7 +228,7 @@ typedef struct Fts5IndexIter Fts5IndexIter;
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX 0x0001       /* Prefix query */
+#define FTS5INDEX_QUERY_PREFIX  0x0001      /* Prefix query */
 #define FTS5INDEX_QUERY_DESC    0x0002      /* Docs in descending rowid order */
 
 /*
@@ -259,16 +259,13 @@ int sqlite3Fts5IndexQuery(
 );
 
 /*
-** Docid list iteration.
+** The various operations on open token or token prefix iterators opened
+** using sqlite3Fts5IndexQuery().
 */
 int sqlite3Fts5IterEof(Fts5IndexIter*);
 int sqlite3Fts5IterNext(Fts5IndexIter*);
 int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch);
 i64 sqlite3Fts5IterRowid(Fts5IndexIter*);
-
-/*
-** Obtain the position list that corresponds to the current position.
-*/
 int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn);
 
 /*
@@ -365,13 +362,6 @@ int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v);
 */
 typedef struct Fts5Hash Fts5Hash;
 
-typedef struct Fts5Data Fts5Data;
-struct Fts5Data {
-  u8 *p;                          /* Pointer to buffer containing record */
-  int n;                          /* Size of record in bytes */
-  int nRef;                       /* Ref count */
-};
-
 /*
 ** Create a hash table, free a hash table.
 */
@@ -405,7 +395,20 @@ int sqlite3Fts5HashIterate(
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
   const char *pTerm, int nTerm,   /* Query term */
-  Fts5Data **ppData               /* OUT: Query result */
+  const char **ppDoclist,         /* OUT: Pointer to doclist for pTerm */
+  int *pnDoclist                  /* OUT: Size of doclist in bytes */
+);
+
+void sqlite3Fts5HashScanInit(
+  Fts5Hash*,                      /* Hash table to query */
+  const char *pTerm, int nTerm    /* Query prefix */
+);
+void sqlite3Fts5HashScanNext(Fts5Hash*);
+int sqlite3Fts5HashScanEof(Fts5Hash*);
+void sqlite3Fts5HashScanEntry(Fts5Hash *,
+  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  const char **ppDoclist,         /* OUT: pointer to doclist */
+  int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
 
 

ext/fts5/fts5_buffer.c

@@ -74,6 +74,7 @@ void sqlite3Fts5BufferAppendBlob(
   int nData, 
   const u8 *pData
 ){
+  assert( nData>=0 );
   if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return;
   memcpy(&pBuf->p[pBuf->n], pData, nData);
   pBuf->n += nData;

ext/fts5/fts5_hash.c

@@ -27,6 +27,7 @@ struct Fts5Hash {
   int *pnByte;                    /* Pointer to bytes counter */
   int nEntry;                     /* Number of entries currently in hash */
   int nSlot;                      /* Size of aSlot[] array */
+  Fts5HashEntry *pScan;           /* Current ordered scan item */
   Fts5HashEntry **aSlot;          /* Array of hash slots */
 };
 
@@ -52,7 +53,8 @@ struct Fts5Hash {
 **   Bytes of data written since iRowidOff.
 */
 struct Fts5HashEntry {
-  Fts5HashEntry *pNext;           /* Next hash entry with same hash-key */
+  Fts5HashEntry *pHashNext;       /* Next hash entry with same hash-key */
+  Fts5HashEntry *pScanNext;       /* Next entry in sorted order */
   
   int nAlloc;                     /* Total size of allocation */
   int iSzPoslist;                 /* Offset of space for 4-byte poslist size */
@@ -124,7 +126,7 @@ void sqlite3Fts5HashClear(Fts5Hash *pHash){
     Fts5HashEntry *pNext;
     Fts5HashEntry *pSlot;
     for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){
-      pNext = pSlot->pNext;
+      pNext = pSlot->pHashNext;
       sqlite3_free(pSlot);
     }
   }
@@ -158,9 +160,9 @@ static int fts5HashResize(Fts5Hash *pHash){
     while( apOld[i] ){
       int iHash;
       Fts5HashEntry *p = apOld[i];
-      apOld[i] = p->pNext;
+      apOld[i] = p->pHashNext;
       iHash = fts5HashKey(nNew, p->zKey, strlen(p->zKey));
-      p->pNext = apNew[iHash];
+      p->pHashNext = apNew[iHash];
       apNew[iHash] = p;
     }
   }
@@ -184,7 +186,7 @@ int sqlite3Fts5HashWrite(
   int nIncr = 0;                  /* Amount to increment (*pHash->pnByte) by */
 
   /* Attempt to locate an existing hash entry */
-  for(p=pHash->aSlot[iHash]; p; p=p->pNext){
+  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break;
   }
 
@@ -210,7 +212,7 @@ int sqlite3Fts5HashWrite(
     p->iSzPoslist = p->nData;
     p->nData += 4;
     p->iRowid = iRowid;
-    p->pNext = pHash->aSlot[iHash];
+    p->pHashNext = pHash->aSlot[iHash];
     pHash->aSlot[iHash] = p;
     pHash->nEntry++;
     nIncr += p->nData;
@@ -232,7 +234,7 @@ int sqlite3Fts5HashWrite(
     pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
     if( pNew==0 ) return SQLITE_NOMEM;
     pNew->nAlloc = nNew;
-    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pNext);
+    for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
     *pp = pNew;
     p = pNew;
   }
@@ -301,13 +303,13 @@ static Fts5HashEntry *fts5HashEntryMerge(
       if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){
         /* p2 is smaller */
         *ppOut = p2;
-        ppOut = &p2->pNext;
+        ppOut = &p2->pScanNext;
-        p2 = p2->pNext;
+        p2 = p2->pScanNext;
       }else{
         /* p1 is smaller */
         *ppOut = p1;
-        ppOut = &p1->pNext;
+        ppOut = &p1->pScanNext;
-        p1 = p1->pNext;
+        p1 = p1->pScanNext;
       }
       *ppOut = 0;
     }
@@ -322,7 +324,11 @@ static Fts5HashEntry *fts5HashEntryMerge(
 ** the responsibility of the caller to free the elements of the returned
 ** list.
 */
-static int fts5HashEntrySort(Fts5Hash *pHash, Fts5HashEntry **ppSorted){
+static int fts5HashEntrySort(
+  Fts5Hash *pHash, 
+  const char *pTerm, int nTerm,   /* Query prefix, if any */
+  Fts5HashEntry **ppSorted
+){
   const int nMergeSlot = 32;
   Fts5HashEntry **ap;
   Fts5HashEntry *pList;
@@ -335,15 +341,17 @@ static int fts5HashEntrySort(Fts5Hash *pHash, Fts5HashEntry **ppSorted){
   memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot);
 
   for(iSlot=0; iSlot<pHash->nSlot; iSlot++){
-    while( pHash->aSlot[iSlot] ){
+    Fts5HashEntry *pIter;
-      Fts5HashEntry *pEntry = pHash->aSlot[iSlot];
+    for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
-      pHash->aSlot[iSlot] = pEntry->pNext;
+      if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){
-      pEntry->pNext = 0;
+        Fts5HashEntry *pEntry = pIter;
-      for(i=0; ap[i]; i++){
+        pEntry->pScanNext = 0;
-        pEntry = fts5HashEntryMerge(pEntry, ap[i]);
+        for(i=0; ap[i]; i++){
-        ap[i] = 0;
+          pEntry = fts5HashEntryMerge(pEntry, ap[i]);
+          ap[i] = 0;
+        }
+        ap[i] = pEntry;
       }
-      ap[i] = pEntry;
     }
   }
 
@@ -368,10 +376,11 @@ int sqlite3Fts5HashIterate(
   Fts5HashEntry *pList;
   int rc;
 
-  rc = fts5HashEntrySort(pHash, &pList);
+  rc = fts5HashEntrySort(pHash, 0, 0, &pList);
   if( rc==SQLITE_OK ){
+    memset(pHash->aSlot, 0, sizeof(Fts5HashEntry*) * pHash->nSlot);
     while( pList ){
-      Fts5HashEntry *pNext = pList->pNext;
+      Fts5HashEntry *pNext = pList->pScanNext;
       if( rc==SQLITE_OK ){
         const int nSz = pList->nData - pList->iSzPoslist - 4;
         const int nKey = strlen(pList->zKey);
@@ -406,3 +415,70 @@ int sqlite3Fts5HashIterate(
   return rc;
 }
 
+/*
+** Query the hash table for a doclist associated with term pTerm/nTerm.
+*/
+int sqlite3Fts5HashQuery(
+  Fts5Hash *pHash,                /* Hash table to query */
+  const char *pTerm, int nTerm,   /* Query term */
+  const char **ppDoclist,         /* OUT: Pointer to doclist for pTerm */
+  int *pnDoclist                  /* OUT: Size of doclist in bytes */
+){
+  unsigned int iHash = fts5HashKey(pHash->nSlot, pTerm, nTerm);
+  Fts5HashEntry *p;
+
+  for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
+    if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break;
+  }
+
+  if( p ){
+    u8 *pPtr = (u8*)p;
+    fts5Put4ByteVarint(&pPtr[p->iSzPoslist], p->nData - p->iSzPoslist - 4);
+    *ppDoclist = &p->zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(*p) + nTerm + 1);
+  }else{
+    *ppDoclist = 0;
+    *pnDoclist = 0;
+  }
+
+  return SQLITE_OK;
+}
+
+void sqlite3Fts5HashScanInit(
+  Fts5Hash *p,                    /* Hash table to query */
+  const char *pTerm, int nTerm    /* Query prefix */
+){
+  fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
+}
+
+void sqlite3Fts5HashScanNext(Fts5Hash *p){
+  if( p->pScan ){
+    p->pScan = p->pScan->pScanNext;
+  }
+}
+
+int sqlite3Fts5HashScanEof(Fts5Hash *p){
+  return (p->pScan==0);
+}
+
+void sqlite3Fts5HashScanEntry(
+  Fts5Hash *pHash,
+  const char **pzTerm,            /* OUT: term (nul-terminated) */
+  const char **ppDoclist,         /* OUT: pointer to doclist */
+  int *pnDoclist                  /* OUT: size of doclist in bytes */
+){
+  Fts5HashEntry *p;
+  if( (p = pHash->pScan) ){
+    u8 *pPtr = (u8*)p;
+    int nTerm = strlen(p->zKey);
+    fts5Put4ByteVarint(&pPtr[p->iSzPoslist], p->nData - p->iSzPoslist - 4);
+    *pzTerm = p->zKey;
+    *ppDoclist = &p->zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(*p) + nTerm + 1);
+  }else{
+    *pzTerm = 0;
+    *ppDoclist = 0;
+    *pnDoclist = 0;
+  }
+}
+

ext/fts5/fts5_index.c

@@ -274,6 +274,7 @@ int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; }
 typedef struct Fts5BtreeIter Fts5BtreeIter;
 typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel;
 typedef struct Fts5ChunkIter Fts5ChunkIter;
+typedef struct Fts5Data Fts5Data;
 typedef struct Fts5DlidxIter Fts5DlidxIter;
 typedef struct Fts5MultiSegIter Fts5MultiSegIter;
 typedef struct Fts5NodeIter Fts5NodeIter;
@@ -286,6 +287,12 @@ typedef struct Fts5Structure Fts5Structure;
 typedef struct Fts5StructureLevel Fts5StructureLevel;
 typedef struct Fts5StructureSegment Fts5StructureSegment;
 
+struct Fts5Data {
+  u8 *p;                          /* Pointer to buffer containing record */
+  int n;                          /* Size of record in bytes */
+  int nRef;                       /* Ref count */
+};
+
 /*
 ** One object per %_data table.
 */
@@ -1514,7 +1521,7 @@ static void fts5SegIterNextPage(
   Fts5SegIter *pIter              /* Iterator to advance to next page */
 ){
   Fts5StructureSegment *pSeg = pIter->pSeg;
-  if( pIter->pLeaf ) fts5DataRelease(pIter->pLeaf);
+  fts5DataRelease(pIter->pLeaf);
   pIter->iLeafPgno++;
   if( pIter->iLeafPgno<=pSeg->pgnoLast ){
     pIter->pLeaf = fts5DataRead(p, 
@@ -1775,6 +1782,26 @@ static void fts5SegIterNext(
         }else{
           pIter->iRowid += iDelta;
         }
+      }else if( pIter->pSeg==0 ){
+        const char *pList = 0;
+        const char *zTerm;
+        int nList;
+        if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){
+          sqlite3Fts5HashScanNext(p->apHash[0]);
+          sqlite3Fts5HashScanEntry(p->apHash[0], &zTerm, &pList, &nList);
+        }
+        if( pList==0 ){
+          fts5DataRelease(pIter->pLeaf);
+          pIter->pLeaf = 0;
+        }else{
+          pIter->pLeaf->p = (u8*)pList;
+          pIter->pLeaf->n = nList;
+          sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm);
+          pIter->iLeafOffset = getVarint((u8*)pList, (u64*)&pIter->iRowid);
+          if( pIter->flags & FTS5_SEGITER_REVERSE ){
+            fts5SegIterReverseInitPage(p, pIter);
+          }
+        }
       }else{
         iOff = 0;
         /* Next entry is not on the current page */
@@ -2018,6 +2045,58 @@ static void fts5SegIterSeekInit(
   }
 }
 
+/*
+** Initialize the object pIter to point to term pTerm/nTerm within the
+** in-memory hash table iIdx. If there is no such term in the table, the 
+** iterator is set to EOF.
+**
+** If an error occurs, Fts5Index.rc is set to an appropriate error code. If 
+** an error has already occurred when this function is called, it is a no-op.
+*/
+static void fts5SegIterHashInit(
+  Fts5Index *p,                   /* FTS5 backend */
+  int iIdx,                       /* Config.aHash[] index of FTS index */
+  const u8 *pTerm, int nTerm,     /* Term to seek to */
+  int flags,                      /* Mask of FTS5INDEX_XXX flags */
+  Fts5SegIter *pIter              /* Object to populate */
+){
+  Fts5Hash *pHash = p->apHash[iIdx];
+  const char *pList = 0;
+  int nList = 0;
+  const u8 *z = 0;
+  int n = 0;
+
+  assert( pHash );
+
+  if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){
+    sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm);
+    sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList);
+    n = (z ? strlen((const char*)z) : 0);
+  }else{
+    pIter->flags |= FTS5_SEGITER_ONETERM;
+    sqlite3Fts5HashQuery(pHash, (const char*)pTerm, nTerm, &pList, &nList);
+    z = pTerm;
+    n = nTerm;
+  }
+
+  if( pList ){
+    Fts5Data *pLeaf;
+    sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z);
+    pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data));
+    if( pLeaf==0 ) return;
+    pLeaf->nRef = 1;
+    pLeaf->p = (u8*)pList;
+    pLeaf->n = nList;
+    pIter->pLeaf = pLeaf;
+    pIter->iLeafOffset = getVarint(pLeaf->p, (u64*)&pIter->iRowid);
+
+    if( flags & FTS5INDEX_QUERY_DESC ){
+      pIter->flags |= FTS5_SEGITER_REVERSE;
+      fts5SegIterReverseInitPage(p, pIter);
+    }
+  }
+}
+
 /*
 ** Zero the iterator passed as the only argument.
 */
@@ -2261,6 +2340,7 @@ static void fts5MultiIterNew(
   /* Allocate space for the new multi-seg-iterator. */
   if( iLevel<0 ){
     nSeg = fts5StructureCountSegments(pStruct);
+    nSeg += (p->apHash ? 1 : 0);
   }else{
     nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
   }
@@ -2280,6 +2360,11 @@ static void fts5MultiIterNew(
   /* Initialize each of the component segment iterators. */
   if( iLevel<0 ){
     Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
+    if( p->apHash ){
+      /* Add a segment iterator for the current contents of the hash table. */
+      Fts5SegIter *pIter = &pNew->aSeg[iIter++];
+      fts5SegIterHashInit(p, iIdx, pTerm, nTerm, flags, pIter);
+    }
     for(pLvl=&pStruct->aLevel[0]; pLvl<pEnd; pLvl++){
       for(iSeg=pLvl->nSeg-1; iSeg>=0; iSeg--){
         Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];
@@ -2406,13 +2491,19 @@ static void fts5ChunkIterInit(
   Fts5SegIter *pSeg,              /* Segment iterator to read poslist from */
   Fts5ChunkIter *pIter            /* Initialize this object */
 ){
-  int iId = pSeg->pSeg->iSegid;
-  i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno);
   Fts5Data *pLeaf = pSeg->pLeaf;
   int iOff = pSeg->iLeafOffset;
 
   memset(pIter, 0, sizeof(*pIter));
-  pIter->iLeafRowid = rowid;
+  /* If Fts5SegIter.pSeg is NULL, then this iterator iterates through data
+  ** currently stored in a hash table. In this case there is no leaf-rowid
+  ** to calculate.  */
+  if( pSeg->pSeg ){
+    int iId = pSeg->pSeg->iSegid;
+    i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno);
+    pIter->iLeafRowid = rowid;
+  }
+
   if( iOff<pLeaf->n ){
     fts5DataReference(pLeaf);
     pIter->pLeaf = pLeaf;
@@ -3100,6 +3191,7 @@ fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl);
 fflush(stdout);
 #endif
 
+  assert( iLvl>=0 );
   for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter);
       fts5MultiIterEof(p, pIter)==0;
       fts5MultiIterNext(p, pIter, 0, 0)

ext/fts5/test/fts5ab.test

@@ -247,14 +247,21 @@ do_execsql_test 6.0 {
     INSERT INTO s3 VALUES('A B C');
 }
 
-do_execsql_test 6.1 {
+do_execsql_test 6.1.1 {
   SELECT rowid FROM s3 WHERE s3 MATCH 'a'
+} {1 2}
+
+do_execsql_test 6.1.2 {
+  SELECT rowid FROM s3 WHERE s3 MATCH 'a' ORDER BY rowid DESC
 } {2 1}
 
 do_execsql_test 6.2 {
   COMMIT;
+}
+
+do_execsql_test 6.3 {
   SELECT rowid FROM s3 WHERE s3 MATCH 'a'
-} {2 1}
+} {1 2}
 
 finish_test
 

ext/fts5/test/fts5ac.test

@@ -22,11 +22,6 @@ ifcapable !fts5 {
   return
 }
 
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE xx USING fts5(x,y);
-  INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
-}
-
 set data {
     0   {p o q e z k z p n f y u z y n y}   {l o o l v v k}
     1   {p k h h p y l l h i p v n}         {p p l u r i f a j g e r r x w}
@@ -130,13 +125,6 @@ set data {
     99  {r c v w i v h a t a c v c r e}     {h h u m g o f b a e o}
 }
 
-do_test 1.1 {
-  foreach {id x y} $data {
-    execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
-  }
-  execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
-} {}
-
 # Usage:
 #
 #   poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2...
@@ -302,112 +290,94 @@ proc fts5_test_poslist {cmd} {
   set res
 }
 
-sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
 
-
+foreach {tn2 sql} {
-#-------------------------------------------------------------------------
+  1  {}
-# Test phrase queries.
+  2  {BEGIN}
-#
-foreach {tn phrase} {
-  1 "o"
-  2 "b q"
-  3 "e a e"
-  4 "m d g q q b k b w f q q p p"
-  5 "l o o l v v k"
-  6 "a"
-  7 "b"
-  8 "c"
-  9 "no"
-  10 "L O O L V V K"
 } {
-  set expr "\"$phrase\""
+  reset_db
-  set res [matchdata 1 $expr]
+  sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
 
-  do_execsql_test 1.2.$tn.[llength $res] { 
+  do_execsql_test 1.0 {
-    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    CREATE VIRTUAL TABLE xx USING fts5(x,y);
-  } $res
+    INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
-}
+  }
 
-#-------------------------------------------------------------------------
+  execsql $sql
-# Test some AND and OR queries.
-#
-foreach {tn expr} {
-  1.1 "a   AND b"
-  1.2 "a+b AND c"
-  1.3 "d+c AND u"
-  1.4 "d+c AND u+d"
 
-  2.1 "a   OR b"
+  do_test $tn2.1.1 {
-  2.2 "a+b OR c"
+    foreach {id x y} $data {
-  2.3 "d+c OR u"
+      execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
-  2.4 "d+c OR u+d"
+    }
+    execsql { INSERT INTO xx(xx) VALUES('integrity-check') }
+  } {}
 
-  3.1 { a AND b AND c }
+  #-------------------------------------------------------------------------
-} {
+  # Test phrase queries.
-  set res [matchdata 1 $expr]
+  #
-  do_execsql_test 2.$tn.[llength $res] { 
+  foreach {tn phrase} {
-    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    1 "o"
-  } $res
+    2 "b q"
-}
+    3 "e a e"
+    4 "m d g q q b k b w f q q p p"
+    5 "l o o l v v k"
+    6 "a"
+    7 "b"
+    8 "c"
+    9 "no"
+    10 "L O O L V V K"
+  } {
+    set expr "\"$phrase\""
+    set res [matchdata 1 $expr]
 
-#-------------------------------------------------------------------------
+    do_execsql_test $tn2.1.2.$tn.[llength $res] { 
-# Queries on a specific column.
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-#
+    } $res
-foreach {tn expr} {
+  }
-  1 "x:a"
-  2 "y:a"
-  3 "x:b"
-  4 "y:b"
-} {
-  set res [matchdata 1 $expr]
-  do_execsql_test 3.$tn.[llength $res] { 
-    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-  } $res
-}
 
-#-------------------------------------------------------------------------
+  #-------------------------------------------------------------------------
-# Some NEAR queries.
+  # Test some AND and OR queries.
-#
+  #
-foreach {tn expr} {
-  1 "NEAR(a b)"
-  2 "NEAR(r c)"
-  2 { NEAR(r c, 5) }
-  3 { NEAR(r c, 3) }
-  4 { NEAR(r c, 2) }
-  5 { NEAR(r c, 0) }
-  6 { NEAR(a b c) }
-  7 { NEAR(a b c, 8) }
-  8  { x : NEAR(r c) }
-  9  { y : NEAR(r c) }
-} {
-  set res [matchdata 1 $expr]
-  do_execsql_test 4.1.$tn.[llength $res] { 
-    SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
-  } $res
-}
-
-do_test 4.1  { poslist {{a b c}} -- a } {0.0.0}
-do_test 4.2  { poslist {{a b c}} -- c } {0.0.2}
-
-foreach {tn expr tclexpr} {
-  1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
-} {
-  do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr]
-}
-
-#-------------------------------------------------------------------------
-#
-do_execsql_test 6.integrity {
-  INSERT INTO xx(xx) VALUES('integrity-check');
-}
-#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
-foreach {bAsc sql} {
-  1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
-  0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
-} {
   foreach {tn expr} {
-    0.1 x
+    1.1 "a   AND b"
-    1 { NEAR(r c) }
+    1.2 "a+b AND c"
+    1.3 "d+c AND u"
+    1.4 "d+c AND u+d"
+
+    2.1 "a   OR b"
+    2.2 "a+b OR c"
+    2.3 "d+c OR u"
+    2.4 "d+c OR u+d"
+
+    3.1 { a AND b AND c }
+  } {
+    set res [matchdata 1 $expr]
+    do_execsql_test $tn2.2.$tn.[llength $res] { 
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    } $res
+  }
+
+  #-------------------------------------------------------------------------
+  # Queries on a specific column.
+  #
+  foreach {tn expr} {
+    1 "x:a"
+    2 "y:a"
+    3 "x:b"
+    4 "y:b"
+  } {
+    set res [matchdata 1 $expr]
+    do_execsql_test $tn2.3.$tn.[llength $res] { 
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    } $res
+  }
+
+  #-------------------------------------------------------------------------
+  # Some NEAR queries.
+  #
+  foreach {tn expr} {
+    1 "NEAR(a b)"
+    2 "NEAR(r c)"
     2 { NEAR(r c, 5) }
     3 { NEAR(r c, 3) }
     4 { NEAR(r c, 2) }
@@ -416,20 +386,60 @@ foreach {bAsc sql} {
     7 { NEAR(a b c, 8) }
     8  { x : NEAR(r c) }
     9  { y : NEAR(r c) }
-    10 { x : "r c" }
-    11 { y : "r c" }
-    12 { a AND b }
-    13 { a AND b AND c }
-    14a { a }
-    14b { a OR b }
-    15 { a OR b AND c }
-    16 { c AND b OR a }
-    17 { c AND (b OR a) }
-    18 { c NOT (b OR a) }
-    19 { c NOT b OR a AND d }
   } {
-    set res [matchdata 0 $expr $bAsc]
+    set res [matchdata 1 $expr]
-    do_execsql_test 6.$bAsc.$tn.[llength $res] $sql $res
+    do_execsql_test $tn2.4.1.$tn.[llength $res] { 
+      SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
+    } $res
+  }
+
+  do_test $tn2.4.1  { poslist {{a b c}} -- a } {0.0.0}
+  do_test $tn2.4.2  { poslist {{a b c}} -- c } {0.0.2}
+
+  foreach {tn expr tclexpr} {
+    1 {a b} {[N $x -- {a}] && [N $x -- {b}]}
+  } {
+    do_execsql_test $tn2.5.$tn {
+      SELECT fts5_expr_tcl($expr, 'N $x')
+    } [list $tclexpr]
+  }
+
+  #-------------------------------------------------------------------------
+  #
+  do_execsql_test $tn2.6.integrity {
+    INSERT INTO xx(xx) VALUES('integrity-check');
+  }
+  #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
+  foreach {bAsc sql} {
+    1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
+    0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
+  } {
+    foreach {tn expr} {
+      0.1 x
+      1 { NEAR(r c) }
+      2 { NEAR(r c, 5) }
+      3 { NEAR(r c, 3) }
+      4 { NEAR(r c, 2) }
+      5 { NEAR(r c, 0) }
+      6 { NEAR(a b c) }
+      7 { NEAR(a b c, 8) }
+      8  { x : NEAR(r c) }
+      9  { y : NEAR(r c) }
+      10 { x : "r c" }
+      11 { y : "r c" }
+      12 { a AND b }
+      13 { a AND b AND c }
+      14a { a }
+      14b { a OR b }
+      15 { a OR b AND c }
+      16 { c AND b OR a }
+      17 { c AND (b OR a) }
+      18 { c NOT (b OR a) }
+      19 { c NOT b OR a AND d }
+    } {
+      set res [matchdata 0 $expr $bAsc]
+      do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
+    }
   }
 }
 

ext/fts5/test/fts5ad.test

@@ -62,6 +62,18 @@ foreach {T create} {
     INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   }
 
+  4 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+    BEGIN;
+  }
+  
+  5 {
+    CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
+    INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+    BEGIN;
+  }
+
 } {
 
   do_test $T.1 { 
@@ -194,8 +206,8 @@ foreach {T create} {
   }
   
   foreach {bAsc sql} {
-    1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
     0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC}
+    1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
   } {
     foreach {tn prefix} {
       1  {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} 
@@ -211,9 +223,12 @@ foreach {T create} {
         set res [lsort -integer -increasing $res]
       }
       set n [llength $res]
+      if {$T==5} breakpoint 
       do_execsql_test $T.$bAsc.$tn.$n $sql $res
     }
   }
+
+  catchsql COMMIT
 }
 
 finish_test

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\swith\sfts5\sdoclist-indexes\sthat\soccured\sif\sthe\sfirst\srowid\sof\sthe\sfirst\snon-term\spage\sof\sa\sdoclist\sis\szero.
+C Fix\ssome\sproblems\swith\stransactions\sthat\sboth\sread\sand\swrite\san\sfts5\stable.
-D 2015-01-27T20:41:00.681
+D 2015-01-29T20:59:34.380
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e
 F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
 F ext/fts5/fts5.c f2e899fba27ca33c8897635752c4c83a40dcb18d
 F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a
-F ext/fts5/fts5Int.h e3b9344d8209c9639825c711662d5d039eb70322
+F ext/fts5/fts5Int.h f7cf9331f34c5a5a83a88f43148161daa4cc0233
 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f
-F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9
+F ext/fts5/fts5_buffer.c b92ba0eb67532d174934087f93716caf9a2168c7
 F ext/fts5/fts5_config.c e3421a76c2abd33a05ac09df0c97c64952d1e700
 F ext/fts5/fts5_expr.c 473e3428a9a637fa6e61d64d8ca3796ec57a58e9
-F ext/fts5/fts5_hash.c 4ab952b75f27d5ed3ef0f3b4f7fa1464744483e8
+F ext/fts5/fts5_hash.c b54822ca901fb76d79c6a09daecbc464e5fe02c1
-F ext/fts5/fts5_index.c ef6c7764a9f4968465936839c8f7e7423d8458c2
+F ext/fts5/fts5_index.c 1550befd9622d009520fdadfa0b42154e0ac54c0
 F ext/fts5/fts5_storage.c f7c12c9f454b2a525827b3d85fd222789236f548
 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5
 F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0
@@ -121,9 +121,9 @@ F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
 F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32
 F ext/fts5/test/fts5aa.test 065767c60ad301f77ad95f24369305e13347aa00
-F ext/fts5/test/fts5ab.test 127769288519ed549c57d7e11628dbe5b9952ad5
+F ext/fts5/test/fts5ab.test 5da2e92a8047860b9e22b6fd3990549639d631b1
-F ext/fts5/test/fts5ac.test cc39f7debda6f10ca2422e17163f9b6f078d5560
+F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e
-F ext/fts5/test/fts5ad.test 6c970531caf865b65f4e1dd9d6d43bd6ea37d754
+F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d
 F ext/fts5/test/fts5ae.test 347c96db06aab23ff00cf6a6b4064a8dbb182e42
 F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
 F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505
@@ -1283,7 +1283,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 5206ca6005bfa9dfc7346d4b89430c9748d32c10
+P f704bc059e06b01f1d68fa7dad89e33eace6c389
-R dc023966ceb63b949d8070662e553f89
+R f45496311c450f4a551203517eb9c071
 U dan
-Z 99344f3fa1c5e2c02514e48da6c76a56
+Z d89173c476e3f912e9f3a6ccba8c9b1b

manifest.uuid

@@ -1 +1 @@
-f704bc059e06b01f1d68fa7dad89e33eace6c389
+0e225b15357765f132c3364b222f9931a608a5b2