Merge latest trunk changes into this branch.

FossilOrigin-Name: edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746

ext/fts5/fts5Int.h

@@ -247,7 +247,8 @@ struct Fts5Config {
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
   int bSecureDelete;              /* 'secure-delete' */
-  int nDeleteMerge;           /* 'deletemerge' */
+  int nDeleteMerge;               /* 'deletemerge' */
+  int bPrefixInsttoken;           /* 'prefix-insttoken' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
@@ -504,7 +505,14 @@ int sqlite3Fts5StructureTest(Fts5Index*, void*);
 /*
 ** Used by xInstToken():
 */
-int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
+int sqlite3Fts5IterToken(
+  Fts5IndexIter *pIndexIter, 
+  const char *pToken, int nToken,
+  i64 iRowid,
+  int iCol, 
+  int iOff, 
+  const char **ppOut, int *pnOut
+);
 
 /*
 ** Insert or remove data to or from the index. Each time a document is 

ext/fts5/fts5_config.c

@@ -1026,6 +1026,19 @@ int sqlite3Fts5ConfigSetValue(
     }else{
       pConfig->bSecureDelete = (bVal ? 1 : 0);
     }
+  }
+
+  else if( 0==sqlite3_stricmp(zKey, "insttoken") ){
+    int bVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      bVal = sqlite3_value_int(pVal);
+    }
+    if( bVal<0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->bPrefixInsttoken = (bVal ? 1 : 0);
+    }
+
   }else{
     *pbBadkey = 1;
   }

ext/fts5/fts5_expr.c

@@ -3046,7 +3046,7 @@ static int fts5ExprPopulatePoslistsCb(
         int rc = sqlite3Fts5PoslistWriterAppend(
             &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         );
-        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
+        if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){
           int iCol = p->iOff>>32;
           int iTokOff = p->iOff & 0x7FFFFFFF;
           rc = sqlite3Fts5IndexIterWriteTokendata(
@@ -3239,15 +3239,14 @@ int sqlite3Fts5ExprInstToken(
     return SQLITE_RANGE;
   }
   pTerm = &pPhrase->aTerm[iToken];
-  if( pTerm->bPrefix==0 ){
-    if( pExpr->pConfig->bTokendata ){
-      rc = sqlite3Fts5IterToken(
-          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
-      );
-    }else{
-      *ppOut = pTerm->pTerm;
-      *pnOut = pTerm->nFullTerm;
-    }
+  if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){
+    rc = sqlite3Fts5IterToken(
+        pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, 
+        iRowid, iCol, iOff+iToken, ppOut, pnOut
+    );
+  }else{
+    *ppOut = pTerm->pTerm;
+    *pnOut = pTerm->nFullTerm;
   }
   return rc;
 }

ext/fts5/fts5_index.c

@@ -6203,6 +6203,383 @@ static void fts5MergePrefixLists(
   *p1 = out;
 }
 
+
+/*
+** Iterate through a range of entries in the FTS index, invoking the xVisit
+** callback for each of them.
+**
+** Parameter pToken points to an nToken buffer containing an FTS index term
+** (i.e. a document term with the preceding 1 byte index identifier -
+** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits
+** all entries for terms that have pToken/nToken as a prefix. If bPrefix
+** is false, then only entries with pToken/nToken as the entire key are
+** visited. 
+**
+** If the current table is a tokendata=1 table, then if bPrefix is true then
+** each index term is treated separately. However, if bPrefix is false, then
+** all index terms corresponding to pToken/nToken are collapsed into a single
+** term before the callback is invoked.
+**
+** The callback invoked for each entry visited is specified by paramter xVisit.
+** Each time it is invoked, it is passed a pointer to the Fts5Index object,
+** a copy of the 7th paramter to this function (pCtx) and a pointer to the
+** iterator that indicates the current entry. If the current entry is the
+** first with a new term (i.e. different from that of the previous entry,
+** including the very first term), then the final two parameters are passed
+** a pointer to the term and its size in bytes, respectively. If the current
+** entry is not the first associated with its term, these two parameters
+** are passed 0.
+**
+** If parameter pColset is not NULL, then it is used to filter entries before
+** the callback is invoked.
+*/
+static int fts5VisitEntries(
+  Fts5Index *p,                   /* Fts5 index object */
+  Fts5Colset *pColset,            /* Columns filter to apply, or NULL */
+  u8 *pToken,                     /* Buffer containing token */
+  int nToken,                     /* Size of buffer pToken in bytes */
+  int bPrefix,                    /* True for a prefix scan */
+  void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int),
+  void *pCtx                      /* Passed as second argument to xVisit() */
+){
+  const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0)
+                  | FTS5INDEX_QUERY_SKIPEMPTY 
+                  | FTS5INDEX_QUERY_NOOUTPUT;
+  Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
+  int bNewTerm = 1;
+  Fts5Structure *pStruct = fts5StructureRead(p);
+
+  fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
+  fts5IterSetOutputCb(&p->rc, p1);
+  for( /* no-op */ ;
+      fts5MultiIterEof(p, p1)==0;
+      fts5MultiIterNext2(p, p1, &bNewTerm)
+  ){
+    Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
+    int nNew = 0;
+    const u8 *pNew = 0;
+
+    p1->xSetOutputs(p1, pSeg);
+    if( p->rc ) break;
+
+    if( bNewTerm ){
+      nNew = pSeg->term.n;
+      pNew = pSeg->term.p;
+      if( nNew<nToken || memcmp(pToken, pNew, nToken) ) break;
+    }
+
+    xVisit(p, pCtx, p1, pNew, nNew);
+  }
+  fts5MultiIterFree(p1);
+
+  fts5StructureRelease(pStruct);
+  return p->rc;
+}
+
+
+/*
+** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
+** array of these for each row it visits (so all iRowid fields are the same).
+** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an
+** array of these for the entire query (in which case iRowid fields may take
+** a variety of values).
+**
+** Each instance in the array indicates the iterator (and therefore term)
+** associated with position iPos of rowid iRowid. This is used by the
+** xInstToken() API.
+**
+** iRowid:
+**   Rowid for the current entry.
+**
+** iPos:
+**   Position of current entry within row. In the usual ((iCol<<32)+iOff)
+**   format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()).
+**
+** iIter:
+**   If the Fts5TokenDataIter iterator that the entry is part of is
+**   actually an iterator (i.e. with nIter>0, not just a container for
+**   Fts5TokenDataMap structures), then this variable is an index into
+**   the apIter[] array. The corresponding term is that which the iterator
+**   at apIter[iIter] currently points to.
+**
+**   Or, if the Fts5TokenDataIter iterator is just a container object
+**   (nIter==0), then iIter is an index into the term.p[] buffer where
+**   the term is stored.
+**
+** nByte:
+**   In the case where iIter is an index into term.p[], this variable
+**   is the size of the term in bytes. If iIter is an index into apIter[],
+**   this variable is unused.
+*/
+struct Fts5TokenDataMap {
+  i64 iRowid;                     /* Row this token is located in */
+  i64 iPos;                       /* Position of token */
+  int iIter;                      /* Iterator token was read from */
+  int nByte;                      /* Length of token in bytes (or 0) */
+};
+
+/*
+** An object used to supplement Fts5Iter for tokendata=1 iterators.
+**
+** This object serves two purposes. The first is as a container for an array
+** of Fts5TokenDataMap structures, which are used to find the token required
+** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and
+** aMap[] variables.
+*/
+struct Fts5TokenDataIter {
+  int nMapAlloc;                  /* Allocated size of aMap[] in entries */
+  int nMap;                       /* Number of valid entries in aMap[] */
+  Fts5TokenDataMap *aMap;         /* Array of (rowid+pos -> token) mappings */
+
+  /* The following are used for prefix-queries only. */
+  Fts5Buffer terms;
+
+  /* The following are used for other full-token tokendata queries only. */
+  int nIter;
+  int nIterAlloc;
+  Fts5PoslistReader *aPoslistReader;
+  int *aPoslistToIter;
+  Fts5Iter *apIter[1];
+};
+
+/*
+** The two input arrays - a1[] and a2[] - are in sorted order. This function
+** merges the two arrays together and writes the result to output array 
+** aOut[]. aOut[] is guaranteed to be large enough to hold the result.
+**
+** Duplicate entries are copied into the output. So the size of the output
+** array is always (n1+n2) entries.
+*/
+static void fts5TokendataMerge(
+  Fts5TokenDataMap *a1, int n1,   /* Input array 1 */
+  Fts5TokenDataMap *a2, int n2,   /* Input array 2 */
+  Fts5TokenDataMap *aOut          /* Output array */
+){
+  int i1 = 0;
+  int i2 = 0;
+
+  assert( n1>=0 && n2>=0 );
+  while( i1<n1 || i2<n2 ){
+    Fts5TokenDataMap *pOut = &aOut[i1+i2];
+    if( i2>=n2 || (i1<n1 && (
+        a1[i1].iRowid<a2[i2].iRowid
+     || (a1[i1].iRowid==a2[i2].iRowid && a1[i1].iPos<=a2[i2].iPos)
+    ))){
+      memcpy(pOut, &a1[i1], sizeof(Fts5TokenDataMap));
+      i1++;
+    }else{
+      memcpy(pOut, &a2[i2], sizeof(Fts5TokenDataMap));
+      i2++;
+    }
+  }
+}
+
+
+/*
+** Append a mapping to the token-map belonging to object pT.
+*/
+static void fts5TokendataIterAppendMap(
+  Fts5Index *p, 
+  Fts5TokenDataIter *pT, 
+  int iIter,
+  int nByte,
+  i64 iRowid, 
+  i64 iPos
+){
+  if( p->rc==SQLITE_OK ){
+    if( pT->nMap==pT->nMapAlloc ){
+      int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
+      int nByte = nNew * sizeof(Fts5TokenDataMap);
+      Fts5TokenDataMap *aNew;
+
+      aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte);
+      if( aNew==0 ){
+        p->rc = SQLITE_NOMEM;
+        return;
+      }
+
+      pT->aMap = aNew;
+      pT->nMapAlloc = nNew;
+    }
+
+    pT->aMap[pT->nMap].iRowid = iRowid;
+    pT->aMap[pT->nMap].iPos = iPos;
+    pT->aMap[pT->nMap].iIter = iIter;
+    pT->aMap[pT->nMap].nByte = nByte;
+    pT->nMap++;
+  }
+}
+
+/*
+** Sort the contents of the pT->aMap[] array.
+**
+** The sorting algorithm requries a malloc(). If this fails, an error code
+** is left in Fts5Index.rc before returning.
+*/
+static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){
+  Fts5TokenDataMap *aTmp = 0;
+  int nByte = pT->nMap * sizeof(Fts5TokenDataMap);
+
+  aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte);
+  if( aTmp ){
+    Fts5TokenDataMap *a1 = pT->aMap;
+    Fts5TokenDataMap *a2 = aTmp;
+    i64 nHalf;
+
+    for(nHalf=1; nHalf<pT->nMap; nHalf=nHalf*2){
+      int i1;
+      for(i1=0; i1<pT->nMap; i1+=(nHalf*2)){
+        int n1 = MIN(nHalf, pT->nMap-i1);
+        int n2 = MIN(nHalf, pT->nMap-i1-n1);
+        fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]);
+      }
+      SWAPVAL(Fts5TokenDataMap*, a1, a2);
+    }
+
+    if( a1!=pT->aMap ){
+      memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap));
+    }
+    sqlite3_free(aTmp);
+
+#ifdef SQLITE_DEBUG
+    {
+      int ii;
+      for(ii=1; ii<pT->nMap; ii++){
+        Fts5TokenDataMap *p1 = &pT->aMap[ii-1];
+        Fts5TokenDataMap *p2 = &pT->aMap[ii];
+        assert( p1->iRowid<p2->iRowid 
+             || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos)
+        );
+      }
+    }
+#endif
+  }
+}
+
+/*
+** Delete an Fts5TokenDataIter structure and its contents.
+*/
+static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
+  if( pSet ){
+    int ii;
+    for(ii=0; ii<pSet->nIter; ii++){
+      fts5MultiIterFree(pSet->apIter[ii]);
+    }
+    fts5BufferFree(&pSet->terms);
+    sqlite3_free(pSet->aPoslistReader);
+    sqlite3_free(pSet->aMap);
+    sqlite3_free(pSet);
+  }
+}
+
+
+/*
+** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata()
+** to pass data to prefixIterSetupTokendataCb().
+*/
+typedef struct TokendataSetupCtx TokendataSetupCtx;
+struct TokendataSetupCtx {
+  Fts5TokenDataIter *pT;          /* Object being populated with mappings */
+  int iTermOff;                   /* Offset of current term in terms.p[] */
+  int nTermByte;                  /* Size of current term in bytes */
+};
+
+/*
+** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This
+** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each
+** position in the current position-list. It doesn't matter that some of
+** these may be out of order - they will be sorted later.
+*/
+static void prefixIterSetupTokendataCb(
+  Fts5Index *p, 
+  void *pCtx, 
+  Fts5Iter *p1, 
+  const u8 *pNew,
+  int nNew
+){
+  TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx;
+  int iPosOff = 0;
+  i64 iPos = 0;
+
+  if( pNew ){
+    pSetup->nTermByte = nNew-1;
+    pSetup->iTermOff = pSetup->pT->terms.n;
+    fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1);
+  }
+
+  while( 0==sqlite3Fts5PoslistNext64(
+     p1->base.pData, p1->base.nData, &iPosOff, &iPos
+  ) ){
+    fts5TokendataIterAppendMap(p, 
+        pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos
+    );
+  }
+}
+
+
+/*
+** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries().
+*/
+typedef struct PrefixSetupCtx PrefixSetupCtx;
+struct PrefixSetupCtx {
+  void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
+  void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
+  i64 iLastRowid;
+  int nMerge;
+  Fts5Buffer *aBuf;
+  int nBuf;
+  Fts5Buffer doclist;
+  TokendataSetupCtx *pTokendata;
+};
+
+/*
+** fts5VisitEntries() callback used by fts5SetupPrefixIter()
+*/
+static void prefixIterSetupCb(
+  Fts5Index *p, 
+  void *pCtx, 
+  Fts5Iter *p1, 
+  const u8 *pNew,
+  int nNew
+){
+  PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx;
+  const int nMerge = pSetup->nMerge;
+
+  if( p1->base.nData>0 ){
+    if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){
+      int i;
+      for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){
+        int i1 = i*nMerge;
+        int iStore;
+        assert( i1+nMerge<=pSetup->nBuf );
+        for(iStore=i1; iStore<i1+nMerge; iStore++){
+          if( pSetup->aBuf[iStore].n==0 ){
+            fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]);
+            fts5BufferZero(&pSetup->doclist);
+            break;
+          }
+        }
+        if( iStore==i1+nMerge ){
+          pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]);
+          for(iStore=i1; iStore<i1+nMerge; iStore++){
+            fts5BufferZero(&pSetup->aBuf[iStore]);
+          }
+        }
+      }
+      pSetup->iLastRowid = 0;
+    }
+
+    pSetup->xAppend(
+        p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist
+    );
+    pSetup->iLastRowid = p1->base.iRowid;
+  }
+
+  if( pSetup->pTokendata ){
+    prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew);
+  }
+}
+
 static void fts5SetupPrefixIter(
   Fts5Index *p,                   /* Index to read from */
   int bDesc,                      /* True for "ORDER BY rowid DESC" */
@@ -6213,38 +6590,41 @@ static void fts5SetupPrefixIter(
   Fts5Iter **ppIter               /* OUT: New iterator */
 ){
   Fts5Structure *pStruct;
-  Fts5Buffer *aBuf;
-  int nBuf = 32;
-  int nMerge = 1;
+  PrefixSetupCtx s;
+  TokendataSetupCtx s2;
 
-  void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*);
-  void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*);
-  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
-    xMerge = fts5MergeRowidLists;
-    xAppend = fts5AppendRowid;
-  }else{
-    nMerge = FTS5_MERGE_NLIST-1;
-    nBuf = nMerge*8;   /* Sufficient to merge (16^8)==(2^32) lists */
-    xMerge = fts5MergePrefixLists;
-    xAppend = fts5AppendPoslist;
+  memset(&s, 0, sizeof(s));
+  memset(&s2, 0, sizeof(s2));
+
+  s.nMerge = 1;
+  s.iLastRowid = 0;
+  s.nBuf = 32;
+  if( iIdx==0 
+   && p->pConfig->eDetail==FTS5_DETAIL_FULL 
+   && p->pConfig->bPrefixInsttoken 
+  ){
+    s.pTokendata = &s2;
+    s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT));
   }
 
-  aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
+  if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){
+    s.xMerge = fts5MergeRowidLists;
+    s.xAppend = fts5AppendRowid;
+  }else{
+    s.nMerge = FTS5_MERGE_NLIST-1;
+    s.nBuf = s.nMerge*8;   /* Sufficient to merge (16^8)==(2^32) lists */
+    s.xMerge = fts5MergePrefixLists;
+    s.xAppend = fts5AppendPoslist;
+  }
+
+  s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf);
   pStruct = fts5StructureRead(p);
-  assert( p->rc!=SQLITE_OK || (aBuf && pStruct) );
+  assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) );
 
   if( p->rc==SQLITE_OK ){
-    const int flags = FTS5INDEX_QUERY_SCAN 
-                    | FTS5INDEX_QUERY_SKIPEMPTY 
-                    | FTS5INDEX_QUERY_NOOUTPUT;
+    void *pCtx = (void*)&s;
     int i;
-    i64 iLastRowid = 0;
-    Fts5Iter *p1 = 0;     /* Iterator used to gather data from index */
     Fts5Data *pData;
-    Fts5Buffer doclist;
-    int bNewTerm = 1;
-
-    memset(&doclist, 0, sizeof(doclist));
 
     /* If iIdx is non-zero, then it is the number of a prefix-index for
     ** prefixes 1 character longer than the prefix being queried for. That
@@ -6252,94 +6632,43 @@ static void fts5SetupPrefixIter(
     ** corresponding to the prefix itself. That one is extracted from the
     ** main term index here.  */
     if( iIdx!=0 ){
-      int dummy = 0;
-      const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT;
       pToken[0] = FTS5_MAIN_PREFIX;
-      fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1);
-      fts5IterSetOutputCb(&p->rc, p1);
-      for(;
-        fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext2(p, p1, &dummy)
-      ){
-        Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
-        p1->xSetOutputs(p1, pSeg);
-        if( p1->base.nData ){
-          xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
-          iLastRowid = p1->base.iRowid;
-        }
-      }
-      fts5MultiIterFree(p1);
+      fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx);
     }
 
     pToken[0] = FTS5_MAIN_PREFIX + iIdx;
-    fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1);
-    fts5IterSetOutputCb(&p->rc, p1);
+    fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx);
 
-    for( /* no-op */ ;
-        fts5MultiIterEof(p, p1)==0;
-        fts5MultiIterNext2(p, p1, &bNewTerm)
-    ){
-      Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ];
-      int nTerm = pSeg->term.n;
-      const u8 *pTerm = pSeg->term.p;
-      p1->xSetOutputs(p1, pSeg);
-
-      assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
-      if( bNewTerm ){
-        if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
-      }
-
-      if( p1->base.nData==0 ) continue;
-      if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){
-        for(i=0; p->rc==SQLITE_OK && doclist.n; i++){
-          int i1 = i*nMerge;
-          int iStore;
-          assert( i1+nMerge<=nBuf );
-          for(iStore=i1; iStore<i1+nMerge; iStore++){
-            if( aBuf[iStore].n==0 ){
-              fts5BufferSwap(&doclist, &aBuf[iStore]);
-              fts5BufferZero(&doclist);
-              break;
-            }
-          }
-          if( iStore==i1+nMerge ){
-            xMerge(p, &doclist, nMerge, &aBuf[i1]);
-            for(iStore=i1; iStore<i1+nMerge; iStore++){
-              fts5BufferZero(&aBuf[iStore]);
-            }
-          }
-        }
-        iLastRowid = 0;
-      }
-
-      xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist);
-      iLastRowid = p1->base.iRowid;
-    }
-
-    assert( (nBuf%nMerge)==0 );
-    for(i=0; i<nBuf; i+=nMerge){
+    assert( (s.nBuf%s.nMerge)==0 );
+    for(i=0; i<s.nBuf; i+=s.nMerge){
       int iFree;
       if( p->rc==SQLITE_OK ){
-        xMerge(p, &doclist, nMerge, &aBuf[i]);
+        s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]);
       }
-      for(iFree=i; iFree<i+nMerge; iFree++){
-        fts5BufferFree(&aBuf[iFree]);
+      for(iFree=i; iFree<i+s.nMerge; iFree++){
+        fts5BufferFree(&s.aBuf[iFree]);
       }
     }
-    fts5MultiIterFree(p1);
 
-    pData = fts5IdxMalloc(p, sizeof(*pData)+doclist.n+FTS5_DATA_ZERO_PADDING);
+    pData = fts5IdxMalloc(p, sizeof(*pData)+s.doclist.n+FTS5_DATA_ZERO_PADDING);
     if( pData ){
       pData->p = (u8*)&pData[1];
-      pData->nn = pData->szLeaf = doclist.n;
-      if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n);
+      pData->nn = pData->szLeaf = s.doclist.n;
+      if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n);
       fts5MultiIterNew2(p, pData, bDesc, ppIter);
     }
-    fts5BufferFree(&doclist);
+
+    if( p->rc==SQLITE_OK && s.pTokendata ){
+      fts5TokendataIterSortMap(p, s2.pT);
+      (*ppIter)->pTokenDataIter = s2.pT;
+      s2.pT = 0;
+    }
   }
 
+  fts5TokendataIterDelete(s2.pT);
+  fts5BufferFree(&s.doclist);
   fts5StructureRelease(pStruct);
-  sqlite3_free(aBuf);
+  sqlite3_free(s.aBuf);
 }
 
 
@@ -6593,38 +6922,6 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){
   pSeg->pLeaf = 0;
 }
 
-/*
-** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an
-** array of these for each row it visits. Or, for an iterator used by an
-** "ORDER BY rank" query, it accumulates an array of these for the entire
-** query.
-**
-** Each instance in the array indicates the iterator (and therefore term)
-** associated with position iPos of rowid iRowid. This is used by the
-** xInstToken() API.
-*/
-struct Fts5TokenDataMap {
-  i64 iRowid;                     /* Row this token is located in */
-  i64 iPos;                       /* Position of token */
-  int iIter;                      /* Iterator token was read from */
-};
-
-/*
-** An object used to supplement Fts5Iter for tokendata=1 iterators.
-*/
-struct Fts5TokenDataIter {
-  int nIter;
-  int nIterAlloc;
-
-  int nMap;
-  int nMapAlloc;
-  Fts5TokenDataMap *aMap;
-
-  Fts5PoslistReader *aPoslistReader;
-  int *aPoslistToIter;
-  Fts5Iter *apIter[1];
-};
-
 /*
 ** This function appends iterator pAppend to Fts5TokenDataIter pIn and 
 ** returns the result.
@@ -6661,54 +6958,6 @@ static Fts5TokenDataIter *fts5AppendTokendataIter(
   return pRet;
 }
 
-/*
-** Delete an Fts5TokenDataIter structure and its contents.
-*/
-static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){
-  if( pSet ){
-    int ii;
-    for(ii=0; ii<pSet->nIter; ii++){
-      fts5MultiIterFree(pSet->apIter[ii]);
-    }
-    sqlite3_free(pSet->aPoslistReader);
-    sqlite3_free(pSet->aMap);
-    sqlite3_free(pSet);
-  }
-}
-
-/*
-** Append a mapping to the token-map belonging to object pT.
-*/
-static void fts5TokendataIterAppendMap(
-  Fts5Index *p, 
-  Fts5TokenDataIter *pT, 
-  int iIter,
-  i64 iRowid, 
-  i64 iPos
-){
-  if( p->rc==SQLITE_OK ){
-    if( pT->nMap==pT->nMapAlloc ){
-      int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64;
-      int nByte = nNew * sizeof(Fts5TokenDataMap);
-      Fts5TokenDataMap *aNew;
-
-      aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte);
-      if( aNew==0 ){
-        p->rc = SQLITE_NOMEM;
-        return;
-      }
-
-      pT->aMap = aNew;
-      pT->nMapAlloc = nNew;
-    }
-
-    pT->aMap[pT->nMap].iRowid = iRowid;
-    pT->aMap[pT->nMap].iPos = iPos;
-    pT->aMap[pT->nMap].iIter = iIter;
-    pT->nMap++;
-  }
-}
-
 /*
 ** The iterator passed as the only argument must be a tokendata=1 iterator
 ** (pIter->pTokenDataIter!=0). This function sets the iterator output
@@ -6749,7 +6998,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){
     pIter->base.iRowid = iRowid;
 
     if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){
-      fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1);
+      fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1);
     }else
     if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){
       int nReader = 0;
@@ -7002,6 +7251,7 @@ static Fts5Iter *fts5SetupTokendataIter(
     pRet = fts5MultiIterAlloc(p, 0);
   }
   if( pRet ){
+    pRet->nSeg = 0;
     pRet->pTokenDataIter = pSet;
     if( pSet ){
       fts5IterSetOutputsTokendata(pRet);
@@ -7017,7 +7267,6 @@ static Fts5Iter *fts5SetupTokendataIter(
   return pRet;
 }
 
-
 /*
 ** Open a new iterator to iterate though all rowid that match the 
 ** specified token or token prefix.
@@ -7042,6 +7291,11 @@ int sqlite3Fts5IndexQuery(
     int bTokendata = pConfig->bTokendata;
     if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken);
 
+    /* The NOTOKENDATA flag is set when each token in a tokendata=1 table
+    ** should be treated individually, instead of merging all those with
+    ** a common prefix into a single entry. This is used, for example, by
+    ** queries performed as part of an integrity-check, or by the fts5vocab
+    ** module.  */
     if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){
       bTokendata = 0;
     }
@@ -7072,7 +7326,7 @@ int sqlite3Fts5IndexQuery(
     }
 
     if( bTokendata && iIdx==0 ){
-      buf.p[0] = '0';
+      buf.p[0] = FTS5_MAIN_PREFIX;
       pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset);
     }else if( iIdx<=pConfig->nPrefix ){
       /* Straight index lookup */
@@ -7085,7 +7339,7 @@ int sqlite3Fts5IndexQuery(
         fts5StructureRelease(pStruct);
       }
     }else{
-      /* Scan multiple terms in the main index */
+      /* Scan multiple terms in the main index for a prefix query. */
       int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0;
       fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet);
       if( pRet==0 ){
@@ -7121,7 +7375,8 @@ int sqlite3Fts5IndexQuery(
 int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   assert( pIter->pIndex->rc==SQLITE_OK );
-  if( pIter->pTokenDataIter ){
+  if( pIter->nSeg==0 ){
+    assert( pIter->pTokenDataIter );
     fts5TokendataIterNext(pIter, 0, 0);
   }else{
     fts5MultiIterNext(pIter->pIndex, pIter, 0, 0);
@@ -7158,7 +7413,8 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){
 */
 int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter->pTokenDataIter ){
+  if( pIter->nSeg==0 ){
+    assert( pIter->pTokenDataIter );
     fts5TokendataIterNext(pIter, 1, iMatch);
   }else{
     fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch);
@@ -7177,14 +7433,60 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){
   return (z ? &z[1] : 0);
 }
 
+/*
+** pIter is a prefix query. This function populates pIter->pTokenDataIter
+** with an Fts5TokenDataIter object containing mappings for all rows
+** matched by the query.
+*/
+static int fts5SetupPrefixIterTokendata(
+  Fts5Iter *pIter,
+  const char *pToken,             /* Token prefix to search for */
+  int nToken                      /* Size of pToken in bytes */
+){
+  Fts5Index *p = pIter->pIndex;
+  Fts5Buffer token = {0, 0, 0};
+  TokendataSetupCtx ctx;
+
+  memset(&ctx, 0, sizeof(ctx));
+
+  fts5BufferGrow(&p->rc, &token, nToken+1);
+  ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT));
+
+  if( p->rc==SQLITE_OK ){
+
+    /* Fill in the token prefix to search for */
+    token.p[0] = FTS5_MAIN_PREFIX;
+    memcpy(&token.p[1], pToken, nToken);
+    token.n = nToken+1;
+
+    fts5VisitEntries(
+        p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx
+    );
+
+    fts5TokendataIterSortMap(p, ctx.pT);
+  }
+
+  if( p->rc==SQLITE_OK ){
+    pIter->pTokenDataIter = ctx.pT;
+  }else{
+    fts5TokendataIterDelete(ctx.pT);
+  }
+  fts5BufferFree(&token);
+
+  return fts5IndexReturn(p);
+}
+
 /*
 ** This is used by xInstToken() to access the token at offset iOff, column
 ** iCol of row iRowid. The token is returned via output variables *ppOut
 ** and *pnOut. The iterator passed as the first argument must be a tokendata=1
 ** iterator (pIter->pTokenDataIter!=0).
+**
+** pToken/nToken:
 */
 int sqlite3Fts5IterToken(
   Fts5IndexIter *pIndexIter, 
+  const char *pToken, int nToken,
   i64 iRowid,
   int iCol, 
   int iOff, 
@@ -7192,13 +7494,22 @@ int sqlite3Fts5IterToken(
 ){
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   Fts5TokenDataIter *pT = pIter->pTokenDataIter;
-  Fts5TokenDataMap *aMap = pT->aMap;
   i64 iPos = (((i64)iCol)<<32) + iOff;
-
+  Fts5TokenDataMap *aMap = 0;
   int i1 = 0;
-  int i2 = pT->nMap;
+  int i2 = 0;
   int iTest = 0;
 
+  assert( pT || (pToken && pIter->nSeg>0) );
+  if( pT==0 ){
+    int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken);
+    if( rc!=SQLITE_OK ) return rc;
+    pT = pIter->pTokenDataIter;
+  }
+
+  i2 = pT->nMap;
+  aMap = pT->aMap;
+
   while( i2>i1 ){
     iTest = (i1 + i2) / 2;
 
@@ -7221,9 +7532,15 @@ int sqlite3Fts5IterToken(
   }
 
   if( i2>i1 ){
-    Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
-    *ppOut = (const char*)pMap->aSeg[0].term.p+1;
-    *pnOut = pMap->aSeg[0].term.n-1;
+    if( pIter->nSeg==0 ){
+      Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter];
+      *ppOut = (const char*)pMap->aSeg[0].term.p+1;
+      *pnOut = pMap->aSeg[0].term.n-1;
+    }else{
+      Fts5TokenDataMap *p = &aMap[iTest];
+      *ppOut = (const char*)&pT->terms.p[p->iIter];
+      *pnOut = aMap[iTest].nByte;
+    }
   }
 
   return SQLITE_OK;
@@ -7235,7 +7552,9 @@ int sqlite3Fts5IterToken(
 */
 void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
-  if( pIter && pIter->pTokenDataIter ){
+  if( pIter && pIter->pTokenDataIter 
+   && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL)
+  ){
     pIter->pTokenDataIter->nMap = 0;
   }
 }
@@ -7255,17 +7574,30 @@ int sqlite3Fts5IndexIterWriteTokendata(
   Fts5Iter *pIter = (Fts5Iter*)pIndexIter;
   Fts5TokenDataIter *pT = pIter->pTokenDataIter;
   Fts5Index *p = pIter->pIndex;
-  int ii;
+  i64 iPos = (((i64)iCol)<<32) + iOff;
 
   assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL );
-  assert( pIter->pTokenDataIter );
-
-  for(ii=0; ii<pT->nIter; ii++){
-    Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
-    if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
-  }
-  if( ii<pT->nIter ){
-    fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff);
+  assert( pIter->pTokenDataIter || pIter->nSeg>0 );
+  if( pIter->nSeg>0 ){
+    /* This is a prefix term iterator. */
+    Fts5TokenDataIter *pT = pIter->pTokenDataIter;
+    if( pT==0 ){
+      pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT));
+      pIter->pTokenDataIter = pT;
+    }
+    if( pT ){
+      fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos);
+      fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken);
+    }
+  }else{
+    int ii;
+    for(ii=0; ii<pT->nIter; ii++){
+      Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term;
+      if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break;
+    }
+    if( ii<pT->nIter ){
+      fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos);
+    }
   }
   return fts5IndexReturn(p);
 }

ext/fts5/fts5_main.c

@@ -93,6 +93,7 @@ struct Fts5Global {
 #define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr ))
 #define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr))
 
+#define FTS5_INSTTOKEN_SUBTYPE 73
 
 /*
 ** Each auxiliary function registered with the FTS5 module is represented
@@ -1417,6 +1418,7 @@ static int fts5FilterMethod(
   sqlite3_value *pRowidGe = 0;    /* rowid >= ? expression (or NULL) */
   int iCol;                       /* Column on LHS of MATCH operator */
   char **pzErrmsg = pConfig->pzErrmsg;
+  int bPrefixInsttoken = pConfig->bPrefixInsttoken;
   int i;
   int iIdxStr = 0;
   Fts5Expr *pExpr = 0;
@@ -1452,6 +1454,9 @@ static int fts5FilterMethod(
         rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset);
         if( rc!=SQLITE_OK ) goto filter_out;
         if( zText==0 ) zText = "";
+        if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){
+          pConfig->bPrefixInsttoken = 1;
+        }
 
         iCol = 0;
         do{
@@ -1592,6 +1597,7 @@ static int fts5FilterMethod(
  filter_out:
   sqlite3Fts5ExprFree(pExpr);
   pConfig->pzErrmsg = pzErrmsg;
+  pConfig->bPrefixInsttoken = bPrefixInsttoken;
   return rc;
 }
 
@@ -3651,6 +3657,19 @@ static void fts5LocaleFunc(
   }
 }
 
+/*
+** Implementation of fts5_insttoken() function.
+*/
+static void fts5InsttokenFunc(
+  sqlite3_context *pCtx,          /* Function call context */
+  int nArg,                       /* Number of args */
+  sqlite3_value **apArg           /* Function arguments */
+){
+  assert( nArg==1 );
+  sqlite3_result_value(pCtx, apArg[0]);
+  sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE);
+}
+
 /*
 ** Return true if zName is the extension on one of the shadow tables used
 ** by this module.
@@ -3780,10 +3799,17 @@ static int fts5Init(sqlite3 *db){
     if( rc==SQLITE_OK ){
       rc = sqlite3_create_function(
           db, "fts5_locale", 2, 
-          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE,
+          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE,
           p, fts5LocaleFunc, 0, 0
       );
     }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(
+          db, "fts5_insttoken", 1, 
+          SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE,
+          p, fts5InsttokenFunc, 0, 0
+      );
+    }
   }
 
   /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file

ext/fts5/test/fts5origintext.test

@@ -22,34 +22,40 @@ ifcapable !fts5 {
 }
 
 foreach_detail_mode $testprefix {
+foreach {tn insttoken} {
+  1 0
+  2 1
+} {
+reset_db
 
 sqlite3_fts5_register_origintext db
-do_execsql_test 1.0 {
+do_execsql_test $tn.1.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
       x, tokenize="origintext unicode61", detail=%DETAIL%
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }
 
-do_execsql_test 1.1 {
+do_execsql_test $tn.1.1 {
   INSERT INTO ft VALUES('Hello world');
 }
 
-do_execsql_test 1.2 {
+do_execsql_test $tn.1.2 {
   INSERT INTO ft(ft) VALUES('integrity-check');
 }
 
 proc b {x} { string map [list "\0" "."] $x }
 db func b b
 
-do_execsql_test 1.3 {
+do_execsql_test $tn.1.3 {
   select b(term) from vocab;
 } {
   hello.Hello
   world
 }
 
-do_execsql_test 1.4 {
+do_execsql_test $tn.1.4 {
   SELECT rowid FROM ft('Hello');
 } {1}
 
@@ -88,33 +94,34 @@ proc document {} {
 db func document document
 
 sqlite3_fts5_register_origintext db
-do_execsql_test 2.0 {
+do_execsql_test $tn.2.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
       x, tokenize="origintext unicode61", detail=%DETAIL%
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
   INSERT INTO ft(ft, rank) VALUES('pgsz', 128);
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 }
 
-do_test 2.1 {
+do_test $tn.2.1 {
   for {set ii 0} {$ii < 500} {incr ii} {
     execsql { INSERT INTO ft VALUES( document() ) }
   }
 } {}
 
-do_execsql_test 2.2 {
+do_execsql_test $tn.2.2 {
   INSERT INTO ft(ft) VALUES('integrity-check');
 }
 
-do_execsql_test 2.3 {
+do_execsql_test $tn.2.3 {
   INSERT INTO ft(ft, rank) VALUES('merge', 16);
 }
 
-do_execsql_test 2.4 {
+do_execsql_test $tn.2.4 {
   INSERT INTO ft(ft) VALUES('integrity-check');
 }
 
-do_execsql_test 2.5 {
+do_execsql_test $tn.2.5 {
   INSERT INTO ft(ft) VALUES('optimize');
 }
 
@@ -122,10 +129,11 @@ do_execsql_test 2.5 {
 reset_db
 
 sqlite3_fts5_register_origintext db
-do_execsql_test 3.0 {
+do_execsql_test $tn.3.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
       x, tokenize="origintext unicode61", detail=%DETAIL%
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
   CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
 
   INSERT INTO ft(rowid, x) VALUES(1, 'hello');
@@ -137,16 +145,17 @@ do_execsql_test 3.0 {
 #db func b b
 #execsql_pp { SELECT b(term) FROM vocab }
 
-do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1
-do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2
-do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3
+do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1
+do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2
+do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3
 
-do_execsql_test 3.2 {
+do_execsql_test $tn.3.2 {
   CREATE VIRTUAL TABLE ft2 USING fts5(x, 
       tokenize="origintext unicode61", 
       tokendata=1,
       detail=%DETAIL%
   );
+  INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken);
   CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance);
 
   INSERT INTO ft2(rowid, x) VALUES(1, 'hello');
@@ -160,11 +169,18 @@ do_execsql_test 3.2 {
 #db func b b
 #execsql_pp { SELECT b(term) FROM vocab }
 
-do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
-do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
-do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
+do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3}
+do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3}
+do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3}
 
-do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10}
+
+do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} {
+  3 2 1
+}
+do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} {
+  3 2 1
+}
 
 #-------------------------------------------------------------------------
 #
@@ -176,36 +192,37 @@ proc querytoken {cmd iPhrase iToken} {
 }
 sqlite3_fts5_create_function db querytoken querytoken
 
-do_execsql_test 4.0 {
+do_execsql_test $tn.4.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
       x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
   INSERT INTO ft VALUES('one two three four');
 }
 
-do_execsql_test 4.1 {
+do_execsql_test $tn.4.1 {
   SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO')
 } {1 two.TwO}
-do_execsql_test 4.2 {
+do_execsql_test $tn.4.2 {
   SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE')
 } {1 one}
-do_execsql_test 4.3 {
+do_execsql_test $tn.4.3 {
   SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE')
 } {1 two.TWO}
 
 if {"%DETAIL%"=="full"} {
   # Phrase queries are only supported for detail=full.
   #
-  do_execsql_test 4.4 {
+  do_execsql_test $tn.4.4 {
     SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"')
   } {1 three.ThreE}
-  do_catchsql_test 4.5 {
+  do_catchsql_test $tn.4.5 {
     SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"')
   } {1 SQLITE_RANGE}
-  do_catchsql_test 4.6 {
+  do_catchsql_test $tn.4.6 {
     SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"')
   } {1 SQLITE_RANGE}
-  do_catchsql_test 4.7 {
+  do_catchsql_test $tn.4.7 {
     SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"')
   } {1 SQLITE_RANGE}
 }
@@ -221,14 +238,15 @@ proc insttoken {cmd iIdx iToken} {
 sqlite3_fts5_create_function db insttoken insttoken
 fts5_aux_test_functions db
 
-do_execsql_test 5.0 {
+do_execsql_test $tn.5.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
       x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL%
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
   INSERT INTO ft VALUES('one ONE One oNe oNE one');
 }
 
-do_execsql_test 5.1 {
+do_execsql_test $tn.5.1 {
   SELECT insttoken(ft, 0, 0), 
          insttoken(ft, 1, 0),
          insttoken(ft, 2, 0),
@@ -240,13 +258,37 @@ do_execsql_test 5.1 {
   one one.ONE one.One one.oNe one.oNE one
 }
 
-do_execsql_test 5.2 {
+do_execsql_test $tn.5.2 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft('on*');
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test $tn.5.3 {
+  SELECT insttoken(ft, 0, 0), 
+         insttoken(ft, 1, 0),
+         insttoken(ft, 2, 0),
+         insttoken(ft, 3, 0),
+         insttoken(ft, 4, 0),
+         insttoken(ft, 5, 0)
+  FROM ft(fts5_insttoken('on*'));
+} {
+  one one.ONE one.One one.oNe one.oNE one
+}
+
+do_execsql_test $tn.5.4 {
   SELECT insttoken(ft, 1, 0) FROM ft('one');
 } {
   one.ONE
 }
 
-do_execsql_test 5.3 {
+do_execsql_test $tn.5.5 {
   SELECT fts5_test_poslist(ft) FROM ft('one');
 } {
   {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
@@ -260,10 +302,11 @@ do_execsql_test 5.3 {
 #
 reset_db
 sqlite3_fts5_register_origintext db
-do_execsql_test 6.0 {
+do_execsql_test $tn.6.0 {
   CREATE VIRTUAL TABLE ft USING fts5( 
       x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
   );
+  INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken);
 
   INSERT INTO ft VALUES('One Two', 'Three two');
   INSERT INTO ft VALUES('three Three', 'one One');
@@ -279,34 +322,35 @@ proc tokens {cmd} {
 }
 sqlite3_fts5_create_function db tokens tokens
 
-do_execsql_test 6.1 {
+do_execsql_test $tn.6.1 {
   SELECT rowid, tokens(ft) FROM ft('One');
 } {1 one.One 2 one.One}
 
-do_execsql_test 6.2 {
+do_execsql_test $tn.6.2 {
   SELECT rowid, tokens(ft) FROM ft('on*');
-} {1 {{}} 2 {{} {}}}
+} {1 one.One 2 {one one.One}}
 
-do_execsql_test 6.3 {
+do_execsql_test $tn.6.3 {
   SELECT rowid, tokens(ft) FROM ft('Three*');
-} {1 {{}} 2 {{}}}
+} {1 three.Three 2 three.Three}
 
 fts5_aux_test_functions db
-do_catchsql_test 6.4 {
+do_catchsql_test $tn.6.4 {
   SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
 } {1 SQLITE_RANGE}
 
-do_catchsql_test 6.5 {
+do_catchsql_test $tn.6.5 {
   SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
 } {1 SQLITE_RANGE}
 
-do_catchsql_test 6.6 {
+do_catchsql_test $tn.6.6 {
   CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
 } {1 {malformed tokendata=... directive}}
-do_catchsql_test 6.7 {
+do_catchsql_test $tn.6.7 {
   CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
 } {1 {malformed tokendata=... directive}}
 
+}
 }
 
 finish_test

ext/fts5/test/fts5origintext3.test

@@ -22,6 +22,11 @@ ifcapable !fts5 {
 }
 
 foreach_detail_mode $testprefix {
+  foreach {tn insttoken} {
+    1 0
+    2 1
+  } {
+
   reset_db
 
   sqlite3_fts5_register_origintext db
@@ -32,21 +37,25 @@ foreach_detail_mode $testprefix {
   }
   sqlite3_fts5_create_function db insttoken insttoken
   
-  do_execsql_test 1.0 {
+  do_execsql_test $tn.1.0 {
     CREATE VIRTUAL TABLE ft USING fts5(
         x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
     );
   }
+
+  do_execsql_test $tn.1.0.1 {
+    INSERT INTO ft(ft, rank) VALUES('insttoken', 1);
+  }
   
-  do_execsql_test 1.1 {
+  do_execsql_test $tn.1.1 {
     INSERT INTO ft VALUES('Hello world HELLO WORLD hello');
   }
   
-  do_execsql_test 1.2 {
+  do_execsql_test $tn.1.2 {
     SELECT fts5_test_poslist(ft) FROM ft('hello');
   } {{0.0.0 0.0.2 0.0.4}}
 
-  do_execsql_test 1.3 {
+  do_execsql_test $tn.1.3 {
     SELECT 
       insttoken(ft, 0, 0),
       insttoken(ft, 1, 0),
@@ -54,7 +63,15 @@ foreach_detail_mode $testprefix {
     FROM ft('hello');
   } {hello.Hello hello.HELLO hello}
 
-  do_execsql_test 1.4 {
+  do_execsql_test $tn.1.3.1 {
+    SELECT 
+      insttoken(ft, 0, 0),
+      insttoken(ft, 1, 0),
+      insttoken(ft, 2, 0)
+    FROM ft('hel*');
+  } {hello.Hello hello.HELLO hello}
+
+  do_execsql_test $tn.1.4 {
     SELECT 
       insttoken(ft, 0, 0),
       insttoken(ft, 1, 0),
@@ -62,7 +79,7 @@ foreach_detail_mode $testprefix {
     FROM ft('hello') ORDER BY rank;
   } {hello.Hello hello.HELLO hello}
 
-  do_execsql_test 1.5 {
+  do_execsql_test $tn.1.5 {
     CREATE VIRTUAL TABLE ft2 USING fts5(
         x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
     );
@@ -71,11 +88,11 @@ foreach_detail_mode $testprefix {
     INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE');
   }
 
-  do_execsql_test 1.6 {
+  do_execsql_test $tn.1.6 {
     SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank;
   } {three.THREE 3 three 1 three 2}
 
-  do_execsql_test 1.7 {
+  do_execsql_test $tn.1.7 {
     INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB');
     INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb');
     INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb');
@@ -92,9 +109,32 @@ foreach_detail_mode $testprefix {
     INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB');
   }
 
-  do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
-  do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
+  do_execsql_test $tn.1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24}
+  do_execsql_test $tn.1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24}
 
+  do_execsql_test $tn.2.0 {
+    CREATE VIRTUAL TABLE ft3 USING fts5(
+        x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+        prefix=2
+    );
+  }
+  do_execsql_test $tn.2.1 {
+    INSERT INTO ft3(rowid, x) VALUES(1, 'one');
+    INSERT INTO ft3(rowid, x) VALUES(2, 'ONE');
+    INSERT INTO ft3(rowid, x) VALUES(3, 'ONT');
+    INSERT INTO ft3(rowid, x) VALUES(4, 'on');
+    INSERT INTO ft3(rowid, x) VALUES(5, 'On');
+  }
+
+  do_execsql_test $tn.2.2 {
+    SELECT rowid FROM ft3('on*');
+  } {1 2 3 4 5}
+
+  do_execsql_test $tn.2.3 {
+    SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*');
+  } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On}
+
+  }
 }
 
 finish_test

ext/fts5/test/fts5origintext6.test

@@ -0,0 +1,209 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5origintext6
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc insert_data {tbl} {
+  db eval "
+  INSERT INTO $tbl (rowid, x, y) VALUES
+    (1, 'ChH BDd HhG efc BjJ BGi GBG FdD','ciJ AFf ADf fBJ fhC GFI JEH fcA'),
+    (2, 'deg AIG Fie jII cCd Hbf igF fEE','GeA Ija gJg EDc HFi DDI dCf aDd'),
+    (3, 'IJC hga deC Jfa Aeg hfh CcH dfb','ajD hgC Jaf IfH CHe jIG AjD adF'),
+    (4, 'FiH GJH IDA AiG bBc CGG Eih bIH','hHg JaH aii IHE Ggd gcH gji CGc'),
+    (5, 'ceg CAd jFI GAB BGg EeC IdH acG','bBC eIG ifH eDE Adj bjb GCj ebA'),
+    (6, 'Eac Fbh aFF Eea jeG EIj HCc JJH','hbd giE Gfe eiI dEF abE cJf cAb'),
+    (7, 'dic hAc jEC AiG FEF jHc HiD HBI','aEd ebE Gfi AJG EBA faj GiG jjE'),
+    (8, 'Fca iEe EgE jjJ gce ijf EGc EBi','gaI dhH bFg CFc HeC CjI Jfg ccH'),
+    (9, 'cfd iaa HCf iHJ HjG ffh ABb ibi','CfG bia Dai eii Ejg Jeg fCg hDb'),
+    (10, 'Jjf hJC IID HJj bGB EbJ cgg eBj','jci jhi JAF jIg Bei Bcd cAC AJd'),
+    (11, 'egG Cdi bFf fEB hfH jDH jia Efd','FAd eCg fAi aiC baC eJG acF iGE'),
+    (12, 'Ada Gde CJI ADG gJA Cbb ccF iAB','eAE ajC FBB ccd Jgh fJg ieg hGE'),
+    (13, 'gBb fDG Jdd HdD fiJ Bed Cig iGg','heC FeI iaj gdg ebB giC HaD FIe'),
+    (14, 'FiI iDd Ffe igI bgB EJf FHG hDF','cjC AeI abf Fah cbJ ffH jEb aib'),
+    (15, 'jaF hBI jIH Gdh FEc Fij hgj jFh','dGA ADH feh AAI AfJ DbC gBi hGH'),
+    (16, 'gjH BGg iGj aFE CAH edI idf HEH','hIf DDg fjB hGi cHF BCH FjG Bgd'),
+    (17, 'iaI JGH hji gcj Dda eeG jDd CBi','cHg jeh caG gIc feF ihG hgJ Abj'),
+    (18, 'jHI iDB eFf AiH EFB CDb IAj GbC','Ghe dEI gdI jai gib dAG BIa djb'),
+    (19, 'abI fHG Ccf aAc FDa fiC agF bdB','afi hde IgE bGF cfg DHD diE aca'),
+    (20, 'IFh eDJ jfh cDg dde JGJ GAf fIJ','IBa EfH faE aeI FIF baJ FGj EIH'),
+    (21, 'Dee bFC bBA dEI CEj aJI ghA dCH','hBA ddA HJh dfj egI Dij dFE bGE'),
+    (22, 'JFE BCj FgA afc Jda FGD iHJ HDh','eAI jHe BHD Gah bbD Bgj gbh eGB'),
+    (23, 'edE CJE FjG aFI edA Cea FId iFe','ABG jcA ddj EEc Dcg hAI agA biA'),
+    (24, 'AgE cfc eef cGh aFB DcH efJ hcH','eGF HaB diG fgi bdc iGJ FGJ fFB'),
+    (25, 'aCa AgI GhC DDI hGJ Hgc Gcg bbG','iID Fga jHa jIj idj DFD bAC AFJ'),
+    (26, 'gjC JGh Fge faa eCA iGG gHE Gai','bDi hFE BbI DHD Adb Fgi hCa Hij'),
+    (27, 'Eji jEI jhF DFC afH cDh AGc dHA','IDe GcA ChF DIb Bif HfH agD DGh'),
+    (28, 'gDD AEE Dfg ICf Cbi JdE jgH eEi','eEb dBG FDE jgf cAI FaJ jaA cDd'),
+    (29, 'cbe Gec hgB Egi bca dHg bAJ jBf','EFB DgD GJc fDb EeE bBA GFC Hbe'),
+    (30, 'Adc eHB afI hDc Bhh baE hcJ BBd','JAH deg bcF Dab Bgj Gbb JHi FIB'),
+    (31, 'agF dIj AJJ Hfg cCG hED Igc fHC','JEf eia dHf Ggc Agj geD bEE Gei'),
+    (32, 'DAd cCe cbJ FjG gJe gba dJA GCf','eAf hFc bGE ABI hHA IcE abF CCE'),
+    (33, 'fFh jJe DhJ cDJ EBi AfD eFI IhG','fEG GCc Bjd EFF ggg CFe EHd ciB'),
+    (34, 'Ejb BjI eAF HaD eEJ FaG Eda AHC','Iah hgD EJG fdD cIE Daj IFf eJh'),
+    (35, 'aHG eCe FjA djJ dAJ jiJ IaE GGB','Acg iEF JfB FIC Eei ggj dic Iii'),
+    (36, 'Fdb EDF GaF JjB ehH IgC hgi DCG','cag DHI Fah hAJ bbh egG Hia hgJ'),
+    (37, 'HGg icC JEC AFJ Ddh dhi hfC Ich','fEg bED Bff hCJ EiA cIf bfG cGA'),
+    (38, 'aEJ jGI BCi FaA ebA BHj cIJ GcC','dCH ADd bGB cFE AgF geD cbG jIc'),
+    (39, 'JFB bBi heA BFA hgB Ahj EIE CgI','EIJ JFG FJE GeA Hdg HeH ACh GiA'),
+    (40, 'agB DDC CED igC Dfc DhI eiC fHi','dAB dcg iJF cej Fcc cAc AfB Fdd'),
+    (41, 'BdF DHj Ege hcG DEd eFa dCf gBb','FBG ChB cej iGd Hbh fCc Ibe Abh'),
+    (42, 'Bgc DjI cbC jGD bdb hHB IJA IJH','heg cii abb IGf eDe hJc dii fcE'),
+    (43, 'fhf ECa FiA aDh Jbf CiB Jhe ajD','GFE bIF aeD gDE BIE Jea DfC BEc'),
+    (44, 'GjE dBj DbJ ICF aDh EEH Ejb jFb','dJj aEc IBg bEG Faf fjA hjf FAF'),
+    (45, 'BfA efd IIJ AHG dDF eGg dIJ Gcb','Bfj jeb Ahc dAE ACH Dfb ieb dhC'),
+    (46, 'Ibj ege geC dJh CIi hbD EAG fGA','DEb BFe Bjg FId Fhg HeF JAc BbE'),
+    (47, 'dhB afC hgG bEJ aIe Cbe iEE JCD','bdg Ajc FGA jbh Jge iAj fIA jbE'),
+    (48, 'egH iDi bfH iiI hGC jFF Hfd AHB','bjE Beb iCc haB gIH Dea bga dfd'),
+    (49, 'jgf chc jGc Baj HBb jdE hgh heI','FFB aBd iEB EIG HGf Bbj EIi JbI'),
+    (50, 'jhe EGi ajA fbH geh EHe FdC bij','jDE bBC gbH HeE dcH iBH IFE AHi'),
+    (51, 'aCb JiD cgJ Bjj iAI Hbe IAF FhH','ijf bhE Jdf FED dCH bbG HcJ ebH');
+  "
+}
+
+foreach_detail_mode $testprefix {
+foreach external {0 1 2} {
+  reset_db
+
+  proc tokens {cmd} { 
+    set ret [list]
+    for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+      set txt [$cmd xInstToken $iTok 0]
+      set txt [string map [list "\0" "."] $txt]
+      lappend ret $txt
+    }
+    set ret
+  }
+  sqlite3_fts5_create_function db tokens tokens
+  sqlite3_fts5_register_origintext db
+
+  set E(0) internal
+  set E(1) external
+  set E(2) contentless
+  set e $E($external)
+
+  db eval { CREATE TABLE ex(x, y) }
+  switch -- $external {
+    0 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%
+        );
+      }
+    }
+
+    1 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+            content=ex
+        );
+      }
+    }
+
+    2 {
+      do_execsql_test 1.$e.0 {
+        CREATE VIRTUAL TABLE ft USING fts5(
+            x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%,
+            content=
+        );
+      }
+    }
+  }
+  insert_data ex
+  insert_data ft
+  
+  proc prefixquery {prefix bInst bYOnly} {
+    set ret [list]
+    db eval { SELECT rowid, x, y FROM ex ORDER BY rowid } {
+      set row [list]
+      set bSeen 0
+
+      set T [concat $x $y]
+      if {$bYOnly} { set T $y }
+
+      foreach w $T {
+        if {[string match -nocase $prefix $w]} {
+          set bSeen 1
+          if {$bInst} {
+            set v [string tolower $w]
+            if {$w != $v} { append v ".$w" }
+            lappend row $v
+          }
+        }
+      }
+  
+      if {$bSeen} {
+        lappend ret $rowid
+        lappend ret $row
+      }
+    }
+  
+    set ret
+  }
+  
+  proc do_prefixquery_test {tn prefix} {
+    set bInst [expr {$::e!="contentless" || "%DETAIL%"=="full"}]
+    set expect [prefixquery $prefix $bInst 0]
+    set expect2 [prefixquery $prefix $bInst 1]
+
+    uplevel [list do_execsql_test $tn.1 "
+        SELECT rowid, tokens(ft) FROM ft('$prefix')
+    " $expect]
+    uplevel [list do_execsql_test $tn.2 "
+        SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('$prefix'))
+    " $expect]
+    db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) }
+    uplevel [list do_execsql_test $tn.3 "
+        SELECT rowid, tokens(ft) FROM ft('$prefix')
+    " $expect]
+    db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) }
+
+    if {"%DETAIL%"!="none"} {
+      uplevel [list do_execsql_test $tn.4 "
+          SELECT rowid, tokens(ft) FROM ft('y: $prefix')
+      " $expect2]
+      uplevel [list do_execsql_test $tn.5 "
+          SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('y: $prefix'))
+      " $expect2]
+      db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) }
+      uplevel [list do_execsql_test $tn.6 "
+          SELECT rowid, tokens(ft) FROM ft('y: $prefix')
+      " $expect2]
+      db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) }
+    }
+  }
+  
+  do_prefixquery_test 1.$e.1 a*
+  do_prefixquery_test 1.$e.2 b*
+  do_prefixquery_test 1.$e.3 c*
+  do_prefixquery_test 1.$e.4 d*
+  do_prefixquery_test 1.$e.5 e*
+  do_prefixquery_test 1.$e.6 f*
+  do_prefixquery_test 1.$e.7 g*
+  do_prefixquery_test 1.$e.8 h*
+  do_prefixquery_test 1.$e.9 i*
+  do_prefixquery_test 1.$e.10 j*
+}}
+
+
+
+finish_test
+

ext/fts5/test/fts5tokendata.test

@@ -0,0 +1,105 @@
+# 2014 Jan 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focused on phrase queries.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokendata
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+foreach_detail_mode $testprefix {
+
+  sqlite3_fts5_register_origintext db
+  fts5_aux_test_functions db
+  proc b {x} { string map [list "\0" "."] $x }
+  db func b b
+
+  do_execsql_test 1.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(a, b, tokendata=1,
+      tokenize="origintext unicode61", detail=%DETAIL%
+    );
+    CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance);
+  }
+
+  do_execsql_test 1.1 {
+    INSERT INTO ft(rowid, a, b) VALUES
+      (1, 'Pedagog Pedal Pedant', 'Peculier Day Today'),
+      (2, 'Pedant pedantic pecked', 'Peck Penalize Pen');
+
+    INSERT INTO ft(rowid, a, b) VALUES
+      (3, 'Penalty Pence Penciled', 'One Two Three'),
+      (4, 'Pedant Pedal Pedant', 'Peculier Day Today');
+  }
+
+  do_execsql_test 1.2 {
+    SELECT DISTINCT b(term) FROM vocab
+  } {
+    day.Day one.One peck.Peck pecked peculier.Peculier pedagog.Pedagog
+    pedal.Pedal pedant.Pedant pedantic pen.Pen penalize.Penalize 
+    penalty.Penalty pence.Pence penciled.Penciled three.Three 
+    today.Today two.Two
+  }
+
+  do_execsql_test 1.3.1 {
+    SELECT rowid FROM ft('pe*')
+  } {
+    1 2 3 4
+  }
+
+  do_execsql_test 1.3.2 {
+    SELECT rowid FROM ft('pe*') ORDER BY rowid DESC
+  } {
+    4 3 2 1
+  }
+  
+  if {"%DETAIL%"!="none"} {
+    do_execsql_test 1.3.3 {
+      SELECT rowid FROM ft WHERE a MATCH 'pe*' ORDER BY rowid DESC
+    } {
+      4 3 2 1
+    }
+  }
+
+  do_execsql_test 1.4 {
+    SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pedant')
+  } {
+    1 pedant.Pedant
+    2 pedant.Pedant
+    4 pedant.Pedant
+  }
+
+  do_execsql_test 1.5 {
+    SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pe*')
+  } {
+    1 pedagog.Pedagog
+    2 pedant.Pedant
+    3 penalty.Penalty
+    4 pedant.Pedant
+  }
+
+  do_execsql_test 1.6 {
+    SELECT rowid, fts5_test_poslist(ft) FROM ft('pe*')
+  } {
+    1 {0.0.0 0.0.1 0.0.2 0.1.0} 
+    2 {0.0.0 0.0.1 0.0.2 0.1.0 0.1.1 0.1.2}
+    3 {0.0.0 0.0.1 0.0.2}
+    4 {0.0.0 0.0.1 0.0.2 0.1.0}
+  }
+}
+
+finish_test
+

main.mk

@@ -425,7 +425,6 @@ $(MAKE_SANITY_CHECK): $(MAKEFILE_LIST) $(TOP)/auto.def
 	@if [ x = "x$(TOP)" ]; then echo "Missing TOP var" 1>&2; exit 1; fi
 	@if [ ! -d "$(TOP)" ]; then echo "$(TOP) is not a directory" 1>&2; exit 1; fi
 	@if [ ! -f "$(TOP)/auto.def" ]; then echo "$(TOP) does not appear to be the top-most source dir" 1>&2; exit 1; fi
-	@if [ x = "x$(PACKAGE_VERSION)" ]; then echo "PACKAGE_VERSION must be set to the library's X.Y.Z-format version number" 1>&2; exit 1; fi
 	@if [ x = "x$(B.cc)" ]; then echo "Missing B.cc var" 1>&2; exit 1; fi
 	@if [ x = "x$(T.cc)" ]; then echo "Missing T.cc var" 1>&2; exit 1; fi
 	@if [ x = "x$(B.tclsh)" ]; then echo "Missing B.tclsh var" 1>&2; exit 1; fi

manifest

@@ -1,5 +1,5 @@
-C Re-phrase\ssome\s(#if\s!SQLITE_CORE)\sto\s(#ifndef\sSQLITE_CORE),\sas\sdiscussed\sin\sforum:cea40371c5e34b09\s|\sfor\spost\scea40371c5e34b09].
-D 2024-11-06T12:58:31.469
+C Merge\slatest\strunk\schanges\sinto\sthis\sbranch.
+D 2024-11-06T17:31:48.766
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d
@@ -107,14 +107,14 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
 F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15
 F ext/fts5/fts5.h 6b4b92df890965567360db5f1ead24fd13a72cb23b95e4ed2ff58d1d89f7aa42
-F ext/fts5/fts5Int.h bf0d3efa144f36e00f9b5206626aec2f436f58186a0835092394f2202e9828e3
+F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67
 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
-F ext/fts5/fts5_config.c a6633d88596758941c625b526075b85d3d9fd1089d8d9eab5db6e8a71fd347ad
-F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33
+F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8
+F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
-F ext/fts5/fts5_index.c 368a968570ce12ba40223e284a588d9f93ee23a0133727f0df1fcd64086b1fb6
-F ext/fts5/fts5_main.c 50eb059e51d730e8e0c77df4e568b018079e112a755c094488b0d5b1aa06afbb
+F ext/fts5/fts5_index.c 2cef40d6fdd761229dd4127e0b1ddcb61dfd6a4ac7e73653b7fddbe0075e50be
+F ext/fts5/fts5_main.c b2ec6bf97fc378906c0e78c61f10ca8e64f15e03237f2521f7d81736983be378
 F ext/fts5/fts5_storage.c 337b05e4c66fc822d031e264d65bde807ec2fab08665ca2cc8aaf9c5fa06792c
 F ext/fts5/fts5_tcl.c aee6ae6d0c6968564c392bf0d09aaabb4d8bea9ca69fd224dc9b44243324acbf
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -215,11 +215,12 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47
 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed
 F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3
-F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d
+F ext/fts5/test/fts5origintext.test 3b73aa036ce5244bb7c5782c5441b979585bdca026accf75d16026a2a8119c09
 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba
-F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b
+F ext/fts5/test/fts5origintext3.test 4988b6375acc3bbb0515667765f57e389caf449814af9c1095c053f7de2b4223
 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8
 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b
+F ext/fts5/test/fts5origintext6.test 09eb1347cb0dceaebbebf3d3e6bd5d24c7c1006efddc2984540450324bbdafa4
 F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c
 F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626
 F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0
@@ -248,6 +249,7 @@ F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988
 F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87
 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
+F ext/fts5/test/fts5tokendata.test 7cad79af82e8e81b7a36b450087233d2fca051bb0d584421afc375d6dd26d6f6
 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98
 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6
 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3
@@ -699,7 +701,7 @@ F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65a
 F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
 F ext/wasm/wasmfs.make bc8bb227f35d5bd3863a7bd2233437c37472a0d81585979f058f9b9b503bef35
 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0
-F main.mk 949721d1f7400fd9f2c9a8818d66eed7b4c5937c9fd2635f8364755858ddcc78
+F main.mk c1a91dc50ffc1cb828b85d767d02d80d9eabeb0ef47e77e57bd72617749c5dbf
 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271
 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504
 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421
@@ -2200,9 +2202,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P d3887895a33742fb1fc97235cf897d295b237a9fb5a84031826f9c1018106f18 cd82e4c0f5f8ff16468b909d84dd5545c0456f624db61a4d112467a7cafed2fc
-R 2ce6f48a525cff0a5e55030616b254d4
-T +closed cd82e4c0f5f8ff16468b909d84dd5545c0456f624db61a4d112467a7cafed2fc Closed\sby\sintegrate-merge.
-U stephan
-Z 49b3cdd634f6b6d641022069d2594f18
+P 9cc04331a01760189d88697233009dbe8a60eda589792ad01b56300499e9f54d 5495b12569c318d5020b4b5a625a392ef8e777b81c0200624fbbc2a6b5eddef9
+R 43b03038eb634e877d13ab18dcd80180
+U dan
+Z 70a74ef30cc348badfc60dd108f53b23
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-5495b12569c318d5020b4b5a625a392ef8e777b81c0200624fbbc2a6b5eddef9
+edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746