Fixes to fts5 snippet() function.

FossilOrigin-Name: d464a7b18d212720a4f4e2fa4e204f1aca99c837
2016-08-27 18:34:06 +00:00 · 2016-08-27 18:34:06 +00:00 · 57b60432ba
commit 57b60432ba
parent 1ad78c5791 37ff4d4c77
6 changed files with 262 additions and 96 deletions
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@ -246,6 +246,115 @@ static void fts5HighlightFunction(
 ** End of highlight() implementation.
 **************************************************************************/

+/*
+** Context object passed to the fts5SentenceFinderCb() function.
+*/
+typedef struct Fts5SFinder Fts5SFinder;
+struct Fts5SFinder {
+  int iPos;                       /* Current token position */
+  int nFirstAlloc;                /* Allocated size of aFirst[] */
+  int nFirst;                     /* Number of entries in aFirst[] */
+  int *aFirst;                    /* Array of first token in each sentence */
+  const char *zDoc;               /* Document being tokenized */
+};
+
+/*
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
+** error occurs.
+*/
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
+  if( p->nFirstAlloc==p->nFirst ){
+    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
+    int *aNew;
+
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    p->aFirst = aNew;
+    p->nFirstAlloc = nNew;
+  }
+  p->aFirst[p->nFirst++] = iAdd;
+  return SQLITE_OK;
+}
+
+/*
+** This function is an xTokenize() callback used by the auxiliary snippet()
+** function. Its job is to identify tokens that are the first in a sentence.
+** For each such token, an entry is added to the SFinder.aFirst[] array.
+*/
+static int fts5SentenceFinderCb(
+  void *pContext,                 /* Pointer to HighlightContext object */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
+){
+  int rc = SQLITE_OK;
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+    Fts5SFinder *p = (Fts5SFinder*)pContext;
+    if( p->iPos>0 ){
+      int i;
+      char c = 0;
+      for(i=iStartOff-1; i>=0; i--){
+        c = p->zDoc[i];
+        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
+      }
+      if( i!=iStartOff-1 && (c=='.' || c==':') ){
+        rc = fts5SentenceFinderAdd(p, p->iPos);
+      }
+    }else{
+      rc = fts5SentenceFinderAdd(p, 0);
+    }
+    p->iPos++;
+  }
+  return rc;
+}
+
+static int fts5SnippetScore(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nDocsize,                   /* Size of column in tokens */
+  unsigned char *aSeen,           /* Array with one element per query phrase */
+  int iCol,                       /* Column to score */
+  int iPos,                       /* Starting offset to score */
+  int nToken,                     /* Max tokens per snippet */
+  int *pnScore,                   /* OUT: Score */
+  int *piPos                      /* OUT: Adjusted offset */
+){
+  int rc;
+  int i;
+  int ip = 0;
+  int ic = 0;
+  int iOff = 0;
+  int iFirst = -1;
+  int nInst;
+  int nScore = 0;
+  int iLast = 0;
+
+  rc = pApi->xInstCount(pFts, &nInst);
+  for(i=0; i<nInst && rc==SQLITE_OK; i++){
+    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
+      nScore += (aSeen[ip] ? 1 : 1000);
+      aSeen[ip] = 1;
+      if( iFirst<0 ) iFirst = iOff;
+      iLast = iOff + pApi->xPhraseSize(pFts, ip);
+    }
+  }
+
+  *pnScore = nScore;
+  if( piPos ){
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
+    if( iAdj<0 ) iAdj = 0;
+    *piPos = iAdj;
+  }
+
+  return rc;
+}
+
 /*
 ** Implementation of snippet() function.
 */
@ -267,9 +376,10 @@ static void fts5SnippetFunction(
  unsigned char *aSeen;           /* Array of "seen instance" flags */
  int iBestCol;                   /* Column containing best snippet */
  int iBestStart = 0;             /* First token of best snippet */
-  int iBestLast;                  /* Last token of best snippet */
  int nBestScore = 0;             /* Score of best snippet */
  int nColSize = 0;               /* Total size of iBestCol in tokens */
+  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
+  int nCol;

  if( nVal!=5 ){
    const char *zErr = "wrong number of arguments to function snippet()";
@ -277,13 +387,13 @@ static void fts5SnippetFunction(
    return;
  }

+  nCol = pApi->xColumnCount(pFts);
  memset(&ctx, 0, sizeof(HighlightContext));
  iCol = sqlite3_value_int(apVal[0]);
  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
  zEllips = (const char*)sqlite3_value_text(apVal[3]);
  nToken = sqlite3_value_int(apVal[4]);
-  iBestLast = nToken-1;

  iBestCol = (iCol>=0 ? iCol : 0);
  nPhrase = pApi->xPhraseCount(pFts);
@ -291,59 +401,82 @@ static void fts5SnippetFunction(
  if( aSeen==0 ){
    rc = SQLITE_NOMEM;
  }
-
  if( rc==SQLITE_OK ){
    rc = pApi->xInstCount(pFts, &nInst);
  }
-  for(i=0; rc==SQLITE_OK && i<nInst; i++){
-    int ip, iSnippetCol, iStart;
-    memset(aSeen, 0, nPhrase);
-    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
-    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
-      int nScore = 1000;
-      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
-      int j;
-      aSeen[ip] = 1;

-      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
-        int ic; int io; int iFinal;
-        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
-        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
-        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
-          nScore += aSeen[ip] ? 1000 : 1;
-          aSeen[ip] = 1;
-          if( iFinal>iLast ) iLast = iFinal;
+  memset(&sFinder, 0, sizeof(Fts5SFinder));
+  for(i=0; i<nCol; i++){
+    if( iCol<0 || iCol==i ){
+      int nDoc;
+      int nDocsize;
+      int ii;
+      sFinder.iPos = 0;
+      sFinder.nFirst = 0;
+      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      );
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xColumnSize(pFts, i, &nDocsize);
+      if( rc!=SQLITE_OK ) break;
+
+      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
+        int ip, ic, io;
+        int iAdj;
+        int nScore;
+        int jj;
+
+        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
+        if( ic!=i || rc!=SQLITE_OK ) continue;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
+            io, nToken, &nScore, &iAdj
+        );
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = iAdj;
+          nColSize = nDocsize;
        }
-      }

-      if( rc==SQLITE_OK && nScore>nBestScore ){
-        iBestCol = iSnippetCol;
-        iBestStart = iStart;
-        iBestLast = iLast;
-        nBestScore = nScore;
+        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
+          for(jj=0; jj<(sFinder.nFirst-1); jj++){
+            if( sFinder.aFirst[jj+1]>io ) break;
+          }
+
+          if( sFinder.aFirst[jj]<io ){
+            int nScore;
+            memset(aSeen, 0, nPhrase);
+            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+              sFinder.aFirst[jj], nToken, &nScore, 0
+            );
+
+            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
+            if( rc==SQLITE_OK && nScore>nBestScore ){
+              nBestScore = nScore;
+              iBestCol = i;
+              iBestStart = sFinder.aFirst[jj];
+              nColSize = nDocsize;
+            }
+          }
+        }
      }
    }
  }

-  if( rc==SQLITE_OK ){
-    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
-  }
  if( rc==SQLITE_OK ){
    rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
  }
+  if( rc==SQLITE_OK && nColSize==0 ){
+    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
+  }
  if( ctx.zIn ){
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
    }

-    if( (iBestStart+nToken-1)>iBestLast ){
-      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
-    }
-    if( iBestStart+nToken>nColSize ){
-      iBestStart = nColSize - nToken;
-    }
-    if( iBestStart<0 ) iBestStart = 0;
-
    ctx.iRangeStart = iBestStart;
    ctx.iRangeEnd = iBestStart + nToken - 1;

@ -365,15 +498,15 @@ static void fts5SnippetFunction(
    }else{
      fts5HighlightAppend(&rc, &ctx, zEllips, -1);
    }
-
-    if( rc==SQLITE_OK ){
-      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
-    }
-    sqlite3_free(ctx.zOut);
  }
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3_free(ctx.zOut);
  sqlite3_free(aSeen);
+  sqlite3_free(sFinder.aFirst);
 }

 /************************************************************************/
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@ -72,45 +72,56 @@ foreach {tn doc res} {
  2.2 {o X o o o o o o} {o [X] o o o o o...}
  2.3 {o o X o o o o o} {o o [X] o o o o...}
  2.4 {o o o X o o o o} {o o o [X] o o o...}
-  2.5 {o o o o X o o o} {...o o o [X] o o o}
-  2.6 {o o o o o X o o} {...o o o o [X] o o}
-  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.5 {o o o o X o o o} {o o o o [X] o o...}
+  2.6 {o o o o o X o o} {o o o o o [X] o...}
+  2.7 {o o o o o o X o} {o o o o o o [X]...}
  2.8 {o o o o o o o X} {...o o o o o o [X]}

+  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
+  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
+  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
+  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+
+
  3.1 {X o o o o o o o o} {[X] o o o o o o...}
  3.2 {o X o o o o o o o} {o [X] o o o o o...}
  3.3 {o o X o o o o o o} {o o [X] o o o o...}
  3.4 {o o o X o o o o o} {o o o [X] o o o...}
-  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
-  3.6 {o o o o o X o o o} {...o o o [X] o o o}
-  3.7 {o o o o o o X o o} {...o o o o [X] o o}
-  3.8 {o o o o o o o X o} {...o o o o o [X] o}
-  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}

  4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
-  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
-  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}

  5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
-  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
-  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
-  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}

  6.1 {X o o o X o o o} {[X] o o o [X] o o...}
  6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
-  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
-  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
-  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}

  7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
  7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
-  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
-  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
-  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
-  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}

-  8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
+  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
      {...o o [X] [X] [X] o o...}
+  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
+      {...o o [X] o o o o...}
+  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {o o o o [X] o o...}
 } {
  do_snippet_test 1.$tn $doc X $res
 }
@ -127,24 +138,43 @@ if {[detail_is_full]} {
    2.1 {X Y o o o o o o} {[X Y] o o o o o...}
    2.2 {o X Y o o o o o} {o [X Y] o o o o...}
    2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}

    3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
    3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
    3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
  } {
    do_snippet_test 2.$tn $doc "X + Y" $res
  }
 }

+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
+  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
+} {
+  {1 2 3 4 5...}
+}
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
+  INSERT INTO p1 VALUES(
+    'x a a a a a a a a a a',
+    'a a a a a a a a a a a a a a a a a a a x'
+  );
+}
+do_execsql_test 5.1 {
+  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
+} {{[x] a a a a a...}}
+
 } ;# foreach_detail_mode 

 finish_test
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@ -160,12 +160,12 @@ foreach {tn query snippet} {
     the maximum x value.
  }
  4 "rollback" {
-     ...[ROLLBACK]. Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
  }
  5 "rOllback" {
-     ...[ROLLBACK]. Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
  }
  6 "lang*" {
     Added support for the FTS4 [languageid] option.
--- a/20
+++ b/20
@ -1,5 +1,5 @@
-C Fix\sthe\sextra\scomments\s(added\swith\s-DSQLITE_ENABLE_EXPLAIN_COMMENTS)\sso\sthat\nthe\ssense\sof\s&lt;,\s&lt;=,\s&gt;,\sand\s&gt;=\stests\sis\scorrect\sand\sso\sthat\sthe\nSQLITE_STOREP2\sversion\sis\sshown\scorrectly.\s\sThese\schanges\sare\salready\sin\sthe\nrowvalue\sbranch\sbut\sare\sadded\shere\ssince\sthey\sare\stechnically\sunrelated\sto\nrowvalue.
-D 2016-08-27T14:05:12.248
+C Fixes\sto\sfts5\ssnippet()\sfunction.
+D 2016-08-27T18:34:06.238
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c e4bec077c5190946dbaac72c6555defd823724ca
+F ext/fts5/fts5_aux.c 2f20784a344701d4c72986e2e692062dd47d568c
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b6afd7c28ad62d546c30f387fb971f3aaebaac0d
+F ext/fts5/test/fts5af.test c92825778ed2adc80014832762c056bbe968ef88
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@ -190,7 +190,7 @@ F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
+F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
@ -349,7 +349,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/insert.c 3edb5a1bda44df13531fedfcde5fbcc2fc04c222
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
 F src/loadext.c dd7a2b77902cc66c22555aef02e1a682554b7aec
-F src/main.c 0bfcf1a5b612dddb4dd0fd04b53ef08ace0f48ff
+F src/main.c 9821bb4d2399bc5a0b8a5dfc06bede2d1520255c
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7090147903337864d54ecfb2cd84a3f823973167
-R ad53b888d897b0241520026ffdb78d22
-U drh
-Z c3f9980f15e4ee005b5e39955b2d9131
+P 4d43c4698eef4e3db7556813f0274b4018c7c2b9 b174ed2bde43a1872374ce18b85ac9f067e86fbb
+R 1806c129c8fa52174952b67042480d44
+U dan
+Z 3f2d594eefc4bc080c151857cd7756e9
--- a/manifest.uuid
+++ b/manifest.uuid
@ -1 +1 @@
-4d43c4698eef4e3db7556813f0274b4018c7c2b9
+d464a7b18d212720a4f4e2fa4e204f1aca99c837
--- a/src/main.c
+++ b/src/main.c
@ -2955,11 +2955,20 @@ static int openDatabase(
  */
  sqlite3Error(db, SQLITE_OK);
  sqlite3RegisterPerConnectionBuiltinFunctions(db);
+  rc = sqlite3_errcode(db);
+
+#ifdef SQLITE_ENABLE_FTS5
+  /* Register any built-in FTS5 module before loading the automatic
+  ** extensions. This allows automatic extensions to register FTS5 
+  ** tokenizers and auxiliary functions.  */
+  if( !db->mallocFailed && rc==SQLITE_OK ){
+    rc = sqlite3Fts5Init(db);
+  }
+#endif

  /* Load automatic extensions - extensions that have been registered
  ** using the sqlite3_automatic_extension() API.
  */
-  rc = sqlite3_errcode(db);
  if( rc==SQLITE_OK ){
    sqlite3AutoLoadExtensions(db);
    rc = sqlite3_errcode(db);
@ -2988,12 +2997,6 @@ static int openDatabase(
  }
 #endif

-#ifdef SQLITE_ENABLE_FTS5
-  if( !db->mallocFailed && rc==SQLITE_OK ){
-    rc = sqlite3Fts5Init(db);
-  }
-#endif
-
 #ifdef SQLITE_ENABLE_ICU
  if( !db->mallocFailed && rc==SQLITE_OK ){
    rc = sqlite3IcuInit(db);