Fix a problem with fts5 synonyms and the xQueryPhrase() auxiliary function API.

FossilOrigin-Name: cf3e45e76d23e10ee06296c3561a341591597a04

ext/fts5/fts5Int.h

@@ -599,7 +599,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
 int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
 int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
 
-int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
+int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
 
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written

ext/fts5/fts5_expr.c

@@ -251,79 +251,6 @@ int sqlite3Fts5ExprNew(
   return sParse.rc;
 }
 
-/*
-** Create a new FTS5 expression by cloning phrase iPhrase of the
-** expression passed as the second argument.
-*/
-int sqlite3Fts5ExprPhraseExpr(
-  Fts5Config *pConfig,
-  Fts5Expr *pExpr, 
-  int iPhrase, 
-  Fts5Expr **ppNew
-){
-  int rc = SQLITE_OK;             /* Return code */
-  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
-  Fts5ExprPhrase *pCopy;          /* Copy of pOrig */
-  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-
-  pOrig = pExpr->apExprPhrase[iPhrase];
-  pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc, 
-      sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
-  );
-  if( pCopy ){
-    int i;                          /* Used to iterate through phrase terms */
-    Fts5ExprPhrase **apPhrase;
-    Fts5ExprNode *pNode;
-    Fts5ExprNearset *pNear;
-
-    pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
-    apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
-        sizeof(Fts5ExprPhrase*)
-    );
-    pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode));
-    pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 
-        sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
-    );
-
-    for(i=0; i<pOrig->nTerm; i++){
-      pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1);
-      pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
-    }
-
-    if( rc==SQLITE_OK ){
-      /* All the allocations succeeded. Put the expression object together. */
-      pNew->pIndex = pExpr->pIndex;
-      pNew->pRoot = pNode;
-      pNew->nPhrase = 1;
-      pNew->apExprPhrase = apPhrase;
-      pNew->apExprPhrase[0] = pCopy;
-
-      pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING);
-      pNode->pNear = pNear;
-
-      pNear->nPhrase = 1;
-      pNear->apPhrase[0] = pCopy;
-
-      pCopy->nTerm = pOrig->nTerm;
-      pCopy->pNode = pNode;
-    }else{
-      /* At least one allocation failed. Free them all. */
-      for(i=0; i<pOrig->nTerm; i++){
-        sqlite3_free(pCopy->aTerm[i].zTerm);
-      }
-      sqlite3_free(pCopy);
-      sqlite3_free(pNear);
-      sqlite3_free(pNode);
-      sqlite3_free(apPhrase);
-      sqlite3_free(pNew);
-      pNew = 0;
-    }
-  }
-
-  *ppNew = pNew;
-  return rc;
-}
-
 /*
 ** Free the expression node object passed as the only argument.
 */
@@ -1550,8 +1477,8 @@ static int fts5ParseTokenize(
   int tflags,                     /* Mask of FTS5_TOKEN_* flags */
   const char *pToken,             /* Buffer containing token */
   int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Start offset of token */
-  int iEnd                        /* End offset of token */
+  int iUnused1,                   /* Start offset of token */
+  int iUnused2                    /* End offset of token */
 ){
   int rc = SQLITE_OK;
   const int SZALLOC = 8;
@@ -1577,7 +1504,7 @@ static int fts5ParseTokenize(
 
       pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, 
           sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
-          );
+      );
       if( pNew==0 ) return SQLITE_NOMEM;
       if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
       pCtx->pPhrase = pPhrase = pNew;
@@ -1675,6 +1602,83 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
   return sCtx.pPhrase;
 }
 
+/*
+** Create a new FTS5 expression by cloning phrase iPhrase of the
+** expression passed as the second argument.
+*/
+int sqlite3Fts5ExprClonePhrase(
+  Fts5Config *pConfig,
+  Fts5Expr *pExpr, 
+  int iPhrase, 
+  Fts5Expr **ppNew
+){
+  int rc = SQLITE_OK;             /* Return code */
+  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
+  Fts5ExprPhrase *pCopy;          /* Copy of pOrig */
+  int i;                          /* Used to iterate through phrase terms */
+
+  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
+  Fts5ExprPhrase **apPhrase;      /* pNew->apPhrase */
+  Fts5ExprNode *pNode;            /* pNew->pRoot */
+  Fts5ExprNearset *pNear;         /* pNew->pRoot->pNear */
+
+  TokenCtx sCtx = {0};            /* Context object for fts5ParseTokenize */
+
+
+  pOrig = pExpr->apExprPhrase[iPhrase];
+
+  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
+  if( rc==SQLITE_OK ){
+    pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
+        sizeof(Fts5ExprPhrase*));
+  }
+  if( rc==SQLITE_OK ){
+    pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, 
+        sizeof(Fts5ExprNode));
+  }
+  if( rc==SQLITE_OK ){
+    pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 
+        sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
+  }
+
+  for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
+    int tflags = 0;
+    Fts5ExprTerm *p;
+    for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
+      const char *zTerm = p->zTerm;
+      rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0);
+      tflags = FTS5_TOKEN_COLOCATED;
+    }
+    if( rc==SQLITE_OK ){
+      sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    /* All the allocations succeeded. Put the expression object together. */
+    pNew->pIndex = pExpr->pIndex;
+    pNew->nPhrase = 1;
+    pNew->apExprPhrase[0] = sCtx.pPhrase;
+    pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
+    pNew->pRoot->pNear->nPhrase = 1;
+    sCtx.pPhrase->pNode = pNew->pRoot;
+
+    if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+      pNew->pRoot->eType = FTS5_TERM;
+    }else{
+      pNew->pRoot->eType = FTS5_STRING;
+    }
+  }else{
+    sqlite3Fts5ExprFree(pNew);
+    fts5ExprPhraseFree(sCtx.pPhrase);
+    pNew = 0;
+  }
+
+  *ppNew = pNew;
+  return rc;
+}
+
+
 /*
 ** Token pTok has appeared in a MATCH expression where the NEAR operator
 ** is expected. If token pTok does not contain "NEAR", store an error

ext/fts5/fts5_main.c

@@ -1860,7 +1860,7 @@ static int fts5ApiQueryPhrase(
     pNew->iFirstRowid = SMALLEST_INT64;
     pNew->iLastRowid = LARGEST_INT64;
     pNew->base.pVtab = (sqlite3_vtab*)pTab;
-    rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
+    rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
   }
 
   if( rc==SQLITE_OK ){

ext/fts5/test/fts5synonym.test

@@ -298,6 +298,32 @@ foreach {tn q res} {
   } $res
 }
 
+# Test that the xQueryPhrase() API works with synonyms.
+#
+proc mit {blob} {
+  set scan(littleEndian) i*
+  set scan(bigEndian) I*
+  binary scan $blob $scan($::tcl_platform(byteOrder)) r
+  return $r
+}
+db func mit mit
+sqlite3_fts5_register_matchinfo db
+
+foreach {tn q res} {
+  1 {one} {
+      1 {1 11 7 2 12 6}     2 {2 11 7 0 12 6} 
+      3 {2 11 7 1 12 6}     4 {1 11 7 2 12 6} 
+      5 {3 11 7 0 12 6}     6 {0 11 7 2 12 6} 
+      7 {0 11 7 3 12 6}     8 {1 11 7 0 12 6} 
+      9 {1 11 7 2 12 6}
+  }
+} {
+  do_execsql_test 5.2.$tn {
+    SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q
+  } $res
+}
+
+
 #-------------------------------------------------------------------------
 # Test terms with more than 4 synonyms.
 #

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sthat\soccurs\swhen\smore\sthan\s4\ssynonyms\sfor\sa\sterm\sappear\swithin\sa\ssingle\srow.
-D 2015-09-01T18:44:33.194
+C Fix\sa\sproblem\swith\sfts5\ssynonyms\sand\sthe\sxQueryPhrase()\sauxiliary\sfunction\sAPI.
+D 2015-09-02T08:22:41.779
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in e2218eb228374422969de7b1680eda6864affcef
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
 F ext/fts5/fts5.h 0784692f406588e6c90e13a78e1f36e7e3236e42
-F ext/fts5/fts5Int.h d46f89aeb357fbcf5b268d71b0d2c5000cd27bd9
+F ext/fts5/fts5Int.h c6f035091fc9fa12a92c7066bf0c266c08cb508b
 F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
 F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
 F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
-F ext/fts5/fts5_expr.c 44caa0ccd7e9a392864fda0c14e9b9829a395a84
+F ext/fts5/fts5_expr.c d92beea335f96f9256cc19203b21fd45826655c9
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
 F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
-F ext/fts5/fts5_main.c b00834ac543431dc35edbe18018b4befe0c7fd42
+F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e
 F ext/fts5/fts5_storage.c c888defbb961d64c12299b3d1725a24a770b047e
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
 F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1
@@ -172,7 +172,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
-F ext/fts5/test/fts5synonym.test 4321e59c29186d9187cf8ab576e70530176eae49
+F ext/fts5/test/fts5synonym.test fd66afccec36bf719ba66a5b6579efe007607b05
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -1381,7 +1381,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P dbcb73802b88f76be17f09f3eb83ffac542de633
-R 07c5ba5d7a2806d5caa7dc123b3f32ad
+P cd359550bdc2bf7be4c294b60130c9fc3f583902
+R f0fefb84cb3010de72f87418a092a81a
 U dan
-Z fc32f7df10846658b7b8e62d59624854
+Z b6230653499a35c239ca936f1dd1428a

manifest.uuid

@@ -1 +1 @@
-cd359550bdc2bf7be4c294b60130c9fc3f583902
+cf3e45e76d23e10ee06296c3561a341591597a04