Update fts5 to support "<colset> : ( <expr> )" for column filtering, as well
as "<colset> : NEAR(...)" and "<colset> : <phrase>". FossilOrigin-Name: c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396
This commit is contained in:
dan
parent
26849e14c2
commit
03155f659f
@ -738,7 +738,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*);
|
||||
void sqlite3Fts5ParseNodeFree(Fts5ExprNode*);
|
||||
|
||||
void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*);
|
||||
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*);
|
||||
void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*);
|
||||
Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*);
|
||||
void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p);
|
||||
void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*);
|
||||
|
||||
@ -1886,25 +1886,110 @@ Fts5Colset *sqlite3Fts5ParseColset(
|
||||
return pRet;
|
||||
}
|
||||
|
||||
/*
|
||||
** If argument pOrig is NULL, or if (*pRc) is set to anything other than
|
||||
** SQLITE_OK when this function is called, NULL is returned.
|
||||
**
|
||||
** Otherwise, a copy of (*pOrig) is made into memory obtained from
|
||||
** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation
|
||||
** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned.
|
||||
*/
|
||||
static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){
|
||||
Fts5Colset *pRet;
|
||||
if( pOrig ){
|
||||
int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int);
|
||||
pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte);
|
||||
if( pRet ){
|
||||
memcpy(pRet, pOrig, nByte);
|
||||
}
|
||||
}else{
|
||||
pRet = 0;
|
||||
}
|
||||
return pRet;
|
||||
}
|
||||
|
||||
/*
|
||||
** Remove from colset pColset any columns that are not also in colset pMerge.
|
||||
*/
|
||||
static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){
|
||||
int iIn = 0; /* Next input in pColset */
|
||||
int iMerge = 0; /* Next input in pMerge */
|
||||
int iOut = 0; /* Next output slot in pColset */
|
||||
|
||||
while( iIn<pColset->nCol && iMerge<pMerge->nCol ){
|
||||
int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge];
|
||||
if( iDiff==0 ){
|
||||
pColset->aiCol[iOut++] = pMerge->aiCol[iMerge];
|
||||
iMerge++;
|
||||
iIn++;
|
||||
}else if( iDiff>0 ){
|
||||
iMerge++;
|
||||
}else{
|
||||
iIn++;
|
||||
}
|
||||
}
|
||||
pColset->nCol = iOut;
|
||||
}
|
||||
|
||||
/*
|
||||
** Recursively apply colset pColset to expression node pNode and all of
|
||||
** its decendents. If (*ppFree) is not NULL, it contains a spare copy
|
||||
** of pColset. This function may use the spare copy and set (*ppFree) to
|
||||
** zero, or it may create copies of pColset using fts5CloneColset().
|
||||
*/
|
||||
static void fts5ParseSetColset(
|
||||
Fts5Parse *pParse,
|
||||
Fts5ExprNode *pNode,
|
||||
Fts5Colset *pColset,
|
||||
Fts5Colset **ppFree
|
||||
){
|
||||
if( pParse->rc==SQLITE_OK ){
|
||||
assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING
|
||||
|| pNode->eType==FTS5_AND || pNode->eType==FTS5_OR
|
||||
|| pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF
|
||||
);
|
||||
if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){
|
||||
Fts5ExprNearset *pNear = pNode->pNear;
|
||||
if( pNear->pColset ){
|
||||
fts5MergeColset(pNear->pColset, pColset);
|
||||
if( pNear->pColset->nCol==0 ){
|
||||
pNode->eType = FTS5_EOF;
|
||||
pNode->xNext = 0;
|
||||
}
|
||||
}else if( *ppFree ){
|
||||
pNear->pColset = pColset;
|
||||
*ppFree = 0;
|
||||
}else{
|
||||
pNear->pColset = fts5CloneColset(&pParse->rc, pColset);
|
||||
}
|
||||
}else{
|
||||
int i;
|
||||
assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 );
|
||||
for(i=0; i<pNode->nChild; i++){
|
||||
fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
** Apply colset pColset to expression node pExpr and all of its descendents.
|
||||
*/
|
||||
void sqlite3Fts5ParseSetColset(
|
||||
Fts5Parse *pParse,
|
||||
Fts5ExprNearset *pNear,
|
||||
Fts5ExprNode *pExpr,
|
||||
Fts5Colset *pColset
|
||||
){
|
||||
Fts5Colset *pFree = pColset;
|
||||
if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
|
||||
pParse->rc = SQLITE_ERROR;
|
||||
pParse->zErr = sqlite3_mprintf(
|
||||
"fts5: column queries are not supported (detail=none)"
|
||||
);
|
||||
sqlite3_free(pColset);
|
||||
return;
|
||||
}
|
||||
|
||||
if( pNear ){
|
||||
pNear->pColset = pColset;
|
||||
}else{
|
||||
sqlite3_free(pColset);
|
||||
fts5ParseSetColset(pParse, pExpr, pColset, &pFree);
|
||||
}
|
||||
sqlite3_free(pFree);
|
||||
}
|
||||
|
||||
static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
|
||||
|
||||
@ -3158,23 +3158,23 @@ static int fts5IndexExtractCol(
|
||||
return p - (*pa);
|
||||
}
|
||||
|
||||
static int fts5IndexExtractColset (
|
||||
static void fts5IndexExtractColset(
|
||||
int *pRc,
|
||||
Fts5Colset *pColset, /* Colset to filter on */
|
||||
const u8 *pPos, int nPos, /* Position list */
|
||||
Fts5Buffer *pBuf /* Output buffer */
|
||||
){
|
||||
int rc = SQLITE_OK;
|
||||
int i;
|
||||
|
||||
fts5BufferZero(pBuf);
|
||||
for(i=0; i<pColset->nCol; i++){
|
||||
const u8 *pSub = pPos;
|
||||
int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
|
||||
if( nSub ){
|
||||
fts5BufferAppendBlob(&rc, pBuf, nSub, pSub);
|
||||
if( *pRc==SQLITE_OK ){
|
||||
int i;
|
||||
fts5BufferZero(pBuf);
|
||||
for(i=0; i<pColset->nCol; i++){
|
||||
const u8 *pSub = pPos;
|
||||
int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]);
|
||||
if( nSub ){
|
||||
fts5BufferAppendBlob(pRc, pBuf, nSub, pSub);
|
||||
}
|
||||
}
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -3298,8 +3298,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){
|
||||
pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]);
|
||||
pIter->base.pData = a;
|
||||
}else{
|
||||
int *pRc = &pIter->pIndex->rc;
|
||||
fts5BufferZero(&pIter->poslist);
|
||||
fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist);
|
||||
fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist);
|
||||
pIter->base.pData = pIter->poslist.p;
|
||||
pIter->base.nData = pIter->poslist.n;
|
||||
}
|
||||
|
||||
@ -89,32 +89,6 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); }
|
||||
%destructor expr { sqlite3Fts5ParseNodeFree($$); }
|
||||
%destructor exprlist { sqlite3Fts5ParseNodeFree($$); }
|
||||
|
||||
expr(A) ::= expr(X) AND expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
|
||||
}
|
||||
expr(A) ::= expr(X) OR expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
|
||||
}
|
||||
expr(A) ::= expr(X) NOT expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
|
||||
}
|
||||
|
||||
expr(A) ::= LP expr(X) RP. {A = X;}
|
||||
expr(A) ::= exprlist(X). {A = X;}
|
||||
|
||||
exprlist(A) ::= cnearset(X). {A = X;}
|
||||
exprlist(A) ::= exprlist(X) cnearset(Y). {
|
||||
A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
|
||||
}
|
||||
|
||||
cnearset(A) ::= nearset(X). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
|
||||
}
|
||||
cnearset(A) ::= colset(X) COLON nearset(Y). {
|
||||
sqlite3Fts5ParseSetColset(pParse, Y, X);
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
|
||||
}
|
||||
|
||||
%type colset {Fts5Colset*}
|
||||
%destructor colset { sqlite3_free($$); }
|
||||
%type colsetlist {Fts5Colset*}
|
||||
@ -138,6 +112,37 @@ colsetlist(A) ::= STRING(X). {
|
||||
A = sqlite3Fts5ParseColset(pParse, 0, &X);
|
||||
}
|
||||
|
||||
expr(A) ::= expr(X) AND expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0);
|
||||
}
|
||||
expr(A) ::= expr(X) OR expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0);
|
||||
}
|
||||
expr(A) ::= expr(X) NOT expr(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0);
|
||||
}
|
||||
|
||||
expr(A) ::= colset(X) COLON LP expr(Y) RP. {
|
||||
sqlite3Fts5ParseSetColset(pParse, Y, X);
|
||||
A = Y;
|
||||
}
|
||||
expr(A) ::= LP expr(X) RP. {A = X;}
|
||||
expr(A) ::= exprlist(X). {A = X;}
|
||||
|
||||
exprlist(A) ::= cnearset(X). {A = X;}
|
||||
exprlist(A) ::= exprlist(X) cnearset(Y). {
|
||||
A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y);
|
||||
}
|
||||
|
||||
cnearset(A) ::= nearset(X). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X);
|
||||
}
|
||||
cnearset(A) ::= colset(X) COLON nearset(Y). {
|
||||
A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y);
|
||||
sqlite3Fts5ParseSetColset(pParse, A, X);
|
||||
}
|
||||
|
||||
|
||||
%type nearset {Fts5ExprNearset*}
|
||||
%type nearphrases {Fts5ExprNearset*}
|
||||
%destructor nearset { sqlite3Fts5ParseNearsetFree($$); }
|
||||
|
||||
@ -44,13 +44,25 @@ foreach_detail_mode $::testprefix {
|
||||
9 "-c : a" {1 2 4}
|
||||
10 "-\"c\" : a" {1 2 4}
|
||||
} {
|
||||
breakpoint
|
||||
do_execsql_test 1.$tn {
|
||||
SELECT rowid FROM t1($q)
|
||||
} $res
|
||||
}
|
||||
|
||||
|
||||
foreach {tn q res} {
|
||||
0 {{a} : (a AND ":")} {}
|
||||
1 "{a b c} : (a AND d)" {2 3}
|
||||
2 "{a b c} : (a AND b:d)" {3}
|
||||
3 "{a b c} : (a AND d:d)" {}
|
||||
4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4}
|
||||
5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3}
|
||||
6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {}
|
||||
7 "{a b c} : (b:a AND c:b)" {2}
|
||||
} {
|
||||
do_execsql_test 2.$tn {
|
||||
SELECT rowid FROM t1($q)
|
||||
} $res
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -107,5 +107,29 @@ do_faultsim_test 3.3 -faults oom* -body {
|
||||
faultsim_test_result {0 {2 3}}
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Test OOM injection with nested colsets.
|
||||
#
|
||||
reset_db
|
||||
do_execsql_test 4.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d);
|
||||
INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1
|
||||
INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2
|
||||
INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3
|
||||
INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4
|
||||
}
|
||||
do_faultsim_test 4.1 -faults oom* -body {
|
||||
execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); }
|
||||
} -test {
|
||||
faultsim_test_result {0 2}
|
||||
}
|
||||
|
||||
do_faultsim_test 4.2 -faults oom* -body {
|
||||
execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') }
|
||||
} -test {
|
||||
faultsim_test_result {0 {2 3}}
|
||||
}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
|
||||
24
manifest
24
manifest
@ -1,5 +1,5 @@
|
||||
C Improved\s\\n\sand\s\\r\sescapes\sin\sthe\sext/misc/dbdump.c\sutility\sfunction.\s\sThe\nimplementation\sof\sdbdump.c\snow\smatches\sthe\simplementation\sin\sthe\sCLI.
|
||||
D 2017-04-12T17:38:24.004
|
||||
C Update\sfts5\sto\ssupport\s"<colset>\s:\s(\s<expr>\s)"\sfor\scolumn\sfiltering,\sas\swell\nas\s"<colset>\s:\sNEAR(...)"\sand\s"<colset>\s:\s<phrase>".
|
||||
D 2017-04-12T17:50:12.869
|
||||
F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
|
||||
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
|
||||
F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a
|
||||
@ -99,13 +99,13 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495
|
||||
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
|
||||
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
|
||||
F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
|
||||
F ext/fts5/fts5Int.h 88c1a9a01cd5cd7b9a3c95699d1f61cdb277b2fa087a07d3fc989b49970437d9
|
||||
F ext/fts5/fts5Int.h d514c81997be588c2b23b82c77f6449e20a43903c503066f13d3d0ed8746ccde
|
||||
F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
|
||||
F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
|
||||
F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
|
||||
F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c
|
||||
F ext/fts5/fts5_expr.c e5522f89e599e7e53c0014003b66480f676d2eff8b9cdb2c03c32d859b4b7cb3
|
||||
F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
|
||||
F ext/fts5/fts5_index.c a3a9ae1c6f8a3652eb45dd732a71cdc92b48df047af134da38cd0c425649e042
|
||||
F ext/fts5/fts5_index.c 551add2b7762a2857336747def3b9d4fa304df476e188fb323492e2fd851772b
|
||||
F ext/fts5/fts5_main.c 24cafdc44c06b9665d73c34b75966f6223e9299cc9fd10184c6bf888e3ff563f
|
||||
F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6
|
||||
F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a
|
||||
@ -115,7 +115,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8
|
||||
F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf
|
||||
F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738
|
||||
F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7
|
||||
F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae
|
||||
F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca
|
||||
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
|
||||
F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841
|
||||
F ext/fts5/test/fts5aa.test b3cb080db4851580705c5e261c9d4308edf030d5cbbc24b5d6668403b73f023b
|
||||
@ -136,7 +136,7 @@ F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3
|
||||
F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
|
||||
F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
|
||||
F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a
|
||||
F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4
|
||||
F ext/fts5/test/fts5colset.test 5d6cc35a7cdf39b2709a36620c3f398f6f8c88de39e1485f367e5294ffc7c153
|
||||
F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
|
||||
F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368
|
||||
F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
|
||||
@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080
|
||||
F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85
|
||||
F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08
|
||||
F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea
|
||||
F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca
|
||||
F ext/fts5/test/fts5faultB.test 8a7e4c89c663e599e88fa1b9f2b8dcf4b85a338bd250e1a94341773b1e9ebfcc
|
||||
F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a
|
||||
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
|
||||
F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9
|
||||
@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 7aae5c0f99aa2fda85654242cfc9e23a0f981d9ce4ab17610d619cd208540b3d
|
||||
R 8af8a0ed024db0518fa75f74af560a36
|
||||
U drh
|
||||
Z 8e506040e94c8b0ee35c49d49bd8bac0
|
||||
P f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262
|
||||
R 0002da416f4fbc93cd5fcd1248781e80
|
||||
U dan
|
||||
Z 28e57cabc052d8f141d26e29a9907759
|
||||
|
||||
@ -1 +1 @@
|
||||
f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262
|
||||
c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396
|
||||
Loading…
Reference in New Issue
Block a user