diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 83e5238b48..1cd6877c9f 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -738,7 +738,7 @@ void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); -void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*); Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index e18691ad0b..f10d3d1043 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -1886,25 +1886,110 @@ Fts5Colset *sqlite3Fts5ParseColset( return pRet; } +/* +** If argument pOrig is NULL, or if (*pRc) is set to anything other than +** SQLITE_OK when this function is called, NULL is returned. +** +** Otherwise, a copy of (*pOrig) is made into memory obtained from +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned. +*/ +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ + Fts5Colset *pRet; + if( pOrig ){ + int nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); + if( pRet ){ + memcpy(pRet, pOrig, nByte); + } + }else{ + pRet = 0; + } + return pRet; +} + +/* +** Remove from colset pColset any columns that are not also in colset pMerge. +*/ +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){ + int iIn = 0; /* Next input in pColset */ + int iMerge = 0; /* Next input in pMerge */ + int iOut = 0; /* Next output slot in pColset */ + + while( iInnCol && iMergenCol ){ + int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge]; + if( iDiff==0 ){ + pColset->aiCol[iOut++] = pMerge->aiCol[iMerge]; + iMerge++; + iIn++; + }else if( iDiff>0 ){ + iMerge++; + }else{ + iIn++; + } + } + pColset->nCol = iOut; +} + +/* +** Recursively apply colset pColset to expression node pNode and all of +** its decendents. If (*ppFree) is not NULL, it contains a spare copy +** of pColset. This function may use the spare copy and set (*ppFree) to +** zero, or it may create copies of pColset using fts5CloneColset(). +*/ +static void fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pNode, + Fts5Colset *pColset, + Fts5Colset **ppFree +){ + if( pParse->rc==SQLITE_OK ){ + assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING + || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR + || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF + ); + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->pColset ){ + fts5MergeColset(pNear->pColset, pColset); + if( pNear->pColset->nCol==0 ){ + pNode->eType = FTS5_EOF; + pNode->xNext = 0; + } + }else if( *ppFree ){ + pNear->pColset = pColset; + *ppFree = 0; + }else{ + pNear->pColset = fts5CloneColset(&pParse->rc, pColset); + } + }else{ + int i; + assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 ); + for(i=0; inChild; i++){ + fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree); + } + } + } +} + +/* +** Apply colset pColset to expression node pExpr and all of its descendents. +*/ void sqlite3Fts5ParseSetColset( Fts5Parse *pParse, - Fts5ExprNearset *pNear, + Fts5ExprNode *pExpr, Fts5Colset *pColset ){ + Fts5Colset *pFree = pColset; if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ pParse->rc = SQLITE_ERROR; pParse->zErr = sqlite3_mprintf( "fts5: column queries are not supported (detail=none)" ); - sqlite3_free(pColset); - return; - } - - if( pNear ){ - pNear->pColset = pColset; }else{ - sqlite3_free(pColset); + fts5ParseSetColset(pParse, pExpr, pColset, &pFree); } + sqlite3_free(pFree); } static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 27e28a84ab..955489a35a 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -3158,23 +3158,23 @@ static int fts5IndexExtractCol( return p - (*pa); } -static int fts5IndexExtractColset ( +static void fts5IndexExtractColset( + int *pRc, Fts5Colset *pColset, /* Colset to filter on */ const u8 *pPos, int nPos, /* Position list */ Fts5Buffer *pBuf /* Output buffer */ ){ - int rc = SQLITE_OK; - int i; - - fts5BufferZero(pBuf); - for(i=0; inCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + if( *pRc==SQLITE_OK ){ + int i; + fts5BufferZero(pBuf); + for(i=0; inCol; i++){ + const u8 *pSub = pPos; + int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); + if( nSub ){ + fts5BufferAppendBlob(pRc, pBuf, nSub, pSub); + } } } - return rc; } /* @@ -3298,8 +3298,9 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); pIter->base.pData = a; }else{ + int *pRc = &pIter->pIndex->rc; fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); + fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, &pIter->poslist); pIter->base.pData = pIter->poslist.p; pIter->base.nData = pIter->poslist.n; } diff --git a/ext/fts5/fts5parse.y b/ext/fts5/fts5parse.y index 1cc4b88e18..1582909aa8 100644 --- a/ext/fts5/fts5parse.y +++ b/ext/fts5/fts5parse.y @@ -89,32 +89,6 @@ input ::= expr(X). { sqlite3Fts5ParseFinished(pParse, X); } %destructor expr { sqlite3Fts5ParseNodeFree($$); } %destructor exprlist { sqlite3Fts5ParseNodeFree($$); } -expr(A) ::= expr(X) AND expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); -} -expr(A) ::= expr(X) OR expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0); -} -expr(A) ::= expr(X) NOT expr(Y). { - A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0); -} - -expr(A) ::= LP expr(X) RP. {A = X;} -expr(A) ::= exprlist(X). {A = X;} - -exprlist(A) ::= cnearset(X). {A = X;} -exprlist(A) ::= exprlist(X) cnearset(Y). { - A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); -} - -cnearset(A) ::= nearset(X). { - A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); -} -cnearset(A) ::= colset(X) COLON nearset(Y). { - sqlite3Fts5ParseSetColset(pParse, Y, X); - A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); -} - %type colset {Fts5Colset*} %destructor colset { sqlite3_free($$); } %type colsetlist {Fts5Colset*} @@ -138,6 +112,37 @@ colsetlist(A) ::= STRING(X). { A = sqlite3Fts5ParseColset(pParse, 0, &X); } +expr(A) ::= expr(X) AND expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_AND, X, Y, 0); +} +expr(A) ::= expr(X) OR expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_OR, X, Y, 0); +} +expr(A) ::= expr(X) NOT expr(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_NOT, X, Y, 0); +} + +expr(A) ::= colset(X) COLON LP expr(Y) RP. { + sqlite3Fts5ParseSetColset(pParse, Y, X); + A = Y; +} +expr(A) ::= LP expr(X) RP. {A = X;} +expr(A) ::= exprlist(X). {A = X;} + +exprlist(A) ::= cnearset(X). {A = X;} +exprlist(A) ::= exprlist(X) cnearset(Y). { + A = sqlite3Fts5ParseImplicitAnd(pParse, X, Y); +} + +cnearset(A) ::= nearset(X). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, X); +} +cnearset(A) ::= colset(X) COLON nearset(Y). { + A = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, Y); + sqlite3Fts5ParseSetColset(pParse, A, X); +} + + %type nearset {Fts5ExprNearset*} %type nearphrases {Fts5ExprNearset*} %destructor nearset { sqlite3Fts5ParseNearsetFree($$); } diff --git a/ext/fts5/test/fts5colset.test b/ext/fts5/test/fts5colset.test index e3333241ee..21918a609d 100644 --- a/ext/fts5/test/fts5colset.test +++ b/ext/fts5/test/fts5colset.test @@ -44,13 +44,25 @@ foreach_detail_mode $::testprefix { 9 "-c : a" {1 2 4} 10 "-\"c\" : a" {1 2 4} } { - breakpoint do_execsql_test 1.$tn { SELECT rowid FROM t1($q) } $res } - + foreach {tn q res} { + 0 {{a} : (a AND ":")} {} + 1 "{a b c} : (a AND d)" {2 3} + 2 "{a b c} : (a AND b:d)" {3} + 3 "{a b c} : (a AND d:d)" {} + 4 "{b} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {3 4} + 5 "{a} : ( {b a} : ( {c b a} : ( {d b c a} : ( d OR c ) ) ) )" {2 3} + 6 "{a} : ( {b a} : ( {c b} : ( {d b c a} : ( d OR c ) ) ) )" {} + 7 "{a b c} : (b:a AND c:b)" {2} + } { + do_execsql_test 2.$tn { + SELECT rowid FROM t1($q) + } $res + } } diff --git a/ext/fts5/test/fts5faultB.test b/ext/fts5/test/fts5faultB.test index 2ce33f4bc4..e30b03a13e 100644 --- a/ext/fts5/test/fts5faultB.test +++ b/ext/fts5/test/fts5faultB.test @@ -107,5 +107,29 @@ do_faultsim_test 3.3 -faults oom* -body { faultsim_test_result {0 {2 3}} } +#------------------------------------------------------------------------- +# Test OOM injection with nested colsets. +# +reset_db +do_execsql_test 4.0 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, d); + INSERT INTO t1 VALUES('a', 'b', 'c', 'd'); -- 1 + INSERT INTO t1 VALUES('d', 'a', 'b', 'c'); -- 2 + INSERT INTO t1 VALUES('c', 'd', 'a', 'b'); -- 3 + INSERT INTO t1 VALUES('b', 'c', 'd', 'a'); -- 4 +} +do_faultsim_test 4.1 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (b:a AND c:b)'); } +} -test { + faultsim_test_result {0 2} +} + +do_faultsim_test 4.2 -faults oom* -body { + execsql { SELECT rowid FROM t1('{a b c} : (a AND d)') } +} -test { + faultsim_test_result {0 {2 3}} +} + + finish_test diff --git a/manifest b/manifest index ce75a7afa6..59900f1e3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s\\n\sand\s\\r\sescapes\sin\sthe\sext/misc/dbdump.c\sutility\sfunction.\s\sThe\nimplementation\sof\sdbdump.c\snow\smatches\sthe\simplementation\sin\sthe\sCLI. -D 2017-04-12T17:38:24.004 +C Update\sfts5\sto\ssupport\s"\s:\s(\s\s)"\sfor\scolumn\sfiltering,\sas\swell\nas\s"\s:\sNEAR(...)"\sand\s"\s:\s". +D 2017-04-12T17:50:12.869 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a4c0613a18663bda56d8cf76079ab6590a7c3602e54befb4bbdef76bcaa38b6a @@ -99,13 +99,13 @@ F ext/fts3/unicode/mkunicode.tcl ab0543a3b2399092ea2dd75df1bef333405b0d7f6b8c495 F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7 -F ext/fts5/fts5Int.h 88c1a9a01cd5cd7b9a3c95699d1f61cdb277b2fa087a07d3fc989b49970437d9 +F ext/fts5/fts5Int.h d514c81997be588c2b23b82c77f6449e20a43903c503066f13d3d0ed8746ccde F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857 -F ext/fts5/fts5_expr.c c6ecc2280162a3714d15dce2a8f2299f748b627c +F ext/fts5/fts5_expr.c e5522f89e599e7e53c0014003b66480f676d2eff8b9cdb2c03c32d859b4b7cb3 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2 -F ext/fts5/fts5_index.c a3a9ae1c6f8a3652eb45dd732a71cdc92b48df047af134da38cd0c425649e042 +F ext/fts5/fts5_index.c 551add2b7762a2857336747def3b9d4fa304df476e188fb323492e2fd851772b F ext/fts5/fts5_main.c 24cafdc44c06b9665d73c34b75966f6223e9299cc9fd10184c6bf888e3ff563f F ext/fts5/fts5_storage.c 7750986004f3f0c94619a85ecb5dd6cbef53e5e3853488e8a906c269d4d11db6 F ext/fts5/fts5_tcl.c 4a901f00c8553740dba63511603f5527d741c26a @@ -115,7 +115,7 @@ F ext/fts5/fts5_tokenize.c 2ce7b44183538ec46b7907726262ee43ffdd39a8 F ext/fts5/fts5_unicode2.c b450b209b157d598f7b9df9f837afb75a14c24bf F ext/fts5/fts5_varint.c a5aceacda04dafcbae725413d7a16818ecd65738 F ext/fts5/fts5_vocab.c e44fefa7f0c1db252998af071daf06a7147e17e7 -F ext/fts5/fts5parse.y e51b375403421b8b37428a89b095d00597129aae +F ext/fts5/fts5parse.y a070b538e08ae9e2177d15c337ed2a3464408f0f886e746307098f746efd94ca F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl b01c584144b5064f30e6c648145a2dd6bc440841 F ext/fts5/test/fts5aa.test b3cb080db4851580705c5e261c9d4308edf030d5cbbc24b5d6668403b73f023b @@ -136,7 +136,7 @@ F ext/fts5/test/fts5aux.test 5dd158a1e7869e27e9762a2a452b189c728d1be3 F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb F ext/fts5/test/fts5bigtok.test 017a9397b14e7598883a6328ead4a6539b42d59a -F ext/fts5/test/fts5colset.test 1cdf56e079316005aabda790059aee86f2222ee4 +F ext/fts5/test/fts5colset.test 5d6cc35a7cdf39b2709a36620c3f398f6f8c88de39e1485f367e5294ffc7c153 F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07 F ext/fts5/test/fts5config.test 7788b9c058074d640dfcdd81d97b6a9480000368 F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5 @@ -160,7 +160,7 @@ F ext/fts5/test/fts5fault7.test cb14ea3c1f42394f06f2284abc58eecee6ff8080 F ext/fts5/test/fts5fault8.test 6785af34bd1760de74e2824ea9c161965af78f85 F ext/fts5/test/fts5fault9.test e10e395428a9ea0596ebe752ff7123d16ab78e08 F ext/fts5/test/fts5faultA.test fa5d59c0ff62b7125cd14eee38ded1c46e15a7ea -F ext/fts5/test/fts5faultB.test 7f3bba790fa172073ac314f9b8ed197390b61eca +F ext/fts5/test/fts5faultB.test 8a7e4c89c663e599e88fa1b9f2b8dcf4b85a338bd250e1a94341773b1e9ebfcc F ext/fts5/test/fts5faultD.test cc5d1225556e356615e719c612e845d41bff7d5a F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741 F ext/fts5/test/fts5fuzz1.test bece4695fc169b61ab236ada7931c6e4942cbef9 @@ -1571,7 +1571,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7aae5c0f99aa2fda85654242cfc9e23a0f981d9ce4ab17610d619cd208540b3d -R 8af8a0ed024db0518fa75f74af560a36 -U drh -Z 8e506040e94c8b0ee35c49d49bd8bac0 +P f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262 +R 0002da416f4fbc93cd5fcd1248781e80 +U dan +Z 28e57cabc052d8f141d26e29a9907759 diff --git a/manifest.uuid b/manifest.uuid index ceed2cbabf..17ea2da68c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2643315bb41a71eebd79f5d671f9163187e299a52ff8a481186f1e8fa7e5262 \ No newline at end of file +c847543f8bb1376fef52bca72b4191162a32eb7e6c5f0cd1aa0ab116b3183396 \ No newline at end of file