mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Further streamlining of fts5 prefix query code.

Browse Source 
FossilOrigin-Name: ca91bd8ac70a5b3fef127364f73ec675e58bb92c
This commit is contained in:
dan 2016-02-05 19:18:02 +00:00
parent 1cd0f05492
commit b80b8f482c
8 changed files with 101 additions and 52 deletions

1
ext/fts5/fts5Int.h

@ -282,6 +282,7 @@ struct Fts5PoslistWriter {
i64 iPrev;
};
int sqlite3Fts5PoslistWriterAppend(Fts5Buffer*, Fts5PoslistWriter*, i64);
void sqlite3Fts5PoslistSafeAppend(Fts5Buffer*, i64*, i64);
int sqlite3Fts5PoslistNext64(
const u8 *a, int n, /* Buffer containing poslist */

62
ext/fts5/fts5_buffer.c

@ -16,18 +16,20 @@
#include "fts5Int.h"
int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){
u32 nNew = pBuf->nSpace ? pBuf->nSpace*2 : 64;
u8 *pNew;
while( nNew<nByte ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
if( pBuf->nSpace<nByte ){
u32 nNew = pBuf->nSpace ? pBuf->nSpace : 64;
u8 *pNew;
while( nNew<nByte ){
nNew = nNew * 2;
}
pNew = sqlite3_realloc(pBuf->p, nNew);
if( pNew==0 ){
*pRc = SQLITE_NOMEM;
return 1;
}else{
pBuf->nSpace = nNew;
pBuf->p = pNew;
}
}
return 0;
}
@ -208,23 +210,37 @@ int sqlite3Fts5PoslistReaderInit(
return pIter->bEof;
}
/*
** Append position iPos to the position list being accumulated in buffer
** pBuf, which must be already be large enough to hold the new data.
** The previous position written to this list is *piPrev. *piPrev is set
** to iPos before returning.
*/
void sqlite3Fts5PoslistSafeAppend(
Fts5Buffer *pBuf,
i64 *piPrev,
i64 iPos
){
static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
if( (iPos & colmask) != (*piPrev & colmask) ){
pBuf->p[pBuf->n++] = 1;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
*piPrev = (iPos & colmask);
}
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2);
*piPrev = iPos;
}
int sqlite3Fts5PoslistWriterAppend(
Fts5Buffer *pBuf,
Fts5PoslistWriter *pWriter,
i64 iPos
){
static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32;
int rc = SQLITE_OK;
if( 0==fts5BufferGrow(&rc, pBuf, 5+5+5) ){
if( (iPos & colmask) != (pWriter->iPrev & colmask) ){
pBuf->p[pBuf->n++] = 1;
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32));
pWriter->iPrev = (iPos & colmask);
}
pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-pWriter->iPrev)+2);
pWriter->iPrev = iPos;
}
return rc;
int rc;
if( fts5BufferGrow(&rc, pBuf, 5+5+5) ) return rc;
sqlite3Fts5PoslistSafeAppend(pBuf, &pWriter->iPrev, iPos);
return SQLITE_OK;
}
void *sqlite3Fts5MallocZero(int *pRc, int nByte){

55
ext/fts5/fts5_index.c

@ -4704,6 +4704,7 @@ static void fts5MergePrefixLists(
Fts5Buffer *p1, /* First list to merge */
Fts5Buffer *p2 /* Second list to merge */
){
if( p2->n ){
if( p1->n==0 ){
fts5BufferSwap(p1, p2);
@ -4719,7 +4720,6 @@ static void fts5MergePrefixLists(
fts5DoclistIterInit(p2, &i2);
while( 1 ){
if( i1.iRowid<i2.iRowid ){
/* Copy entry from i1 */
fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid);
@ -4742,32 +4742,57 @@ static void fts5MergePrefixLists(
u8 *a1 = &i1.aPoslist[i1.nSize];
u8 *a2 = &i2.aPoslist[i2.nSize];
i64 iPrev = 0;
Fts5PoslistWriter writer;
memset(&writer, 0, sizeof(writer));
/* Merge the two position lists. */
fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid);
fts5BufferZero(&tmp);
sqlite3Fts5BufferSize(&p->rc, &tmp, i1.nPoslist + i2.nPoslist);
if( p->rc ) break;
sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
assert( iPos1>=0 && iPos2>=0 );
while( iPos1>=0 || iPos2>=0 ){
i64 iNew;
if( iPos2<0 || (iPos1>=0 && iPos1<iPos2) ){
iNew = iPos1;
sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
}else{
iNew = iPos2;
sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
if( iPos1==iPos2 ){
sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1,&iPos1);
if( iPos1<iPos2 ){
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
}else{
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
}
if( iPos1>=0 && iPos2>=0 ){
while( 1 ){
if( iPos1<iPos2 ){
if( iPos1!=iPrev ){
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
}
sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1);
if( iPos1<0 ) break;
}else{
if( iPos2!=iPrev ){
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
}
sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2);
if( iPos2<0 ) break;
}
}
if( iNew!=writer.iPrev || tmp.n==0 ){
p->rc = sqlite3Fts5PoslistWriterAppend(&tmp, &writer, iNew);
if( p->rc ) goto error_out;
}
if( iPos1>=0 ){
if( iPos1!=iPrev ){
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1);
}
fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1);
}
else if( iPos2>=0 ){
if( iPos2!=iPrev ){
sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2);
}
fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2);
}
/* WRITEPOSLISTSIZE */
@ -4936,7 +4961,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
int sqlite3Fts5IndexRollback(Fts5Index *p){
fts5CloseReader(p);
fts5IndexDiscardData(p);
assert( p->rc==SQLITE_OK );
/* assert( p->rc==SQLITE_OK ); */
return SQLITE_OK;
}

3
ext/fts5/fts5_main.c

@ -803,7 +803,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){
*/
static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
Fts5Cursor *pCsr = (Fts5Cursor*)pCursor;
int rc = SQLITE_OK;
int rc;
assert( (pCsr->ePlan<3)==
(pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE)
@ -820,6 +820,7 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
switch( pCsr->ePlan ){
case FTS5_PLAN_SPECIAL: {
CsrFlagSet(pCsr, FTS5CSR_EOF);
rc = SQLITE_OK;
break;
}

2
ext/fts5/test/fts5ad.test

@ -11,6 +11,8 @@
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
# More specifically, the focus is on testing prefix queries, both with and
# without prefix indexes.
#
source [file join [file dirname [info script]] fts5_common.tcl]

4
ext/fts5/test/fts5detail.test

@ -85,6 +85,10 @@ do_execsql_test 2.2 {
SELECT fts5_test_poslist(t2) FROM t2('aa');
} {0.0.0}
do_execsql_test 2.3 {
SELECT fts5_test_collist(t2) FROM t2('aa');
} {0.0}
set ::pc 0
#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
#exit

24
manifest

@ -1,5 +1,5 @@
C Make\ssure\sthe\s"bak.db"\sdatabase\sfile\sdoes\snot\sactually\sexist\sbefore\nstarting\sthe\s"quota.test"\stesting.
D 2016-02-05T17:49:43.402
C Further\sstreamlining\sof\sfts5\sprefix\squery\scode.
D 2016-02-05T19:18:02.811
F Makefile.in 027c1603f255390c43a426671055a31c0a65fdb4
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 72b7858f02017611c3ac1ddc965251017fed0845
@ -98,14 +98,14 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
F ext/fts5/fts5Int.h 2095cc38e776f19cc083ca90e00772ea0b204ab3
F ext/fts5/fts5Int.h efb02807dbe5a2bfb0ea592a472d1171cb553d53
F ext/fts5/fts5_aux.c b9bcce753ef5b451267b2232f0ca153ddeb3951d
F ext/fts5/fts5_buffer.c f6e0c6018ffc8e39fc0b333b5daa8b8d528ae6e4
F ext/fts5/fts5_buffer.c ad4bb545c866eea6add1b0f84c2c5029cd689092
F ext/fts5/fts5_config.c 0c384ebdd23fd055e2e50a93277b8d59da538238
F ext/fts5/fts5_expr.c ff5c451a6d025909639ac0f0d0af0cc595b50feb
F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
F ext/fts5/fts5_index.c 614c54544f844387ba6728c613a884a1aa151f06
F ext/fts5/fts5_main.c 7e8a5f27d504bc04e3de7f1cba8867f0332aee9d
F ext/fts5/fts5_index.c 9706959f2188d97d72df750519fee7baccef9964
F ext/fts5/fts5_main.c 6e23df904049edb498538bd3e22e53ec1ab6f4f7
F ext/fts5/fts5_storage.c 2a1f44deae090cd711f02cec0c2af8e660360d24
F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
@ -120,7 +120,7 @@ F ext/fts5/test/fts5_common.tcl 61ff0d1a29d98a91c4553b20b3f410d858834ee9
F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
@ -142,7 +142,7 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599
F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d
F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
@ -1426,7 +1426,7 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P a6b35273239669189f33402144f58328b133d182
R a53abbe37c3df437dd159edcdce10b4b
U drh
Z b28b558e8ce57b27d1473527e7eaafc5
P 1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
R 82ed27d217d52be944f2d4bee3dfe639
U dan
Z ad1bc1867d7f59ff72d365283a40efbf

2
manifest.uuid

@ -1 +1 @@
1cac6c45eeb7dbbcda0c76ba7d3a2ca77bc725cd
ca91bd8ac70a5b3fef127364f73ec675e58bb92c