mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix some problems with FTS3 and 3-way NEAR queries.

Browse Source 
FossilOrigin-Name: 23dc6fb5b28712d1ba18dc7ddb3f2ef3b469d611
This commit is contained in:
dan 2009-12-05 11:37:19 +00:00
parent 8e14c59611
commit 6e053f9c23
4 changed files with 89 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

111
ext/fts3/fts3.c
View File

@ -1064,6 +1064,54 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){
*ppPoslist = pEnd;
}
/*
** Value used to signify the end of an offset-list. This is safe because
** it is not possible to have a document with 2^31 terms.
*/
#define OFFSET_LIST_END 0x7fffffff
/*
** This function is used to help parse offset-lists. When this function is
** called, *pp may point to the start of the next varint in the offset-list
** being parsed, or it may point to 1 byte past the end of the offset-list
** (in which case **pp will be 0x00 or 0x01).
**
** If *pp points past the end of the current offset list, set *pi to
** OFFSET_LIST_END and return. Otherwise, read the next varint from *pp,
** increment the current value of *pi by the value read, and set *pp to
** point to the next value before returning.
*/
static void fts3ReadNextPos(
char **pp, /* IN/OUT: Pointer into offset-list buffer */
sqlite3_int64 *pi /* IN/OUT: Value read from offset-list */
){
if( **pp&0xFE ){
fts3GetDeltaVarint(pp, pi);
*pi -= 2;
}else{
*pi = OFFSET_LIST_END;
}
}
/*
** If parameter iCol is not 0, write an 0x01 byte followed by the value of
** iCol encoded as a varint to *pp.
**
** Set *pp to point to the byte just after the last byte written before
** returning (do not modify it if iCol==0). Return the total number of bytes
** written (0 if iCol==0).
*/
static int fts3PutColNumber(char **pp, int iCol){
int n = 0; /* Number of bytes written */
if( iCol ){
char *p = *pp; /* Output pointer */
n = 1 + sqlite3Fts3PutVarint(&p[1], iCol);
*p = 0x01;
*pp = &p[n];
}
return n;
}
/*
**
*/
@ -1086,42 +1134,37 @@ static void fts3PoslistMerge(
sqlite3_int64 i1 = 0;
sqlite3_int64 i2 = 0;
sqlite3_int64 iPrev = 0;
if( iCol1!=0 ){
int n;
*p++ = 0x01;
n = sqlite3Fts3PutVarint(p, iCol1);
p += n;
p1 += 1 + n;
p2 += 1 + n;
}
while( (*p1&0xFE) || (*p2&0xFE) ){
int n = fts3PutColNumber(&p, iCol1);
p1 += n;
p2 += n;
/* At this point, both p1 and p2 point to the start of offset-lists.
** An offset-list is a list of non-negative delta-encoded varints, each
** incremented by 2 before being stored. Each list is terminated by a 0
** or 1 value (0x00 or 0x01). The following block merges the two lists
** and writes the results to buffer p. p is left pointing to the byte
** after the list written. No terminator (0x00 or 0x01) is written to
** the output.
*/
fts3GetDeltaVarint(&p1, &i1);
fts3GetDeltaVarint(&p2, &i2);
do {
fts3PutDeltaVarint(&p, &iPrev, (i1<i2) ? i1 : i2);
iPrev -= 2;
if( i1==i2 ){
fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
fts3ReadNextPos(&p1, &i1);
fts3ReadNextPos(&p2, &i2);
}else if( i1<i2 ){
fts3GetDeltaVarint(&p1, &i1); i1 -= 2;
fts3ReadNextPos(&p1, &i1);
}else{
fts3GetDeltaVarint(&p2, &i2); i2 -= 2;
fts3ReadNextPos(&p2, &i2);
}
fts3PutDeltaVarint(&p, &iPrev, (i1<i2 ? i1 : i2) + 2); iPrev -= 2;
if( 0==(*p1&0xFE) ) i1 = 0x7FFFFFFF;
if( 0==(*p2&0xFE) ) i2 = 0x7FFFFFFF;
}
}while( i1!=OFFSET_LIST_END || i2!=OFFSET_LIST_END );
}else if( iCol1<iCol2 ){
if( iCol1 ){
int n = sqlite3Fts3PutVarint(&p[1], iCol1);
*p = 0x01;
p += n+1;
p1 += n+1;
}
p1 += fts3PutColNumber(&p, iCol1);
fts3ColumnlistCopy(&p, &p1);
}else{
if( iCol2 ){
int n = sqlite3Fts3PutVarint(&p[1], iCol2);
*p = 0x01;
p += n+1;
p2 += n+1;
}
p2 += fts3PutColNumber(&p, iCol2);
fts3ColumnlistCopy(&p, &p2);
}
}
@ -1188,13 +1231,14 @@ static int fts3PoslistPhraseMerge(
fts3PutDeltaVarint(&p, &iPrev, iSave+2); iPrev -= 2;
pSave = 0;
}
if( iPos2<=iPos1 ){
if( (!isSaveLeft && iPos2<=(iPos1+nToken)) || iPos2<=iPos1 ){
if( (*p2&0xFE)==0 ) break;
fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2;
}else{
if( (*p1&0xFE)==0 ) break;
fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2;
}
}
if( pSave && pp ){
p = pSave;
@ -1241,13 +1285,16 @@ static int fts3PoslistPhraseMerge(
return 1;
}
/*
** Merge two position-lists as required by the NEAR operator.
*/
static int fts3PoslistNearMerge(
char **pp, /* Output buffer */
char *aTmp, /* Temporary buffer space */
int nRight, /* Maximum difference in token positions */
int nLeft, /* Maximum difference in token positions */
char **pp1, /* Left input list */
char **pp2 /* Right input list */
char **pp1, /* IN/OUT: Left input list */
char **pp2 /* IN/OUT: Right input list */
){
char *p1 = *pp1;
char *p2 = *pp2;

26
manifest
View File

@ -1,8 +1,5 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
C Add\sthe\sSQLITE_4_BYTE_ALIGNED_MALLOC\scompile-time\soption\swhich\stells\ssome\nassert()\sstatements\sthat\sthe\sunderlying\ssystem\sonly\srequires\s4-byte\salignment\nof\s8-byte\sdata\sobjects\slike\sdouble\sor\sint64\sand\sthat\ssystem\smalloc()\sonly\nguarantees\s4-byte\salignment\sof\sreturned\spointers.
D 2009-12-04T23:10:13
C Fix\ssome\sproblems\swith\sFTS3\sand\s3-way\sNEAR\squeries.
D 2009-12-05T11:37:19
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in c5827ead754ab32b9585487177c93bb00b9497b3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@ -59,7 +56,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c 54ad3e8f0374148768ad65724cff89a8a5a241a7
F ext/fts3/fts3.c 7d344410bffdddf98c1af623993dd284e156a89f
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h cc716c74afa7da8e0f8ef39404f33ea62a823eb3
F ext/fts3/fts3_expr.c c18794a62c257d3456d3314c5a18e348ae0d84bd
@ -379,7 +376,7 @@ F test/fts2p.test 4b48c35c91e6a7dbf5ac8d1e5691823cc999aafb
F test/fts2q.test b2fbbe038b7a31a52a6079b215e71226d8c6a682
F test/fts2r.test b154c30b63061d8725e320fba1a39e2201cadd5e
F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
F test/fts3.test f4f380d3717493605270dfa3b0fa893ea0afb18d
F test/fts3.test ae0433b09b12def08105640e57693726c4949338
F test/fts3_common.tcl 31935839b1b601a5955572cb4e8060513c96bde0
F test/fts3aa.test 5327d4c1d9b6c61021696746cc9a6cdc5bf159c0
F test/fts3ab.test 09aeaa162aee6513d9ff336b6932211008b9d1f9
@ -779,14 +776,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P 04fc9c7661dd24d080f965e7eae9010a2d346e6a
R 012380c4025d7401ec31300c24d1a49d
U drh
Z 3a60f929898567f03b13abb7e9741c72
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.6 (GNU/Linux)
iD8DBQFLGZbXoxKgR168RlERAgzLAKCGX83GaoDdnZtQhPjMOrZGC9L8TwCcC1Yk
V482pOMKKDD0UFm2zbfBJVE=
=H8qI
-----END PGP SIGNATURE-----
P 08faee686eb2fabe0dde51231ee55880e78541e8
R b8a65915c3bbacc7a62e1d6f19ca7f03
U dan
Z bdcaf8016833561b09539543f7f27b88

2
manifest.uuid
View File

@ -1 +1 @@
08faee686eb2fabe0dde51231ee55880e78541e8
23dc6fb5b28712d1ba18dc7ddb3f2ef3b469d611

1
test/fts3.test
View File

@ -43,6 +43,7 @@ set ISQUICK 1
set EXCLUDE {
fts3.test
fts3malloc.test
fts3rnd.test
}
# Files to include in the test. If this list is empty then everything