Remove some unused code from fts3. Add tests to fts3matchinfo.test.
FossilOrigin-Name: ae40b34cf7c24c9601bdfb5cbe5b20f05a376ea8
This commit is contained in:
dan
parent
1e66e40eb9
commit
ad3acbbfb3
@ -2255,8 +2255,8 @@ static int fts3PhraseSelect(
|
||||
for(ii=0; ii<pPhrase->nToken; ii++){
|
||||
Fts3PhraseToken *pTok; /* Token to find doclist for */
|
||||
int iTok; /* The token being queried this iteration */
|
||||
char *pList; /* Pointer to token doclist */
|
||||
int nList; /* Size of buffer at pList */
|
||||
char *pList = 0; /* Pointer to token doclist */
|
||||
int nList = 0; /* Size of buffer at pList */
|
||||
|
||||
/* Select a token to process. If this is an xFilter() call, then tokens
|
||||
** are processed in order from least to most costly. Otherwise, tokens
|
||||
@ -3275,7 +3275,10 @@ static void fts3MatchinfoFunc(
|
||||
Fts3Cursor *pCsr; /* Cursor handle passed through apVal[0] */
|
||||
assert( nVal==1 || nVal==2 );
|
||||
if( SQLITE_OK==fts3FunctionArg(pContext, "matchinfo", apVal[0], &pCsr) ){
|
||||
const char *zArg = (nVal>1 ? sqlite3_value_text(apVal[1]) : 0);
|
||||
const char *zArg = 0;
|
||||
if( nVal>1 ){
|
||||
zArg = (const char *)sqlite3_value_text(apVal[1]);
|
||||
}
|
||||
sqlite3Fts3Matchinfo(pContext, pCsr, zArg);
|
||||
}
|
||||
}
|
||||
|
||||
@ -291,8 +291,6 @@ int sqlite3Fts3SegReaderIterate(
|
||||
);
|
||||
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
|
||||
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
|
||||
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor*, u32*);
|
||||
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor*, u32*);
|
||||
int sqlite3Fts3ReadLock(Fts3Table *);
|
||||
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
|
||||
|
||||
|
||||
@ -819,6 +819,12 @@ static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
|
||||
** where X is the number of matches for phrase iPhrase is column iCol of all
|
||||
** rows of the table. Y is the number of rows for which column iCol contains
|
||||
** at least one instance of phrase iPhrase.
|
||||
**
|
||||
** If the phrase pExpr consists entirely of deferred tokens, then all X and
|
||||
** Y values are set to nDoc, where nDoc is the number of documents in the
|
||||
** file system. This is done because the full-text index doclist is required
|
||||
** to calculate these values properly, and the full-text index doclist is
|
||||
** not available for deferred tokens.
|
||||
*/
|
||||
static int fts3ExprGlobalHitsCb(
|
||||
Fts3Expr *pExpr, /* Phrase expression node */
|
||||
@ -903,13 +909,13 @@ static int fts3MatchinfoCheck(
|
||||
char cArg,
|
||||
char **pzErr
|
||||
){
|
||||
if( cArg==FTS3_MATCHINFO_NPHRASE
|
||||
|| cArg==FTS3_MATCHINFO_NCOL
|
||||
|| cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat
|
||||
|| cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat
|
||||
|| cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize
|
||||
|| cArg==FTS3_MATCHINFO_LCS
|
||||
|| cArg==FTS3_MATCHINFO_HITS
|
||||
if( (cArg==FTS3_MATCHINFO_NPHRASE)
|
||||
|| (cArg==FTS3_MATCHINFO_NCOL)
|
||||
|| (cArg==FTS3_MATCHINFO_NDOC && pTab->bHasStat)
|
||||
|| (cArg==FTS3_MATCHINFO_AVGLENGTH && pTab->bHasStat)
|
||||
|| (cArg==FTS3_MATCHINFO_LENGTH && pTab->bHasDocsize)
|
||||
|| (cArg==FTS3_MATCHINFO_LCS)
|
||||
|| (cArg==FTS3_MATCHINFO_HITS)
|
||||
){
|
||||
return SQLITE_OK;
|
||||
}
|
||||
@ -1182,7 +1188,7 @@ void sqlite3Fts3Snippet(
|
||||
** columns of the FTS3 table. Otherwise, only column iCol is considered.
|
||||
*/
|
||||
for(iRead=0; iRead<pTab->nColumn; iRead++){
|
||||
SnippetFragment sF;
|
||||
SnippetFragment sF = {0, 0, 0, 0};
|
||||
int iS;
|
||||
if( iCol>=0 && iRead!=iCol ) continue;
|
||||
|
||||
|
||||
@ -2488,75 +2488,6 @@ static void fts3DecodeIntArray(
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
** Fill in the document size auxiliary information for the matchinfo
|
||||
** structure. The auxiliary information is:
|
||||
**
|
||||
** N Total number of documents in the full-text index
|
||||
** a0 Average length of column 0 over the whole index
|
||||
** n0 Length of column 0 on the matching row
|
||||
** ...
|
||||
** aM Average length of column M over the whole index
|
||||
** nM Length of column M on the matching row
|
||||
**
|
||||
** The fts3MatchinfoDocsizeLocal() routine fills in the nX values.
|
||||
** The fts3MatchinfoDocsizeGlobal() routine fills in N and the aX values.
|
||||
*/
|
||||
int sqlite3Fts3MatchinfoDocsizeLocal(Fts3Cursor *pCur, u32 *a){
|
||||
const char *pBlob; /* The BLOB holding %_docsize info */
|
||||
int nBlob; /* Size of the BLOB */
|
||||
sqlite3_stmt *pStmt; /* Statement for reading and writing */
|
||||
int i, j; /* Loop counters */
|
||||
sqlite3_int64 x; /* Varint value */
|
||||
int rc; /* Result code from subfunctions */
|
||||
Fts3Table *p; /* The FTS table */
|
||||
|
||||
p = (Fts3Table*)pCur->base.pVtab;
|
||||
rc = fts3SqlStmt(p, SQL_SELECT_DOCSIZE, &pStmt, 0);
|
||||
if( rc ){
|
||||
return rc;
|
||||
}
|
||||
sqlite3_bind_int64(pStmt, 1, pCur->iPrevId);
|
||||
if( sqlite3_step(pStmt)==SQLITE_ROW ){
|
||||
nBlob = sqlite3_column_bytes(pStmt, 0);
|
||||
pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
|
||||
for(i=j=0; i<p->nColumn && j<nBlob; i++){
|
||||
j = sqlite3Fts3GetVarint(&pBlob[j], &x);
|
||||
a[2+i*2] = (u32)(x & 0xffffffff);
|
||||
}
|
||||
}
|
||||
sqlite3_reset(pStmt);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
int sqlite3Fts3MatchinfoDocsizeGlobal(Fts3Cursor *pCur, u32 *a){
|
||||
const char *pBlob; /* The BLOB holding %_stat info */
|
||||
int nBlob; /* Size of the BLOB */
|
||||
sqlite3_stmt *pStmt; /* Statement for reading and writing */
|
||||
int i, j; /* Loop counters */
|
||||
sqlite3_int64 x; /* Varint value */
|
||||
int nDoc; /* Number of documents */
|
||||
int rc; /* Result code from subfunctions */
|
||||
Fts3Table *p; /* The FTS table */
|
||||
|
||||
p = (Fts3Table*)pCur->base.pVtab;
|
||||
rc = fts3SqlStmt(p, SQL_SELECT_DOCTOTAL, &pStmt, 0);
|
||||
if( rc ){
|
||||
return rc;
|
||||
}
|
||||
if( sqlite3_step(pStmt)==SQLITE_ROW ){
|
||||
nBlob = sqlite3_column_bytes(pStmt, 0);
|
||||
pBlob = (const char*)sqlite3_column_blob(pStmt, 0);
|
||||
j = sqlite3Fts3GetVarint(pBlob, &x);
|
||||
a[0] = nDoc = (u32)(x & 0xffffffff);
|
||||
for(i=0; i<p->nColumn && j<nBlob; i++){
|
||||
j = sqlite3Fts3GetVarint(&pBlob[j], &x);
|
||||
a[1+i*2] = ((u32)(x & 0xffffffff) + nDoc/2)/nDoc;
|
||||
}
|
||||
}
|
||||
sqlite3_reset(pStmt);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/*
|
||||
** Insert the sizes (in tokens) for each column of the document
|
||||
** with docid equal to p->iPrevDocid. The sizes are encoded as
|
||||
|
||||
23
manifest
23
manifest
@ -1,5 +1,5 @@
|
||||
C Experimental\schanges\sto\sfts3\sfunction\smatchinfo().
|
||||
D 2010-11-23T19:16:48
|
||||
C Remove\ssome\sunused\scode\sfrom\sfts3.\sAdd\stests\sto\sfts3matchinfo.test.
|
||||
D 2010-11-24T11:51:56
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -61,19 +61,19 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
|
||||
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c 591b6fdb563dd00cd05b57ea27272f388e8156c8
|
||||
F ext/fts3/fts3.c 916091c7aeeefa8a7017865d97d10d13a5949b52
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3Int.h 0f7b9f189bcb0eed43a9489d2251b99acf06b833
|
||||
F ext/fts3/fts3Int.h 52c818623c60943bc4ac4a22d77b2e8f28395e78
|
||||
F ext/fts3/fts3_expr.c ee48b9278b8b2432a05a03320fbcacba151dbaa5
|
||||
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
|
||||
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
|
||||
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
|
||||
F ext/fts3/fts3_porter.c 8df6f6efcc4e9e31f8bf73a4007c2e9abca1dfba
|
||||
F ext/fts3/fts3_snippet.c aa0a76088024a693ab6a51d0d383e06f97e4fd8c
|
||||
F ext/fts3/fts3_snippet.c 967ca2d3201fd6555062c7e929bcc2b89ef8dcb8
|
||||
F ext/fts3/fts3_tokenizer.c 1301b0ee3ef414caae3257a702215925cc48cd9c
|
||||
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
|
||||
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
|
||||
F ext/fts3/fts3_write.c f5e7db79057ea46c1f3d15a4e2c8b658d306e37f
|
||||
F ext/fts3/fts3_write.c 9b2db92b815fdd50b5531eb6db912c71feca6a70
|
||||
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
|
||||
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
|
||||
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
|
||||
@ -446,7 +446,7 @@ F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
|
||||
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
|
||||
F test/fts3fault.test 81fd40ceb12f33f9d16c5637d0f8d95d4556c456
|
||||
F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b
|
||||
F test/fts3matchinfo.test 051bc3d09152998cedb7b9bdfd5b0730b5c0a939
|
||||
F test/fts3matchinfo.test cf3d0149f749abdb228c7bfe2350cf799b1809f4
|
||||
F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
|
||||
F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
|
||||
F test/fts3rnd.test 707533ce943f490443ce5e696236bb1675a37635
|
||||
@ -889,10 +889,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
|
||||
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
|
||||
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
|
||||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
P 3df3e79b56821201b4f5ecd23f94d485745c48c3
|
||||
R d2823d2eb839cb4c673b301711f75566
|
||||
T *branch * fts3-experimental
|
||||
T *sym-fts3-experimental *
|
||||
T -sym-trunk *
|
||||
P 9cf0f2b76bc68c168e3fa861b7235f384db21d38
|
||||
R 9d3e5478b917e1cb6e46d1d7c8efa28d
|
||||
U dan
|
||||
Z cb34d18554c4ac084411d81cdabfad29
|
||||
Z 2747212062c29415eaa46cf5d82fabf7
|
||||
|
||||
@ -1 +1 @@
|
||||
9cf0f2b76bc68c168e3fa861b7235f384db21d38
|
||||
ae40b34cf7c24c9601bdfb5cbe5b20f05a376ea8
|
||||
@ -66,4 +66,141 @@ do_execsql_test 3.1 {
|
||||
SELECT mtchinfo FROM t3;
|
||||
} {{Beside the lake, beneath the trees}}
|
||||
|
||||
|
||||
#--------------------------------------------------------------------------
|
||||
# Proc [do_matchinfo_test] is used to test the FTSX matchinfo() function.
|
||||
#
|
||||
# The first argument - $tn - is a test identifier. This may be either a
|
||||
# full identifier (i.e. "fts3matchinfo-1.1") or, if global var $testprefix
|
||||
# is set, just the numeric component (i.e. "1.1").
|
||||
#
|
||||
# The second argument is the name of an FTSX table. The third is the
|
||||
# full text of a WHERE/MATCH expression to query the table for
|
||||
# (i.e. "t1 MATCH 'abc'"). The final argument - $results - should be a
|
||||
# key-value list (serialized array) with matchinfo() format specifiers
|
||||
# as keys, and the results of executing the statement:
|
||||
#
|
||||
# SELECT matchinfo($tbl, '$key') FROM $tbl WHERE $expr
|
||||
#
|
||||
# For example:
|
||||
#
|
||||
# CREATE VIRTUAL TABLE t1 USING fts4;
|
||||
# INSERT INTO t1 VALUES('abc');
|
||||
# INSERT INTO t1 VALUES('def');
|
||||
# INSERT INTO t1 VALUES('abc abc');
|
||||
#
|
||||
# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
|
||||
# n {3 3}
|
||||
# p {1 1}
|
||||
# c {1 1}
|
||||
# x {{1 3 2} {2 3 2}}
|
||||
# }
|
||||
#
|
||||
# If the $results list contains keys mapped to "-" instead of a matchinfo()
|
||||
# result, then this command computes the expected results based on other
|
||||
# mappings to test the matchinfo() function. For example, the command above
|
||||
# could be changed to:
|
||||
#
|
||||
# do_matchinfo_test 1.1 t1 "t1 MATCH 'abc'" {
|
||||
# n {3 3} p {1 1} c {1 1} x {{1 3 2} {2 3 2}}
|
||||
# pcx -
|
||||
# }
|
||||
#
|
||||
# And this command would compute the expected results for matchinfo(t1, 'pcx')
|
||||
# based on the results of matchinfo(t1, 'p'), matchinfo(t1, 'c') and
|
||||
# matchinfo(t1, 'x') in order to test 'pcx'.
|
||||
#
|
||||
proc do_matchinfo_test {tn tbl expr results} {
|
||||
|
||||
foreach {fmt res} $results {
|
||||
if {$res == "-"} continue
|
||||
set resarray($fmt) $res
|
||||
}
|
||||
|
||||
set nRow 0
|
||||
foreach {fmt res} [array get resarray] {
|
||||
if {[llength $res]>$nRow} { set nRow [llength $res] }
|
||||
}
|
||||
|
||||
# Construct expected results for any formats for which the caller
|
||||
# supplied result is "-".
|
||||
#
|
||||
foreach {fmt res} $results {
|
||||
if {$res == "-"} {
|
||||
set res [list]
|
||||
for {set iRow 0} {$iRow<$nRow} {incr iRow} {
|
||||
set rowres [list]
|
||||
foreach c [split $fmt ""] {
|
||||
set rowres [concat $rowres [lindex $resarray($c) $iRow]]
|
||||
}
|
||||
lappend res $rowres
|
||||
}
|
||||
set resarray($fmt) $res
|
||||
}
|
||||
}
|
||||
|
||||
# Test each matchinfo() request individually.
|
||||
#
|
||||
foreach {fmt res} [array get resarray] {
|
||||
set sql "SELECT mit(matchinfo($tbl, '$fmt')) FROM $tbl WHERE $expr"
|
||||
do_execsql_test $tn.$fmt $sql [normalize2 $res]
|
||||
}
|
||||
|
||||
# Test them all executed together (multiple invocations of matchinfo()).
|
||||
#
|
||||
set exprlist [list]
|
||||
foreach {format res} [array get resarray] {
|
||||
lappend exprlist "mit(matchinfo($tbl, '$format'))"
|
||||
}
|
||||
set allres [list]
|
||||
for {set iRow 0} {$iRow<$nRow} {incr iRow} {
|
||||
foreach {format res} [array get resarray] {
|
||||
lappend allres [lindex $res $iRow]
|
||||
}
|
||||
}
|
||||
set sql "SELECT [join $exprlist ,] FROM $tbl WHERE $expr"
|
||||
do_execsql_test $tn.multi $sql [normalize2 $allres]
|
||||
}
|
||||
proc normalize2 {list_of_lists} {
|
||||
set res [list]
|
||||
foreach elem $list_of_lists {
|
||||
lappend res [list {*}$elem]
|
||||
}
|
||||
return $res
|
||||
}
|
||||
|
||||
|
||||
do_execsql_test 4.1.0 {
|
||||
CREATE VIRTUAL TABLE t4 USING fts4(x, y);
|
||||
INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
|
||||
INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
|
||||
}
|
||||
do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
|
||||
p {3 3}
|
||||
c {2 2}
|
||||
x {
|
||||
{1 1 1 0 1 1 1 1 1 0 1 1 1 1 1 0 1 1}
|
||||
{0 1 1 1 1 1 0 1 1 1 1 1 0 1 1 1 1 1}
|
||||
}
|
||||
n {2 2}
|
||||
l {{5 5} {5 5}}
|
||||
a {{5 5} {5 5}}
|
||||
|
||||
xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
|
||||
}
|
||||
do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
|
||||
p {1 1}
|
||||
c {2 2}
|
||||
x {
|
||||
{0 1 1 1 1 1}
|
||||
{1 1 1 0 1 1}
|
||||
}
|
||||
n {2 2}
|
||||
l {{5 5} {5 5}}
|
||||
a {{5 5} {5 5}}
|
||||
|
||||
xxxxxxxxxxxxxxxxxx - pcx - xpc - ccc - pppxpcpcx - laxnpc -
|
||||
}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
Loading…
Reference in New Issue
Block a user