464 lines
17 KiB
Plaintext
464 lines
17 KiB
Plaintext
# 2010 January 07
|
|
#
|
|
# The author disclaims copyright to this source code. In place of
|
|
# a legal notice, here is a blessing:
|
|
#
|
|
# May you do good and not evil.
|
|
# May you find forgiveness for yourself and forgive others.
|
|
# May you share freely, never taking more than you give.
|
|
#
|
|
#*************************************************************************
|
|
#
|
|
# The tests in this file test the FTS3 auxillary functions offsets(),
|
|
# snippet() and matchinfo() work. At time of writing, running this file
|
|
# provides full coverage of fts3_snippet.c.
|
|
#
|
|
|
|
set testdir [file dirname $argv0]
|
|
source $testdir/tester.tcl
|
|
|
|
# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
|
|
ifcapable !fts3 { finish_test ; return }
|
|
source $testdir/fts3_common.tcl
|
|
|
|
set sqlite_fts3_enable_parentheses 1
|
|
set DO_MALLOC_TEST 0
|
|
|
|
# Transform the list $L to its "normal" form. So that it can be compared to
|
|
# another list with the same set of elements using [string compare].
|
|
#
|
|
proc normalize {L} {
|
|
set ret [list]
|
|
foreach l $L {lappend ret $l}
|
|
return $ret
|
|
}
|
|
|
|
proc do_offsets_test {name expr args} {
|
|
set result [list]
|
|
foreach a $args {
|
|
lappend result [normalize $a]
|
|
}
|
|
do_select_test $name {
|
|
SELECT offsets(ft) FROM ft WHERE ft MATCH $expr
|
|
} $result
|
|
}
|
|
|
|
# Document text used by a few tests. Contains the English names of all
|
|
# integers between 1 and 300.
|
|
#
|
|
set numbers [normalize {
|
|
one two three four five six seven eight nine ten eleven twelve thirteen
|
|
fourteen fifteen sixteen seventeen eighteen nineteen twenty twentyone
|
|
twentytwo twentythree twentyfour twentyfive twentysix twentyseven
|
|
twentyeight twentynine thirty thirtyone thirtytwo thirtythree thirtyfour
|
|
thirtyfive thirtysix thirtyseven thirtyeight thirtynine forty fortyone
|
|
fortytwo fortythree fortyfour fortyfive fortysix fortyseven fortyeight
|
|
fortynine fifty fiftyone fiftytwo fiftythree fiftyfour fiftyfive fiftysix
|
|
fiftyseven fiftyeight fiftynine sixty sixtyone sixtytwo sixtythree sixtyfour
|
|
sixtyfive sixtysix sixtyseven sixtyeight sixtynine seventy seventyone
|
|
seventytwo seventythree seventyfour seventyfive seventysix seventyseven
|
|
seventyeight seventynine eighty eightyone eightytwo eightythree eightyfour
|
|
eightyfive eightysix eightyseven eightyeight eightynine ninety ninetyone
|
|
ninetytwo ninetythree ninetyfour ninetyfive ninetysix ninetyseven
|
|
ninetyeight ninetynine onehundred onehundredone onehundredtwo
|
|
onehundredthree onehundredfour onehundredfive onehundredsix onehundredseven
|
|
onehundredeight onehundrednine onehundredten onehundredeleven
|
|
onehundredtwelve onehundredthirteen onehundredfourteen onehundredfifteen
|
|
onehundredsixteen onehundredseventeen onehundredeighteen onehundrednineteen
|
|
onehundredtwenty onehundredtwentyone onehundredtwentytwo
|
|
onehundredtwentythree onehundredtwentyfour onehundredtwentyfive
|
|
onehundredtwentysix onehundredtwentyseven onehundredtwentyeight
|
|
onehundredtwentynine onehundredthirty onehundredthirtyone
|
|
onehundredthirtytwo onehundredthirtythree onehundredthirtyfour
|
|
onehundredthirtyfive onehundredthirtysix onehundredthirtyseven
|
|
onehundredthirtyeight onehundredthirtynine onehundredforty
|
|
onehundredfortyone onehundredfortytwo onehundredfortythree
|
|
onehundredfortyfour onehundredfortyfive onehundredfortysix
|
|
onehundredfortyseven onehundredfortyeight onehundredfortynine
|
|
onehundredfifty onehundredfiftyone onehundredfiftytwo onehundredfiftythree
|
|
onehundredfiftyfour onehundredfiftyfive onehundredfiftysix
|
|
onehundredfiftyseven onehundredfiftyeight onehundredfiftynine
|
|
onehundredsixty onehundredsixtyone onehundredsixtytwo onehundredsixtythree
|
|
onehundredsixtyfour onehundredsixtyfive onehundredsixtysix
|
|
onehundredsixtyseven onehundredsixtyeight onehundredsixtynine
|
|
onehundredseventy onehundredseventyone onehundredseventytwo
|
|
onehundredseventythree onehundredseventyfour onehundredseventyfive
|
|
onehundredseventysix onehundredseventyseven onehundredseventyeight
|
|
onehundredseventynine onehundredeighty onehundredeightyone
|
|
onehundredeightytwo onehundredeightythree onehundredeightyfour
|
|
onehundredeightyfive onehundredeightysix onehundredeightyseven
|
|
onehundredeightyeight onehundredeightynine onehundredninety
|
|
onehundredninetyone onehundredninetytwo onehundredninetythree
|
|
onehundredninetyfour onehundredninetyfive onehundredninetysix
|
|
onehundredninetyseven onehundredninetyeight onehundredninetynine twohundred
|
|
twohundredone twohundredtwo twohundredthree twohundredfour twohundredfive
|
|
twohundredsix twohundredseven twohundredeight twohundrednine twohundredten
|
|
twohundredeleven twohundredtwelve twohundredthirteen twohundredfourteen
|
|
twohundredfifteen twohundredsixteen twohundredseventeen twohundredeighteen
|
|
twohundrednineteen twohundredtwenty twohundredtwentyone twohundredtwentytwo
|
|
twohundredtwentythree twohundredtwentyfour twohundredtwentyfive
|
|
twohundredtwentysix twohundredtwentyseven twohundredtwentyeight
|
|
twohundredtwentynine twohundredthirty twohundredthirtyone
|
|
twohundredthirtytwo twohundredthirtythree twohundredthirtyfour
|
|
twohundredthirtyfive twohundredthirtysix twohundredthirtyseven
|
|
twohundredthirtyeight twohundredthirtynine twohundredforty
|
|
twohundredfortyone twohundredfortytwo twohundredfortythree
|
|
twohundredfortyfour twohundredfortyfive twohundredfortysix
|
|
twohundredfortyseven twohundredfortyeight twohundredfortynine
|
|
twohundredfifty twohundredfiftyone twohundredfiftytwo twohundredfiftythree
|
|
twohundredfiftyfour twohundredfiftyfive twohundredfiftysix
|
|
twohundredfiftyseven twohundredfiftyeight twohundredfiftynine
|
|
twohundredsixty twohundredsixtyone twohundredsixtytwo twohundredsixtythree
|
|
twohundredsixtyfour twohundredsixtyfive twohundredsixtysix
|
|
twohundredsixtyseven twohundredsixtyeight twohundredsixtynine
|
|
twohundredseventy twohundredseventyone twohundredseventytwo
|
|
twohundredseventythree twohundredseventyfour twohundredseventyfive
|
|
twohundredseventysix twohundredseventyseven twohundredseventyeight
|
|
twohundredseventynine twohundredeighty twohundredeightyone
|
|
twohundredeightytwo twohundredeightythree twohundredeightyfour
|
|
twohundredeightyfive twohundredeightysix twohundredeightyseven
|
|
twohundredeightyeight twohundredeightynine twohundredninety
|
|
twohundredninetyone twohundredninetytwo twohundredninetythree
|
|
twohundredninetyfour twohundredninetyfive twohundredninetysix
|
|
twohundredninetyseven twohundredninetyeight twohundredninetynine
|
|
threehundred
|
|
}]
|
|
|
|
foreach {DO_MALLOC_TEST enc} {
|
|
0 utf8
|
|
1 utf8
|
|
1 utf16
|
|
} {
|
|
|
|
db close
|
|
forcedelete test.db
|
|
sqlite3 db test.db
|
|
sqlite3_db_config_lookaside db 0 0 0
|
|
db eval "PRAGMA encoding = \"$enc\""
|
|
|
|
# Set variable $T to the test name prefix for this iteration of the loop.
|
|
#
|
|
set T "fts3snippet-$enc"
|
|
|
|
##########################################################################
|
|
# Test the offset function.
|
|
#
|
|
do_test $T.1.1 {
|
|
execsql {
|
|
CREATE VIRTUAL TABLE ft USING fts3;
|
|
INSERT INTO ft VALUES('xxx xxx xxx xxx');
|
|
}
|
|
} {}
|
|
do_offsets_test $T.1.2 {xxx} {0 0 0 3 0 0 4 3 0 0 8 3 0 0 12 3}
|
|
do_offsets_test $T.1.3 {"xxx xxx"} {
|
|
0 0 0 3 0 0 4 3 0 1 4 3 0 0 8 3
|
|
0 1 8 3 0 1 12 3
|
|
}
|
|
do_offsets_test $T.1.4 {"xxx xxx" xxx} {
|
|
0 0 0 3 0 2 0 3 0 0 4 3 0 1 4 3
|
|
0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3
|
|
0 1 12 3 0 2 12 3
|
|
}
|
|
do_offsets_test $T.1.5 {xxx "xxx xxx"} {
|
|
0 0 0 3 0 1 0 3 0 0 4 3 0 1 4 3
|
|
0 2 4 3 0 0 8 3 0 1 8 3 0 2 8 3
|
|
0 0 12 3 0 2 12 3
|
|
}
|
|
|
|
do_test $T.2.1 {
|
|
set v1 [lrange $numbers 0 99]
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3(a, b);
|
|
INSERT INTO ft VALUES($v1, $numbers);
|
|
INSERT INTO ft VALUES($v1, NULL);
|
|
}
|
|
} {}
|
|
|
|
set off [string first "twohundred " $numbers]
|
|
do_offsets_test $T.2.1 {twohundred} [list 1 0 $off 10]
|
|
|
|
set off [string first "onehundred " $numbers]
|
|
do_offsets_test $T.2.2 {onehundred} \
|
|
[list 0 0 $off 10 1 0 $off 10] [list 0 0 $off 10]
|
|
|
|
# Test a corruption case:
|
|
execsql { UPDATE ft_content SET c1b = 'hello world' WHERE c1b = $numbers }
|
|
do_error_test $T.2.3 {
|
|
SELECT offsets(ft) FROM ft WHERE ft MATCH 'onehundred'
|
|
} {database disk image is malformed}
|
|
|
|
##########################################################################
|
|
# Test the snippet function.
|
|
#
|
|
proc do_snippet_test {name expr iCol nTok args} {
|
|
set res [list]
|
|
foreach a $args { lappend res [string trim $a] }
|
|
do_select_test $name {
|
|
SELECT snippet(ft,'{','}','...',$iCol,$nTok) FROM ft WHERE ft MATCH $expr
|
|
} $res
|
|
}
|
|
do_test $T.3.1 {
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3;
|
|
INSERT INTO ft VALUES('one two three four five six seven eight nine ten');
|
|
}
|
|
} {}
|
|
do_snippet_test $T.3.2 one 0 5 "{one} two three four five..."
|
|
do_snippet_test $T.3.3 two 0 5 "one {two} three four five..."
|
|
do_snippet_test $T.3.4 three 0 5 "one two {three} four five..."
|
|
do_snippet_test $T.3.5 four 0 5 "...two three {four} five six..."
|
|
do_snippet_test $T.3.6 five 0 5 "...three four {five} six seven..."
|
|
do_snippet_test $T.3.7 six 0 5 "...four five {six} seven eight..."
|
|
do_snippet_test $T.3.8 seven 0 5 "...five six {seven} eight nine..."
|
|
do_snippet_test $T.3.9 eight 0 5 "...six seven {eight} nine ten"
|
|
do_snippet_test $T.3.10 nine 0 5 "...six seven eight {nine} ten"
|
|
do_snippet_test $T.3.11 ten 0 5 "...six seven eight nine {ten}"
|
|
|
|
do_test $T.4.1 {
|
|
execsql {
|
|
INSERT INTO ft VALUES(
|
|
'one two three four five '
|
|
|| 'six seven eight nine ten '
|
|
|| 'eleven twelve thirteen fourteen fifteen '
|
|
|| 'sixteen seventeen eighteen nineteen twenty '
|
|
|| 'one two three four five '
|
|
|| 'six seven eight nine ten '
|
|
|| 'eleven twelve thirteen fourteen fifteen '
|
|
|| 'sixteen seventeen eighteen nineteen twenty'
|
|
);
|
|
}
|
|
} {}
|
|
|
|
do_snippet_test $T.4.2 {one nine} 0 5 {
|
|
{one} two three...eight {nine} ten
|
|
} {
|
|
{one} two three...eight {nine} ten...
|
|
}
|
|
|
|
do_snippet_test $T.4.3 {one nine} 0 -5 {
|
|
{one} two three four five...six seven eight {nine} ten
|
|
} {
|
|
{one} two three four five...seven eight {nine} ten eleven...
|
|
}
|
|
do_snippet_test $T.4.3 {one nineteen} 0 -5 {
|
|
...eighteen {nineteen} twenty {one} two...
|
|
}
|
|
do_snippet_test $T.4.4 {two nineteen} 0 -5 {
|
|
...eighteen {nineteen} twenty one {two}...
|
|
}
|
|
do_snippet_test $T.4.5 {three nineteen} 0 -5 {
|
|
...{nineteen} twenty one two {three}...
|
|
}
|
|
|
|
do_snippet_test $T.4.6 {four nineteen} 0 -5 {
|
|
...two three {four} five six...seventeen eighteen {nineteen} twenty one...
|
|
}
|
|
do_snippet_test $T.4.7 {four NEAR nineteen} 0 -5 {
|
|
...seventeen eighteen {nineteen} twenty one...two three {four} five six...
|
|
}
|
|
|
|
do_snippet_test $T.4.8 {four nineteen} 0 5 {
|
|
...three {four} five...eighteen {nineteen} twenty...
|
|
}
|
|
do_snippet_test $T.4.9 {four NEAR nineteen} 0 5 {
|
|
...eighteen {nineteen} twenty...three {four} five...
|
|
}
|
|
do_snippet_test $T.4.10 {four NEAR nineteen} 0 -5 {
|
|
...seventeen eighteen {nineteen} twenty one...two three {four} five six...
|
|
}
|
|
do_snippet_test $T.4.11 {four NOT (nineteen twentyone)} 0 5 {
|
|
...two three {four} five six...
|
|
} {
|
|
...two three {four} five six...
|
|
}
|
|
do_snippet_test $T.4.12 {four OR nineteen NEAR twentyone} 0 5 {
|
|
...two three {four} five six...
|
|
} {
|
|
...two three {four} five six...
|
|
}
|
|
|
|
do_test $T.5.1 {
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3(a, b, c);
|
|
INSERT INTO ft VALUES(
|
|
'one two three four five',
|
|
'four five six seven eight',
|
|
'seven eight nine ten eleven'
|
|
);
|
|
}
|
|
} {}
|
|
|
|
do_snippet_test $T.5.2 {five} -1 3 {...three four {five}}
|
|
do_snippet_test $T.5.3 {five} 0 3 {...three four {five}}
|
|
do_snippet_test $T.5.4 {five} 1 3 {four {five} six...}
|
|
do_snippet_test $T.5.5 {five} 2 3 {seven eight nine...}
|
|
|
|
do_test $T.5.6 {
|
|
execsql { UPDATE ft SET b = NULL }
|
|
} {}
|
|
|
|
do_snippet_test $T.5.7 {five} -1 3 {...three four {five}}
|
|
do_snippet_test $T.5.8 {five} 0 3 {...three four {five}}
|
|
do_snippet_test $T.5.9 {five} 1 3 {}
|
|
do_snippet_test $T.5.10 {five} 2 3 {seven eight nine...}
|
|
|
|
do_snippet_test $T.5.11 {one "seven eight nine"} -1 -3 {
|
|
{one} two three...{seven} {eight} {nine}...
|
|
}
|
|
|
|
do_test $T.6.1 {
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3(x);
|
|
INSERT INTO ft VALUES($numbers);
|
|
}
|
|
} {}
|
|
do_snippet_test $T.6.2 {
|
|
one fifty onehundred onehundredfifty twohundredfifty threehundred
|
|
} -1 4 {
|
|
{one}...{fifty}...{onehundred}...{onehundredfifty}...
|
|
}
|
|
do_snippet_test $T.6.3 {
|
|
one fifty onehundred onehundredfifty twohundredfifty threehundred
|
|
} -1 -4 {
|
|
{one} two three four...fortyeight fortynine {fifty} fiftyone...ninetyeight ninetynine {onehundred} onehundredone...onehundredfortyeight onehundredfortynine {onehundredfifty} onehundredfiftyone...
|
|
}
|
|
|
|
do_test $T.7.1 {
|
|
execsql {
|
|
BEGIN;
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3(x);
|
|
}
|
|
set testresults [list]
|
|
for {set i 1} {$i < 150} {incr i} {
|
|
set commas [string repeat , $i]
|
|
execsql {INSERT INTO ft VALUES('one' || $commas || 'two')}
|
|
lappend testresults "{one}$commas{two}"
|
|
}
|
|
execsql COMMIT
|
|
} {}
|
|
eval [list do_snippet_test $T.7.2 {one two} -1 3] $testresults
|
|
|
|
##########################################################################
|
|
# Test the matchinfo function.
|
|
#
|
|
proc mit {blob} {
|
|
set scan(littleEndian) i*
|
|
set scan(bigEndian) I*
|
|
binary scan $blob $scan($::tcl_platform(byteOrder)) r
|
|
return $r
|
|
}
|
|
db func mit mit
|
|
proc do_matchinfo_test {name expr args} {
|
|
set res [list]
|
|
foreach a $args { lappend res [normalize $a] }
|
|
do_select_test $name {
|
|
SELECT mit(matchinfo(ft)) FROM ft WHERE ft MATCH $expr
|
|
} $res
|
|
}
|
|
do_test $T.8.1 {
|
|
set ten {one two three four five six seven eight nine ten}
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3;
|
|
INSERT INTO ft VALUES($ten);
|
|
INSERT INTO ft VALUES($ten || ' ' || $ten);
|
|
}
|
|
} {}
|
|
|
|
do_matchinfo_test $T.8.2 "one" {1 1 1 3 2} {1 1 2 3 2}
|
|
do_matchinfo_test $T.8.3 "one NEAR/3 ten" {2 1 1 1 1 1 1 1}
|
|
do_matchinfo_test $T.8.4 "five NEAR/4 ten" \
|
|
{2 1 1 3 2 1 3 2} {2 1 2 3 2 2 3 2}
|
|
do_matchinfo_test $T.8.5 "six NEAR/3 ten NEAR/3 two" \
|
|
{3 1 1 1 1 1 1 1 1 1 1}
|
|
do_matchinfo_test $T.8.6 "five NEAR/4 ten NEAR/3 two" \
|
|
{3 1 2 2 1 1 1 1 1 1 1}
|
|
|
|
do_test $T.9.1 {
|
|
execsql {
|
|
DROP TABLE IF EXISTS ft;
|
|
CREATE VIRTUAL TABLE ft USING fts3(x, y);
|
|
}
|
|
foreach n {1 2 3} {
|
|
set v1 [lrange $numbers 0 [expr $n*100]]
|
|
set v2 [string trim [string repeat "$numbers " $n]]
|
|
set docid [expr $n * 1000000]
|
|
execsql { INSERT INTO ft(docid, x, y) VALUES($docid, $v1, $v2) }
|
|
}
|
|
} {}
|
|
do_matchinfo_test $T.9.2 {two*} \
|
|
{ 1 2 1 105 3 101 606 3} \
|
|
{ 1 2 3 105 3 202 606 3} \
|
|
{ 1 2 101 105 3 303 606 3}
|
|
|
|
do_matchinfo_test $T.9.4 {"one* two*"} \
|
|
{ 1 2 1 5 3 2 12 3} \
|
|
{ 1 2 2 5 3 4 12 3} \
|
|
{ 1 2 2 5 3 6 12 3}
|
|
|
|
do_matchinfo_test $T.9.5 {twohundredfifty} \
|
|
{ 1 2 0 1 1 1 6 3} \
|
|
{ 1 2 0 1 1 2 6 3} \
|
|
{ 1 2 1 1 1 3 6 3}
|
|
|
|
do_matchinfo_test $T.9.6 {"threehundred one"} \
|
|
{ 1 2 0 0 0 1 3 2} \
|
|
{ 1 2 0 0 0 2 3 2}
|
|
|
|
do_matchinfo_test $T.9.7 {one OR fivehundred} \
|
|
{ 2 2 1 3 3 1 6 3 0 0 0 0 0 0 } \
|
|
{ 2 2 1 3 3 2 6 3 0 0 0 0 0 0 } \
|
|
{ 2 2 1 3 3 3 6 3 0 0 0 0 0 0 }
|
|
|
|
do_matchinfo_test $T.9.8 {two OR "threehundred one"} \
|
|
{ 2 2 1 3 3 1 6 3 0 0 0 0 3 2 } \
|
|
{ 2 2 1 3 3 2 6 3 0 0 0 1 3 2 } \
|
|
{ 2 2 1 3 3 3 6 3 0 0 0 2 3 2 }
|
|
|
|
do_select_test $T.9.9 {
|
|
SELECT mit(matchinfo(ft)), mit(matchinfo(ft))
|
|
FROM ft WHERE ft MATCH 'two OR "threehundred one"'
|
|
} [normalize {
|
|
{2 2 1 3 3 1 6 3 0 0 0 0 3 2}
|
|
{2 2 1 3 3 1 6 3 0 0 0 0 3 2}
|
|
{2 2 1 3 3 2 6 3 0 0 0 1 3 2}
|
|
{2 2 1 3 3 2 6 3 0 0 0 1 3 2}
|
|
{2 2 1 3 3 3 6 3 0 0 0 2 3 2}
|
|
{2 2 1 3 3 3 6 3 0 0 0 2 3 2}
|
|
}]
|
|
|
|
# EVIDENCE-OF: R-40630-02268 If used within a SELECT that uses the
|
|
# "query by rowid" or "linear scan" strategies, then the snippet and
|
|
# offsets both return an empty string, and the matchinfo function
|
|
# returns a blob value zero bytes in size.
|
|
#
|
|
set r 1000000 ;# A rowid that exists in table ft
|
|
do_select_test $T.10.0 { SELECT rowid FROM ft WHERE rowid = $r } $r
|
|
do_select_test $T.10.1 {
|
|
SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft;
|
|
} {0 text 0 text 0 text}
|
|
do_select_test $T.10.2 {
|
|
SELECT length(offsets(ft)), typeof(offsets(ft)) FROM ft WHERE rowid = $r
|
|
} {0 text}
|
|
do_select_test $T.10.3 {
|
|
SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft;
|
|
} {0 text 0 text 0 text}
|
|
do_select_test $T.10.4 {
|
|
SELECT length(snippet(ft)), typeof(snippet(ft)) FROM ft WHERE rowid = $r;
|
|
} {0 text}
|
|
do_select_test $T.10.5 {
|
|
SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft;
|
|
} {0 blob 0 blob 0 blob}
|
|
do_select_test $T.10.6 {
|
|
SELECT length(matchinfo(ft)), typeof(matchinfo(ft)) FROM ft WHERE rowid = $r
|
|
} {0 blob}
|
|
}
|
|
|
|
set sqlite_fts3_enable_parentheses 0
|
|
finish_test
|