98 lines
2.8 KiB
Plaintext
98 lines
2.8 KiB
Plaintext
# 2014 Dec 20
|
|
#
|
|
# The author disclaims copyright to this source code. In place of
|
|
# a legal notice, here is a blessing:
|
|
#
|
|
# May you do good and not evil.
|
|
# May you find forgiveness for yourself and forgive others.
|
|
# May you share freely, never taking more than you give.
|
|
#
|
|
#***********************************************************************
|
|
#
|
|
# Tests focusing on the fts5 tokenizers
|
|
#
|
|
|
|
source [file join [file dirname [info script]] fts5_common.tcl]
|
|
set testprefix fts5tokenizer
|
|
|
|
|
|
do_execsql_test 1.0 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
|
|
DROP TABLE ft1;
|
|
}
|
|
do_execsql_test 1.1 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize='porter');
|
|
DROP TABLE ft1;
|
|
}
|
|
do_execsql_test 1.2 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = porter);
|
|
DROP TABLE ft1;
|
|
}
|
|
do_execsql_test 1.3 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter');
|
|
DROP TABLE ft1;
|
|
}
|
|
do_execsql_test 1.4 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize = 'porter ascii');
|
|
DROP TABLE ft1;
|
|
}
|
|
|
|
do_execsql_test 2.0 {
|
|
CREATE VIRTUAL TABLE ft1 USING fts5(x, tokenize=porter);
|
|
INSERT INTO ft1 VALUES('embedded databases');
|
|
}
|
|
do_execsql_test 2.1 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'embedding' } 1
|
|
do_execsql_test 2.2 { SELECT rowid FROM ft1 WHERE ft1 MATCH 'database' } 1
|
|
do_execsql_test 2.3 {
|
|
SELECT rowid FROM ft1 WHERE ft1 MATCH 'database embedding'
|
|
} 1
|
|
|
|
proc tcl_create {args} {
|
|
set ::targs $args
|
|
error "failed"
|
|
}
|
|
sqlite3_fts5_create_tokenizer db tcl tcl_create
|
|
|
|
foreach {tn directive expected} {
|
|
1 {tokenize='tcl a b c'} {a b c}
|
|
2 {tokenize='tcl ''d'' ''e'' ''f'''} {d e f}
|
|
3 {tokenize="tcl 'g' 'h' 'i'"} {g h i}
|
|
4 {tokenize = tcl} {}
|
|
} {
|
|
do_catchsql_test 3.$tn.1 "
|
|
CREATE VIRTUAL TABLE ft2 USING fts5(x, $directive)
|
|
" {1 {error in tokenizer constructor}}
|
|
do_test 3.$tn.2 { set ::targs } $expected
|
|
}
|
|
|
|
do_catchsql_test 4.1 {
|
|
CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize = tcl abc);
|
|
} {1 {parse error in "tokenize = tcl abc"}}
|
|
do_catchsql_test 4.2 {
|
|
CREATE VIRTUAL TABLE ft2 USING fts5(x y)
|
|
} {1 {unrecognized column option: y}}
|
|
|
|
#-------------------------------------------------------------------------
|
|
# Test the "separators" and "tokenchars" options a bit.
|
|
#
|
|
foreach {tn tokenizer} {1 ascii 2 unicode61} {
|
|
reset_db
|
|
set T "$tokenizer tokenchars ',.:' separators 'xyz'"
|
|
execsql "CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize = \"$T\")"
|
|
do_execsql_test 5.$tn.1 {
|
|
INSERT INTO t1 VALUES('abcxdefyghizjkl.mno,pqr:stu/vwx+yz');
|
|
}
|
|
foreach {tn2 token res} {
|
|
1 abc 1 2 def 1 3 ghi 1 4 jkl {}
|
|
5 mno {} 6 pqr {} 7 stu {} 8 jkl.mno,pqr:stu 1
|
|
9 vw 1
|
|
} {
|
|
do_execsql_test 5.$tn.2.$tn2 "
|
|
SELECT rowid FROM t1 WHERE t1 MATCH '\"$token\"'
|
|
" $res
|
|
}
|
|
}
|
|
|
|
finish_test
|
|
|