e9b919d550
[forum:/forumpost/171bcc2bcd|Forum post 171bcc2bcd]. FossilOrigin-Name: d9f726ade6b258f8723f90d0b04a4682e885e30939eb29773913e4dfc8e85503
110 lines
2.4 KiB
Plaintext
110 lines
2.4 KiB
Plaintext
# 2023 Nov 03
|
|
#
|
|
# The author disclaims copyright to this source code. In place of
|
|
# a legal notice, here is a blessing:
|
|
#
|
|
# May you do good and not evil.
|
|
# May you find forgiveness for yourself and forgive others.
|
|
# May you share freely, never taking more than you give.
|
|
#
|
|
#***********************************************************************
|
|
#
|
|
# Tests focusing on the built-in fts5 tokenizers.
|
|
#
|
|
|
|
source [file join [file dirname [info script]] fts5_common.tcl]
|
|
set testprefix fts5tokenizer2
|
|
|
|
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
|
|
ifcapable !fts5 {
|
|
finish_test
|
|
return
|
|
}
|
|
|
|
sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
|
|
proc get_tst_tokenizer {args} {
|
|
return "tst_tokenizer"
|
|
}
|
|
proc tst_tokenizer {flags txt} {
|
|
set token ""
|
|
set lTok [list]
|
|
|
|
foreach c [split $txt {}] {
|
|
if {$token==""} {
|
|
append token $c
|
|
} else {
|
|
set t1 [string is upper $token]
|
|
set t2 [string is upper $c]
|
|
|
|
if {$t1!=$t2} {
|
|
lappend lTok $token
|
|
set token ""
|
|
}
|
|
append token $c
|
|
}
|
|
}
|
|
if {$token!=""} { lappend lTok $token }
|
|
|
|
set iOff 0
|
|
foreach t $lTok {
|
|
set n [string length $t]
|
|
sqlite3_fts5_token $t $iOff [expr $iOff+$n]
|
|
incr iOff $n
|
|
}
|
|
}
|
|
|
|
do_execsql_test 1.0 {
|
|
CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
|
|
}
|
|
|
|
do_execsql_test 1.1 {
|
|
INSERT INTO t1 VALUES('AAdontBBmess');
|
|
}
|
|
|
|
do_execsql_test 1.2 {
|
|
SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
|
|
} {AAdont>BB<mess}
|
|
|
|
do_execsql_test 1.3 {
|
|
SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
|
|
} {AAdont>BB<mess}
|
|
|
|
do_execsql_test 1.4 {
|
|
SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
|
|
} {>AA<dontBBmess}
|
|
|
|
do_execsql_test 1.5 {
|
|
SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
|
|
} {AA>dont<BBmess}
|
|
|
|
do_execsql_test 1.6 {
|
|
SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
|
|
} {AAdontBB>mess<}
|
|
|
|
do_execsql_test 1.7 {
|
|
SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
|
|
} {AAdont>BBmess<}
|
|
|
|
# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd
|
|
# Error handling of tokenize= arguments.
|
|
#
|
|
foreach {n tkz} {
|
|
1 {ascii none}
|
|
2 {unicode61 none}
|
|
3 {porter none}
|
|
4 {trigram none}
|
|
5 {ascii none 0}
|
|
6 {unicode61 none 0}
|
|
7 {porter none 0}
|
|
8 {trigram none 0}
|
|
} {
|
|
db eval {DROP TABLE IF EXISTS t2;}
|
|
do_catchsql_test 2.$n "
|
|
DROP TABLE IF EXISTS t2;
|
|
CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz');
|
|
" {1 {error in tokenizer constructor}}
|
|
}
|
|
|
|
|
|
finish_test
|