Fix a problem preventing FTS5 from correctly passing the FTS5_TOKENIZE_PREFIX

flag to custom tokenizer implementations.

FossilOrigin-Name: 03c601344ed56b316bcc5fd02f6648b0009ba61b

ext/fts5/fts5_expr.c

@@ -1601,7 +1601,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
-    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
+    int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0);
     int n;
     sqlite3Fts5Dequote(z);
     n = (int)strlen(z);

ext/fts5/test/fts5synonym.test

@@ -152,7 +152,7 @@ foreach {tn expr res} {
   1  {abc}                           {"abc"}
   2  {one}                           {"one"|"i"|"1"}
   3  {3}                             {"3"|"iii"|"three"}
-  4  {3*}                            {"3"|"iii"|"three" *}
+  4  {3*}                            {"3" *}
 } {
   do_execsql_test 4.1.$tn {
     SELECT fts5_expr($expr, 'tokenize=tclnum')

ext/fts5/test/fts5tokenizer.test

@@ -262,5 +262,44 @@ do_execsql_test 8.3 {
   brown dog fox jump lazi over quick the
 }
 
+#-------------------------------------------------------------------------
+# Check that the FTS5_TOKENIZE_PREFIX flag is passed to the tokenizer
+# implementation.
+#
+reset_db
+proc tcl_create {args} { return "tcl_tokenize" }
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+set ::flags [list]
+proc tcl_tokenize {tflags text} {
+  lappend ::flags $tflags
+  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+    sqlite3_fts5_token $w $iStart $iEnd
+  }
+}
+
+do_execsql_test 9.1.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
+  INSERT INTO t1 VALUES('abc');
+  INSERT INTO t1 VALUES('xyz');
+} {}
+do_test 9.1.2 { set ::flags } {document document}
+
+set ::flags [list]
+do_execsql_test 9.2.1 { SELECT * FROM t1('abc'); } {abc}
+do_test 9.2.2 { set ::flags } {query}
+
+set ::flags [list]
+do_execsql_test 9.3.1 { SELECT * FROM t1('ab*'); } {abc}
+do_test 9.3.2 { set ::flags } {prefixquery}
+
+set ::flags [list]
+do_execsql_test 9.4.1 { SELECT * FROM t1('"abc xyz" *'); } {}
+do_test 9.4.2 { set ::flags } {prefixquery}
+
+set ::flags [list]
+do_execsql_test 9.5.1 { SELECT * FROM t1('"abc xyz*"'); } {}
+do_test 9.5.2 { set ::flags } {query}
+
+
 finish_test
 

manifest

@@ -1,5 +1,5 @@
-C Fix\san\suninitialized\svariable\sin\sthe\scommand-line\sshell.
-D 2017-01-22T16:34:35.577
+C Fix\sa\sproblem\spreventing\sFTS5\sfrom\scorrectly\spassing\sthe\sFTS5_TOKENIZE_PREFIX\nflag\sto\scustom\stokenizer\simplementations.
+D 2017-01-23T07:06:27.840
 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da
@@ -102,7 +102,7 @@ F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
 F ext/fts5/fts5_aux.c 67acf8d51723cf28ffc3828210ba662df4b8d267
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
-F ext/fts5/fts5_expr.c dc2cee9f56b1818b85df59304b8104a5dfb8ab60
+F ext/fts5/fts5_expr.c 33f0e8063ac7360bcd71c0ff0dcbacdc05fffe50
 F ext/fts5/fts5_hash.c 880998e596b60f078348d48732ca4ad9a90caad2
 F ext/fts5/fts5_index.c f67032a9a529ba52a545e6e3ab970764199c05d4
 F ext/fts5/fts5_main.c f85281445dcf8be32d18841c93a6f90fe27dbfe2
@@ -184,11 +184,11 @@ F ext/fts5/test/fts5rowid.test 16908a99d6efc9ba21081b4f2b86b3fc699839a6
 F ext/fts5/test/fts5simple.test 5da9b15ed534eb0be9f279d8a2bb2e24d30e4e38
 F ext/fts5/test/fts5simple2.test 00839031878f52391562594fdab0503e424ee071
 F ext/fts5/test/fts5simple3.test 25faa8cb8ad470c6f01f670bcc1317c19a89f091
-F ext/fts5/test/fts5synonym.test 6475d189c2e20d60795808f83e36bf9318708d48
+F ext/fts5/test/fts5synonym.test f964f9a98580dec06c995d4638ecb36eee60ae4a
 F ext/fts5/test/fts5synonym2.test aa4c43bd3b691ff80f658cb064f5ab40690e834e
 F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
-F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
+F ext/fts5/test/fts5tokenizer.test f64ab5fe48fa1317ff6315264bfe26e73cbdefde
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
 F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
@@ -1547,7 +1547,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 772dcb08f400f20d4dbfb74df39de78da24ee5fd
-R 2e31ebcfccc9f64daf4bbf7d84d2d498
-U drh
-Z 386761af3eaa734f6328de143132fe67
+P 06b8001ade62bc59e6ae20f761167a81d85a4272
+R 21442fb50b7964f67b800e2f23be99a7
+U dan
+Z 98433cafc41c67731096a39c749dcd19

manifest.uuid

@@ -1 +1 @@
-06b8001ade62bc59e6ae20f761167a81d85a4272
+03c601344ed56b316bcc5fd02f6648b0009ba61b