Test that the unicode61 tokenchars= and separators= options work with the fts3tokenize virtual table.

FossilOrigin-Name: ed24051462c09220ebfb82a347b4a2b5c820ef63

manifest

@@ -1,5 +1,5 @@
-C Clear\sthe\scurrent\stime\svalue\son\sprepared\sstatements\swhen\sthe\sprepared\sstatement\nis\sreset.
-D 2013-09-17T23:36:33.750
+C Test\sthat\sthe\sunicode61\stokenchars=\sand\sseparators=\soptions\swork\swith\sthe\sfts3tokenize\svirtual\stable.
+D 2013-09-18T11:16:32.742
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -559,7 +559,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584
 F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057
-F test/fts4unicode.test 26a0bd304dc3ecb6231f04cbd760af66f2b42102
+F test/fts4unicode.test ebd937061e1ce096240d2352feb424587f2187b9
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test cd25cf605c5a345d038dc7b84232204c6a901c84
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@@ -1114,7 +1114,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P e64590625774228b5602a66cfb087cf74e2b438b
-R 3e0fb0624f3c4343d466e2fb2e172195
-U drh
-Z 63cad000c8bfce48b583ef7cab5e683e
+P cebd6fc551d26aea0f28cd7d25338fe0b72aae88
+R 912d3a86c8cb4e429312147af7faaf76
+U dan
+Z 5b7e8c5d08bc668689f4627e9e0e9c87

manifest.uuid

@@ -1 +1 @@
-cebd6fc551d26aea0f28cd7d25338fe0b72aae88
+ed24051462c09220ebfb82a347b4a2b5c820ef63

test/fts4unicode.test

@@ -539,6 +539,17 @@ do_execsql_test 10.2 {
   two           * 1 1 two           0 1 1
 }
 
+# Test that the tokenchars and separators options work with the 
+# fts3tokenize table.
+#
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE ft1 USING fts3tokenize(
+    "unicode61", "tokenchars=@.", "separators=1234567890"
+  );
+  SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road';
+} {
+  berlin@street sydney.road
+}
 
 finish_test