Update the highlight() and snippet() functions to use locales when available.

FossilOrigin-Name: 569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085

ext/fts5/fts5_aux.c

@@ -226,6 +226,27 @@ static int fts5HighlightCb(
   return rc;
 }
 
+/*
+** Use xTokenizeSetLocale() to configure the tokenizer to use the locale
+** associated with column iCol of the current row.
+*/
+static int fts5ConfigureTokenizer(
+  const Fts5ExtensionApi *pApi,
+  Fts5Context *pFts,
+  int iCol
+){
+  int rc = SQLITE_OK;
+  const char *zLocale = 0;
+  int nLocale = 0;
+
+  rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
+  if( rc==SQLITE_OK ){
+    rc = pApi->xTokenizeSetLocale(pFts, zLocale, nLocale);
+  }
+  return rc;
+}
+
+
 /*
 ** Implementation of highlight() function.
 */
@@ -260,6 +281,9 @@ static void fts5HighlightFunction(
       rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
     }
 
+    if( rc==SQLITE_OK ){
+      rc = fts5ConfigureTokenizer(pApi, pFts, iCol);
+    }
     if( rc==SQLITE_OK ){
       rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }
@@ -465,6 +489,8 @@ static void fts5SnippetFunction(
       sFinder.nFirst = 0;
       rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
       if( rc!=SQLITE_OK ) break;
+      rc = fts5ConfigureTokenizer(pApi, pFts, i);
+      if( rc!=SQLITE_OK ) break;
       rc = pApi->xTokenize(pFts, 
           sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
       );
@@ -541,6 +567,9 @@ static void fts5SnippetFunction(
       rc = fts5CInstIterNext(&ctx.iter);
     }
 
+    if( rc==SQLITE_OK ){
+      rc = fts5ConfigureTokenizer(pApi, pFts, iBestCol);
+    }
     if( rc==SQLITE_OK ){
       rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }

ext/fts5/test/fts5locale.test

@@ -300,6 +300,40 @@ foreach {tn opt} {
   } {
     2 3 2 4
   }
+
+  do_execsql_test 5.$tn.4 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
+  } {
+    2 3
+  }
+
+  do_execsql_test 5.$tn.5 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
+  } {
+    2 3
+  }
+
+  do_execsql_test 5.$tn.6 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
+  } {
+    4 4
+  }
+
+  do_execsql_test 5.$tn.7 {
+    SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
+  } {
+    4 {which it hath been used to [have]}
+  }
+
+  do_execsql_test 5.$tn.8 {
+    SELECT rowid, 
+           highlight(y1, 0, '[', ']'),
+           snippet(y1, 0, '[', ']', '...', 10)
+    FROM y1('Liberties + Customs');
+  } {
+    3 {[Liberties and Customs]}
+      {[Liberties and Customs]}
+  }
 }
 
 

manifest

@@ -1,5 +1,5 @@
-C Fix\sxColumnText()\sand\sxColumnSize()\sAPIs.\sAdd\sxColumnLocale()\sand\sxTokenizeSetLocale().
-D 2024-07-29T20:31:17.644
+C Update\sthe\shighlight()\sand\ssnippet()\sfunctions\sto\suse\slocales\swhen\savailable.
+D 2024-07-30T15:55:51.884
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,7 +94,7 @@ F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a0
 F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
 F ext/fts5/fts5.h 7cd47e5dfe68d798e667caa76722374f0c909f2db05bb4d42b1ec5300d18e658
 F ext/fts5/fts5Int.h 330b1e2dad9ea9cccc9fa93817062fa21e89f00e7eac9a84be440f7e93bf7c3c
-F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1
+F ext/fts5/fts5_aux.c 652f839dc0c77431295f10b08f268631560bb5630e65fd701de7a58744428a82
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
 F ext/fts5/fts5_config.c 0c96490fbad746b3780174f38b2ee5e3d719f2f81ee6b58ca828772871e0f680
 F ext/fts5/fts5_expr.c c7336d5f9ecc0e2b014d700be2bec0ea383b0e82c494a7c5c4ac622327c2bfad
@@ -184,7 +184,7 @@ F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fd
 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1
 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad
 F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c
-F ext/fts5/test/fts5locale.test 14e46de0fbab3a01e16d8f61143260a7865cbe9d1759a564927635a1b08e9310
+F ext/fts5/test/fts5locale.test 7c332f882080a69ed4be9790d99ee49747ca44cf241d45f762298e17fea6e7a7
 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3
 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
@@ -2196,8 +2196,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989
-R 76149d45da5064496ccc1cbc5f33d6bf
+P 14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c
+R 8ced4a5b525d31957945a9ad2a3de4d9
 U dan
-Z 4addb1a2e7bbe8797df686b1934d0dc8
+Z 395467bf290b92744f082b81bcb8dffe
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c
+569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085