diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 730a2fb5e3..5a2008882f 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -263,7 +263,7 @@ struct Fts5PhraseIter { ** See xPhraseFirstColumn above. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ void *(*xUserData)(Fts5Context*); diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 4aa578559b..df25fd9ffa 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -773,6 +773,8 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); +int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index a2c6320719..80387a90fd 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3145,3 +3145,29 @@ int sqlite3Fts5ExprPhraseCollist( return rc; } + +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 6e86ca5951..7ddc7b6fcf 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -2323,13 +2323,24 @@ static int fts5ApiPhraseFirstColumn( return rc; } +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} + static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); static const Fts5ExtensionApi sFts5Api = { - 2, /* iVersion */ + 3, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -2349,6 +2360,9 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + 0, + 0 }; /* diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c index fb4bea8e9e..a9390adc9e 100644 --- a/ext/fts5/fts5_tcl.c +++ b/ext/fts5/fts5_tcl.c @@ -244,6 +244,8 @@ static int SQLITE_TCLAPI xF5tApi( { "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */ { "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */ { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */ + + { "xQueryToken", 2, "IPHRASE ITERM" }, /* 19 */ { 0, 0, 0} }; @@ -500,6 +502,22 @@ static int SQLITE_TCLAPI xF5tApi( break; } + CASE(18, "xQueryToken") { + const char *pTerm = 0; + int nTerm = 0; + int iPhrase = 0; + int iTerm = 0; + + if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR; + rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm)); + } + + break; + } + default: assert( 0 ); break; diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 3fd5f17e7b..2c9a4ba5a3 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -157,5 +157,48 @@ do_execsql_test 3.1.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} do_execsql_test 3.1.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} +#------------------------------------------------------------------------- +# +reset_db +sqlite3_fts5_register_origintext db +proc querytoken {cmd iPhrase iToken} { + set txt [$cmd xQueryToken $iPhrase $iToken] + string map [list "\0" "."] $txt +} +sqlite3_fts5_create_function db querytoken querytoken + +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, tokenize='origintext unicode61', tokendata=1 + ); + INSERT INTO ft VALUES('one two three four'); +} + +do_execsql_test 3.1 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO') +} {1 two.TwO} +do_execsql_test 3.2 { + SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE') +} {1 one} +do_execsql_test 3.3 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') +} {1 two.TWO} +do_execsql_test 3.4 { + SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') +} {1 three.ThreE} + +do_catchsql_test 3.5 { + SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} +do_catchsql_test 3.6 { + SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} +do_catchsql_test 3.7 { + SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') +} {1 SQLITE_RANGE} + + + + finish_test diff --git a/manifest b/manifest index baff022d13..4a539764cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sdeclarations\sfor\snew\sAPI\sfunctions. -D 2023-11-08T14:55:20.854 +C Add\snew\sfts5\sAPI\sxQueryToken(). +D 2023-11-13T14:29:12.382 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -87,17 +87,17 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 -F ext/fts5/fts5.h 68256dd94eaba3e874762a63578922fd62a5ca87c76a28f7636effee4d9bd781 -F ext/fts5/fts5Int.h a21eb1cf036ac9eb943e45ed307762901ea86f0159bf0848baa2079a112ddc2f +F ext/fts5/fts5.h e27cdb10e38d87cb041dcb56cef97addf7d902aeab07e84e7102f5fc65d3357c +F ext/fts5/fts5Int.h 19b198459a2791415919428d44ebf4c830b59b2da6f27f8faaffe39a876b7ecf F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf -F ext/fts5/fts5_expr.c fd091d0558fda2517602ed5886ec615ce3e1bd76fb0bb0e5d1aa85ba8db287a8 +F ext/fts5/fts5_expr.c 69c81af515ce1cedccf093c7c76f8b3b4f24bafbfb1d03a431af9f5c69a81834 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 F ext/fts5/fts5_index.c 145723e22ffee28dbe2a24933e74ad998d32419223c0ddb8506a1f0c39b952c4 -F ext/fts5/fts5_main.c a07ed863b8bd9e6fefb62db2fd40a3518eb30a5f7dcfda5be915dd2db45efa2f +F ext/fts5/fts5_main.c ddac85dbd28167af81f64e568bfe9020bf9708d650de207d1465ed19938316d1 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d -F ext/fts5/fts5_tcl.c 0d2bb0ff7bf6ee136015be118167f0bd956ddd05a8f02c68bd34299b50648f9f +F ext/fts5/fts5_tcl.c 71641a0c5693c64acfad9d10e64475ec92d9f464d06ba7fd350552de373586d8 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b F ext/fts5/fts5_tokenize.c 83cfcede3898001cab84432a36ce1503e3080cf9b1c682b022ec82e267ea4c13 @@ -188,7 +188,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca -F ext/fts5/test/fts5origintext.test 646df137f1aa5b3d7032374ebe82bfdbe88d9f825d73ce8d44bead480317a9c5 +F ext/fts5/test/fts5origintext.test 8296984d268d1d20f85c9de316f422ffb6ebc12020d3f8a0a18144d6ca7b347f F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15 @@ -2143,8 +2143,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3a869cf1f84b0e9bdcc4de53685430ab41eafacbba1ca7b87e727aa98811c6c5 -R 9e06a9e0b44056d476c02db915884372 +P b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 +R 177f59b3c48b3865f7f595e0ec855966 U dan -Z 6c0dbde9f9a0f05c4a27f8bd01924beb +Z ac818dc8fd8dc9007ff77eae486bea2e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index cf6088cc7c..f6858a05cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8a48cc18c94d15017f898c820fdd784efbaac20d7a45c4d97269333e8f2ec60 \ No newline at end of file +828566392b3ea8db603cb1ae5eccbc8ac035efaa284bc7c15ba89874f634aec9 \ No newline at end of file