mirror of https://github.com/sqlite/sqlite
Add a test that calls fts2_tokenizer() with an argument set via C code. (CVS 4118)
FossilOrigin-Name: fbcf2d75cd2b88d175c122477aa483f0771870e5
This commit is contained in:
danielk1977
parent
f86643b32f
commit
46760820a1
|
|
@ -47,8 +47,8 @@
|
|||
it is registered as tokenizer <tokenizer-name> and a copy of it
|
||||
returned. If only one argument is passed, a pointer to the tokenizer
|
||||
implementation currently registered as <tokenizer-name> is returned,
|
||||
encoded as a blob. Or, if no such tokenizer exists, an SQL NULL value
|
||||
is returned.
|
||||
encoded as a blob. Or, if no such tokenizer exists, an SQL exception
|
||||
(error) is raised.
|
||||
|
||||
SECURITY: If the fts2 extension is used in an environment where potentially
|
||||
malicious users may execute arbitrary SQL (i.e. gears), they should be
|
||||
|
|
@ -86,39 +86,49 @@
|
|||
The following two code samples illustrate the way C code should invoke
|
||||
the fts2_tokenizer() scalar function:
|
||||
|
||||
int registerTokenizer(sqlite3 *db, char *zName, sqlite3_tokenizer_module *p){
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
|
||||
|
||||
rc = sqlite3_prepare_v2(db, zSql, &pStmt);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
|
||||
sqlite3_step(pStmt);
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
int queryTokenizer(sqlite3 *db, char *zName, sqlite3_tokenizer_module **pp){
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?)";
|
||||
|
||||
*pp = 0;
|
||||
rc = sqlite3_prepare_v2(db, zSql, &pStmt);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
|
||||
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
|
||||
int registerTokenizer(
|
||||
sqlite3 *db,
|
||||
char *zName,
|
||||
const sqlite3_tokenizer_module *p
|
||||
){
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
|
||||
|
||||
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
|
||||
sqlite3_step(pStmt);
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
int queryTokenizer(
|
||||
sqlite3 *db,
|
||||
char *zName,
|
||||
const sqlite3_tokenizer_module **pp
|
||||
){
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?)";
|
||||
|
||||
*pp = 0;
|
||||
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
|
||||
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
|
||||
}
|
||||
}
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
}
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -73,6 +73,7 @@ static void scalarFunc(
|
|||
#ifdef SQLITE_TEST
|
||||
|
||||
#include <tcl.h>
|
||||
#include <string.h>
|
||||
|
||||
/*
|
||||
** Implementation of a special SQL scalar function for testing tokenizers
|
||||
|
|
@ -187,6 +188,105 @@ finish:
|
|||
}
|
||||
Tcl_DecrRefCount(pRet);
|
||||
}
|
||||
|
||||
static
|
||||
int registerTokenizer(
|
||||
sqlite3 *db,
|
||||
char *zName,
|
||||
const sqlite3_tokenizer_module *p
|
||||
){
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
|
||||
|
||||
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
|
||||
sqlite3_step(pStmt);
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
static
|
||||
int queryTokenizer(
|
||||
sqlite3 *db,
|
||||
char *zName,
|
||||
const sqlite3_tokenizer_module **pp
|
||||
){
|
||||
int rc;
|
||||
sqlite3_stmt *pStmt;
|
||||
const char zSql[] = "SELECT fts2_tokenizer(?)";
|
||||
|
||||
*pp = 0;
|
||||
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
|
||||
if( rc!=SQLITE_OK ){
|
||||
return rc;
|
||||
}
|
||||
|
||||
sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
|
||||
if( SQLITE_ROW==sqlite3_step(pStmt) ){
|
||||
if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
|
||||
memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
|
||||
}
|
||||
}
|
||||
|
||||
return sqlite3_finalize(pStmt);
|
||||
}
|
||||
|
||||
void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
|
||||
|
||||
/*
|
||||
** Implementation of the scalar function fts2_tokenizer_internal_test().
|
||||
** This function is used for testing only, it is not included in the
|
||||
** build unless SQLITE_TEST is defined.
|
||||
**
|
||||
** The purpose of this is to test that the fts2_tokenizer() function
|
||||
** can be used as designed by the C-code in the queryTokenizer and
|
||||
** registerTokenizer() functions above. These two functions are repeated
|
||||
** in the README.tokenizer file as an example, so it is important to
|
||||
** test them.
|
||||
**
|
||||
** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
|
||||
** function with no arguments. An assert() will fail if a problem is
|
||||
** detected. i.e.:
|
||||
**
|
||||
** SELECT fts2_tokenizer_internal_test();
|
||||
**
|
||||
*/
|
||||
static void intTestFunc(
|
||||
sqlite3_context *context,
|
||||
int argc,
|
||||
sqlite3_value **argv
|
||||
){
|
||||
int rc;
|
||||
const sqlite3_tokenizer_module *p1;
|
||||
const sqlite3_tokenizer_module *p2;
|
||||
sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
|
||||
|
||||
/* Test the query function */
|
||||
sqlite3Fts2SimpleTokenizerModule(&p1);
|
||||
rc = queryTokenizer(db, "simple", &p2);
|
||||
assert( rc==SQLITE_OK );
|
||||
assert( p1==p2 );
|
||||
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
|
||||
assert( rc==SQLITE_ERROR );
|
||||
assert( p2==0 );
|
||||
assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
|
||||
|
||||
/* Test the storage function */
|
||||
rc = registerTokenizer(db, "nosuchtokenizer", p1);
|
||||
assert( rc==SQLITE_OK );
|
||||
rc = queryTokenizer(db, "nosuchtokenizer", &p2);
|
||||
assert( rc==SQLITE_OK );
|
||||
assert( p2==p1 );
|
||||
|
||||
sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
|
@ -213,25 +313,31 @@ int sqlite3Fts2InitHashTable(
|
|||
){
|
||||
int rc;
|
||||
void *p = (void *)pHash;
|
||||
void *pdb = (void *)db;
|
||||
const int any = SQLITE_ANY;
|
||||
char *zTest = 0;
|
||||
char *zTest2 = 0;
|
||||
|
||||
#ifdef SQLITE_TEST
|
||||
zTest = sqlite3_mprintf("%s_test", zName);
|
||||
if( !zTest ){
|
||||
return SQLITE_NOMEM;
|
||||
zTest2 = sqlite3_mprintf("%s_internal_test", zName);
|
||||
if( !zTest || !zTest2 ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}
|
||||
#endif
|
||||
|
||||
if( (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
|
||||
if( rc!=SQLITE_OK
|
||||
|| (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
|
||||
|| (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
|
||||
#ifdef SQLITE_TEST
|
||||
|| (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
|
||||
|| (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
|
||||
|| (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
|
||||
#endif
|
||||
);
|
||||
|
||||
sqlite3_free(zTest);
|
||||
sqlite3_free(zTest2);
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
|
|
|||
16
manifest
16
manifest
|
|
@ -1,5 +1,5 @@
|
|||
C Add\ssome\stests\sfor\sthe\sfts2\sicu\stokenizer.\s(CVS\s4117)
|
||||
D 2007-06-25T11:24:39
|
||||
C Add\sa\stest\sthat\scalls\sfts2_tokenizer()\swith\san\sargument\sset\svia\sC\scode.\s(CVS\s4118)
|
||||
D 2007-06-25T12:05:40
|
||||
F Makefile.in 7f7485a4cc039476a42e534b3f26ec90e2f9753e
|
||||
F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
|
||||
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
|
||||
|
|
@ -35,7 +35,7 @@ F ext/fts1/fulltext.c d935e600d87bc86b7d64f55c7520ea41d6034c5c
|
|||
F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
|
||||
F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d
|
||||
F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9
|
||||
F ext/fts2/README.tokenizers f358364121285c402d7b38fd44ba87b40903859b
|
||||
F ext/fts2/README.tokenizers 2ff290e0a130f6e7611f2e608cb3b5aaea721abc
|
||||
F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts2/fts2.c 841766f2f14d68e623404f9531d98afa0f7cbf05
|
||||
F ext/fts2/fts2.h 591916a822cfb6426518fdbf6069359119bc46eb
|
||||
|
|
@ -43,7 +43,7 @@ F ext/fts2/fts2_hash.c b3f22116d4ef0bc8f2da6e3fdc435c86d0951a9b
|
|||
F ext/fts2/fts2_hash.h e283308156018329f042816eb09334df714e105e
|
||||
F ext/fts2/fts2_icu.c 45b54d1e075020b35db20f69d829f95ca0651111
|
||||
F ext/fts2/fts2_porter.c 991a45463553c7318063fe7773368a6c0f39e35d
|
||||
F ext/fts2/fts2_tokenizer.c 40aa54fab0305b53f883fe4a394552f71dda29ee
|
||||
F ext/fts2/fts2_tokenizer.c 9200889237906d5607496dbd4c8b8c53a617ca71
|
||||
F ext/fts2/fts2_tokenizer.h 6d151c51382e8f6cf689c616bb697fe780478089
|
||||
F ext/fts2/fts2_tokenizer1.c 5c979fe8815f95396beb22b627571da895a025af
|
||||
F ext/fts2/mkfts2amal.tcl 2a9ec76b0760fe7f3669dca5bc0d60728bc1c977
|
||||
|
|
@ -256,7 +256,7 @@ F test/fts2l.test 4c53c89ce3919003765ff4fd8d98ecf724d97dd3
|
|||
F test/fts2m.test 4b30142ead6f3ed076e880a2a464064c5ad58c51
|
||||
F test/fts2n.test a70357e72742681eaebfdbe9007b87ff3b771638
|
||||
F test/fts2o.test 05ce2ac9111c29998418a584de02136a0ded471b
|
||||
F test/fts2token.test 5f349000669bc10bb392c87443ca5d3a587c9df0
|
||||
F test/fts2token.test d8070b241a15ff13592a9ae4a8b7c171af6f445a
|
||||
F test/func.test 605989453d1b42cec1d05c17aa232dc98e3e04e6
|
||||
F test/fuzz.test 62fc19dd36a427777fd671b569df07166548628a
|
||||
F test/fuzz2.test ea38692ce2da99ad79fe0be5eb1a452c1c4d37bb
|
||||
|
|
@ -515,7 +515,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
|
|||
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
|
||||
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
|
||||
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
|
||||
P 5a9eee86587219a68655d548864d129edec969ae
|
||||
R 61c2bfb00d5da35c1ad182696b085ade
|
||||
P b79ced3e0a26b0db13613073c847c2d2ba7e174e
|
||||
R bd43a4e803b49844e3492af252e992ad
|
||||
U danielk1977
|
||||
Z 97c1d47f7790b3efb6130173bff1e39a
|
||||
Z c0d7af5f5f203b9a840aeab00383f3fd
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
b79ced3e0a26b0db13613073c847c2d2ba7e174e
|
||||
fbcf2d75cd2b88d175c122477aa483f0771870e5
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
# of this script is testing the pluggable tokeniser feature of the
|
||||
# FTS2 module.
|
||||
#
|
||||
# $Id: fts2token.test,v 1.2 2007/06/25 11:24:39 danielk1977 Exp $
|
||||
# $Id: fts2token.test,v 1.3 2007/06/25 12:05:40 danielk1977 Exp $
|
||||
#
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
|
|
@ -167,4 +167,8 @@ ifcapable icu {
|
|||
do_icu_test fts2token-4.8 en_US $input $output
|
||||
}
|
||||
|
||||
do_test fts2token-internal {
|
||||
execsql { SELECT fts2_tokenizer_internal_test() }
|
||||
} {ok}
|
||||
|
||||
finish_test
|
||||
|
|
|
|||
Loading…
Reference in New Issue