Add experimental unicode-aware trigram tokenizer to fts5. And support for LIKE and GLOB optimizations for fts5 tables that use said tokenizer.
FossilOrigin-Name: 0d7810c1aea93c0a3da1ccc4911dbce8a1b6e1dbfe1ab7e800289a0c783b5985
This commit is contained in:
dan
parent
fad4dd0fd1
commit
33a99fad08
@ -184,6 +184,7 @@ struct Fts5Config {
|
||||
Fts5Tokenizer *pTok;
|
||||
fts5_tokenizer *pTokApi;
|
||||
int bLock; /* True when table is preparing statement */
|
||||
int ePattern; /* FTS_PATTERN_XXX constant */
|
||||
|
||||
/* Values loaded from the %_config table */
|
||||
int iCookie; /* Incremented when %_config is modified */
|
||||
@ -204,17 +205,19 @@ struct Fts5Config {
|
||||
};
|
||||
|
||||
/* Current expected value of %_config table 'version' field */
|
||||
#define FTS5_CURRENT_VERSION 4
|
||||
#define FTS5_CURRENT_VERSION 4
|
||||
|
||||
#define FTS5_CONTENT_NORMAL 0
|
||||
#define FTS5_CONTENT_NONE 1
|
||||
#define FTS5_CONTENT_EXTERNAL 2
|
||||
|
||||
#define FTS5_DETAIL_FULL 0
|
||||
#define FTS5_DETAIL_NONE 1
|
||||
#define FTS5_DETAIL_COLUMNS 2
|
||||
|
||||
#define FTS5_DETAIL_FULL 0
|
||||
#define FTS5_DETAIL_NONE 1
|
||||
#define FTS5_DETAIL_COLUMNS 2
|
||||
|
||||
#define FTS5_PATTERN_NONE 0
|
||||
#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */
|
||||
#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */
|
||||
|
||||
int sqlite3Fts5ConfigParse(
|
||||
Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char**
|
||||
@ -554,8 +557,7 @@ int sqlite3Fts5GetTokenizer(
|
||||
Fts5Global*,
|
||||
const char **azArg,
|
||||
int nArg,
|
||||
Fts5Tokenizer**,
|
||||
fts5_tokenizer**,
|
||||
Fts5Config*,
|
||||
char **pzErr
|
||||
);
|
||||
|
||||
@ -797,6 +799,10 @@ int sqlite3Fts5AuxInit(fts5_api*);
|
||||
*/
|
||||
|
||||
int sqlite3Fts5TokenizerInit(fts5_api*);
|
||||
int sqlite3Fts5TokenizerPattern(
|
||||
int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
|
||||
Fts5Tokenizer *pTok
|
||||
);
|
||||
/*
|
||||
** End of interface to code in fts5_tokenizer.c.
|
||||
**************************************************************************/
|
||||
|
||||
@ -325,7 +325,7 @@ static int fts5ConfigParseSpecial(
|
||||
rc = SQLITE_ERROR;
|
||||
}else{
|
||||
rc = sqlite3Fts5GetTokenizer(pGlobal,
|
||||
(const char**)azArg, (int)nArg, &pConfig->pTok, &pConfig->pTokApi,
|
||||
(const char**)azArg, (int)nArg, pConfig,
|
||||
pzErr
|
||||
);
|
||||
}
|
||||
@ -397,9 +397,7 @@ static int fts5ConfigParseSpecial(
|
||||
*/
|
||||
static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
|
||||
assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
|
||||
return sqlite3Fts5GetTokenizer(
|
||||
pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0
|
||||
);
|
||||
return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@ -284,6 +284,66 @@ int sqlite3Fts5ExprNew(
|
||||
return sParse.rc;
|
||||
}
|
||||
|
||||
int sqlite3Fts5ExprPattern(
|
||||
Fts5Config *pConfig, int iCol, const char *zText, Fts5Expr **pp
|
||||
){
|
||||
i64 nText = strlen(zText);
|
||||
char *zExpr = (char*)sqlite3_malloc64(nText*4 + 1);
|
||||
int rc = SQLITE_OK;
|
||||
|
||||
if( zExpr==0 ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
char aSpec[3];
|
||||
int iOut = 0;
|
||||
int i = 0;
|
||||
int iFirst = 0;
|
||||
|
||||
if( pConfig->ePattern==FTS5_PATTERN_LIKE ){
|
||||
aSpec[0] = '_';
|
||||
aSpec[1] = '%';
|
||||
aSpec[2] = 0;
|
||||
}else{
|
||||
aSpec[0] = '*';
|
||||
aSpec[1] = '?';
|
||||
aSpec[2] = '[';
|
||||
}
|
||||
|
||||
while( i<=nText ){
|
||||
if( i==nText
|
||||
|| zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2]
|
||||
){
|
||||
if( i-iFirst>=3 ){
|
||||
int jj;
|
||||
zExpr[iOut++] = '"';
|
||||
for(jj=iFirst; jj<i; jj++){
|
||||
zExpr[iOut++] = zText[jj];
|
||||
if( zText[jj]=='"' ) zExpr[iOut++] = '"';
|
||||
}
|
||||
zExpr[iOut++] = '"';
|
||||
zExpr[iOut++] = ' ';
|
||||
}
|
||||
if( zText[i]==aSpec[2] ){
|
||||
i += 2;
|
||||
if( zText[i-1]=='^' ) i++;
|
||||
while( i<nText && zText[i]!=']' ) i++;
|
||||
}
|
||||
iFirst = i+1;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if( iOut>0 ){
|
||||
zExpr[iOut] = '\0';
|
||||
rc = sqlite3Fts5ExprNew(pConfig, iCol, zExpr, pp, pConfig->pzErrmsg);
|
||||
}else{
|
||||
*pp = 0;
|
||||
}
|
||||
sqlite3_free(zExpr);
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Free the expression node object passed as the only argument.
|
||||
*/
|
||||
|
||||
@ -493,7 +493,9 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){
|
||||
**
|
||||
** Match against table column: "m"
|
||||
** Match against rank column: "r"
|
||||
** Match against other column: "<column-number>"
|
||||
** Match against other column: "M<column-number>"
|
||||
** LIKE against other column: "L<column-number>"
|
||||
** GLOB against other column: "G<column-number>"
|
||||
** Equality constraint against the rowid: "="
|
||||
** A < or <= against the rowid: "<"
|
||||
** A > or >= against the rowid: ">"
|
||||
@ -554,7 +556,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
|
||||
return SQLITE_ERROR;
|
||||
}
|
||||
|
||||
idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 6 + 1);
|
||||
idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1);
|
||||
if( idxStr==0 ) return SQLITE_NOMEM;
|
||||
pInfo->idxStr = idxStr;
|
||||
pInfo->needToFreeIdxStr = 1;
|
||||
@ -578,25 +580,29 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
|
||||
if( bSeenRank ) continue;
|
||||
idxStr[iIdxStr++] = 'r';
|
||||
bSeenRank = 1;
|
||||
}else{
|
||||
}else if( iCol>=0 ){
|
||||
bSeenMatch = 1;
|
||||
idxStr[iIdxStr++] = 'm';
|
||||
if( iCol<nCol ){
|
||||
sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
|
||||
idxStr += strlen(&idxStr[iIdxStr]);
|
||||
assert( idxStr[iIdxStr]=='\0' );
|
||||
}
|
||||
idxStr[iIdxStr++] = 'M';
|
||||
sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
|
||||
idxStr += strlen(&idxStr[iIdxStr]);
|
||||
assert( idxStr[iIdxStr]=='\0' );
|
||||
}
|
||||
pInfo->aConstraintUsage[i].argvIndex = ++iCons;
|
||||
pInfo->aConstraintUsage[i].omit = 1;
|
||||
}
|
||||
}
|
||||
else if( p->usable && bSeenEq==0
|
||||
&& p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0
|
||||
){
|
||||
idxStr[iIdxStr++] = '=';
|
||||
bSeenEq = 1;
|
||||
pInfo->aConstraintUsage[i].argvIndex = ++iCons;
|
||||
}else if( p->usable ){
|
||||
if( iCol>=0 && iCol<nCol && pConfig->ePattern==p->op ){
|
||||
assert( p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB );
|
||||
idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G';
|
||||
sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
|
||||
idxStr += strlen(&idxStr[iIdxStr]);
|
||||
pInfo->aConstraintUsage[i].argvIndex = ++iCons;
|
||||
assert( idxStr[iIdxStr]=='\0' );
|
||||
}else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){
|
||||
idxStr[iIdxStr++] = '=';
|
||||
bSeenEq = 1;
|
||||
pInfo->aConstraintUsage[i].argvIndex = ++iCons;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -1229,19 +1235,14 @@ static int fts5FilterMethod(
|
||||
case 'r':
|
||||
pRank = apVal[i];
|
||||
break;
|
||||
case 'm': {
|
||||
case 'M': {
|
||||
const char *zText = (const char*)sqlite3_value_text(apVal[i]);
|
||||
if( zText==0 ) zText = "";
|
||||
|
||||
if( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ){
|
||||
iCol = 0;
|
||||
do{
|
||||
iCol = iCol*10 + (idxStr[iIdxStr]-'0');
|
||||
iIdxStr++;
|
||||
}while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
|
||||
}else{
|
||||
iCol = pConfig->nCol;
|
||||
}
|
||||
iCol = 0;
|
||||
do{
|
||||
iCol = iCol*10 + (idxStr[iIdxStr]-'0');
|
||||
iIdxStr++;
|
||||
}while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
|
||||
|
||||
if( zText[0]=='*' ){
|
||||
/* The user has issued a query of the form "MATCH '*...'". This
|
||||
@ -1261,6 +1262,22 @@ static int fts5FilterMethod(
|
||||
|
||||
break;
|
||||
}
|
||||
case 'L':
|
||||
case 'G': {
|
||||
const char *zText = (const char*)sqlite3_value_text(apVal[i]);
|
||||
iCol = 0;
|
||||
do{
|
||||
iCol = iCol*10 + (idxStr[iIdxStr]-'0');
|
||||
iIdxStr++;
|
||||
}while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' );
|
||||
rc = sqlite3Fts5ExprPattern(pConfig, iCol, zText, &pExpr);
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr);
|
||||
pExpr = 0;
|
||||
}
|
||||
if( rc!=SQLITE_OK ) goto filter_out;
|
||||
break;
|
||||
}
|
||||
case '=':
|
||||
pRowidEq = apVal[i];
|
||||
break;
|
||||
@ -2672,8 +2689,7 @@ int sqlite3Fts5GetTokenizer(
|
||||
Fts5Global *pGlobal,
|
||||
const char **azArg,
|
||||
int nArg,
|
||||
Fts5Tokenizer **ppTok,
|
||||
fts5_tokenizer **ppTokApi,
|
||||
Fts5Config *pConfig,
|
||||
char **pzErr
|
||||
){
|
||||
Fts5TokenizerModule *pMod;
|
||||
@ -2685,16 +2701,22 @@ int sqlite3Fts5GetTokenizer(
|
||||
rc = SQLITE_ERROR;
|
||||
*pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
|
||||
}else{
|
||||
rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok);
|
||||
*ppTokApi = &pMod->x;
|
||||
if( rc!=SQLITE_OK && pzErr ){
|
||||
*pzErr = sqlite3_mprintf("error in tokenizer constructor");
|
||||
rc = pMod->x.xCreate(
|
||||
pMod->pUserData, &azArg[1], (nArg?nArg-1:0), &pConfig->pTok
|
||||
);
|
||||
pConfig->pTokApi = &pMod->x;
|
||||
if( rc!=SQLITE_OK ){
|
||||
if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor");
|
||||
}else{
|
||||
pConfig->ePattern = sqlite3Fts5TokenizerPattern(
|
||||
pMod->x.xCreate, pConfig->pTok
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if( rc!=SQLITE_OK ){
|
||||
*ppTokApi = 0;
|
||||
*ppTok = 0;
|
||||
pConfig->pTokApi = 0;
|
||||
pConfig->pTok = 0;
|
||||
}
|
||||
|
||||
return rc;
|
||||
|
||||
@ -1258,6 +1258,118 @@ static int fts5PorterTokenize(
|
||||
);
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
** Start of trigram implementation.
|
||||
*/
|
||||
|
||||
typedef struct TrigramTokenizer TrigramTokenizer;
|
||||
struct TrigramTokenizer {
|
||||
int bFold;
|
||||
};
|
||||
|
||||
/*
|
||||
** Free a trigram tokenizer.
|
||||
*/
|
||||
static void fts5TriDelete(Fts5Tokenizer *p){
|
||||
sqlite3_free(p);
|
||||
}
|
||||
|
||||
/*
|
||||
** Allocate a trigram tokenizer.
|
||||
*/
|
||||
static int fts5TriCreate(
|
||||
void *pCtx,
|
||||
const char **azArg,
|
||||
int nArg,
|
||||
Fts5Tokenizer **ppOut
|
||||
){
|
||||
int rc = SQLITE_OK;
|
||||
TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
|
||||
if( pNew==0 ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
int i;
|
||||
pNew->bFold = 1;
|
||||
for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
|
||||
const char *zArg = azArg[i+1];
|
||||
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
|
||||
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
|
||||
rc = SQLITE_ERROR;
|
||||
}else{
|
||||
pNew->bFold = (zArg[0]=='0');
|
||||
}
|
||||
}else{
|
||||
rc = SQLITE_ERROR;
|
||||
}
|
||||
}
|
||||
if( rc!=SQLITE_OK ){
|
||||
fts5TriDelete((Fts5Tokenizer*)pNew);
|
||||
pNew = 0;
|
||||
}
|
||||
}
|
||||
*ppOut = (Fts5Tokenizer*)pNew;
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Trigram tokenizer tokenize routine.
|
||||
*/
|
||||
static int fts5TriTokenize(
|
||||
Fts5Tokenizer *pTok,
|
||||
void *pCtx,
|
||||
int flags,
|
||||
const char *pText, int nText,
|
||||
int (*xToken)(void*, int, const char*, int, int, int)
|
||||
){
|
||||
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
|
||||
int rc = SQLITE_OK;
|
||||
char aBuf[32];
|
||||
const unsigned char *zIn = (const unsigned char*)pText;
|
||||
const unsigned char *zEof = &zIn[nText];
|
||||
u32 iCode;
|
||||
|
||||
while( 1 ){
|
||||
char *zOut = aBuf;
|
||||
int iStart = zIn - (const unsigned char*)pText;
|
||||
const unsigned char *zNext;
|
||||
|
||||
READ_UTF8(zIn, zEof, iCode);
|
||||
zNext = zIn;
|
||||
if( zIn<zEof ){
|
||||
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
|
||||
WRITE_UTF8(zOut, iCode);
|
||||
READ_UTF8(zIn, zEof, iCode);
|
||||
}else{
|
||||
break;
|
||||
}
|
||||
if( zIn<zEof ){
|
||||
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
|
||||
WRITE_UTF8(zOut, iCode);
|
||||
READ_UTF8(zIn, zEof, iCode);
|
||||
if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
|
||||
WRITE_UTF8(zOut, iCode);
|
||||
}else{
|
||||
break;
|
||||
}
|
||||
rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
|
||||
if( rc!=SQLITE_OK ) break;
|
||||
zIn = zNext;
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
int sqlite3Fts5TokenizerPattern(
|
||||
int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
|
||||
Fts5Tokenizer *pTok
|
||||
){
|
||||
if( xCreate==fts5TriCreate ){
|
||||
TrigramTokenizer *p = (TrigramTokenizer*)pTok;
|
||||
return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
|
||||
}
|
||||
return FTS5_PATTERN_NONE;
|
||||
}
|
||||
|
||||
/*
|
||||
** Register all built-in tokenizers with FTS5.
|
||||
*/
|
||||
@ -1269,6 +1381,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
|
||||
{ "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
|
||||
{ "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
|
||||
{ "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
|
||||
{ "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
|
||||
};
|
||||
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
|
||||
@ -31,7 +31,7 @@ do_eqp_test 1.1 {
|
||||
} {
|
||||
QUERY PLAN
|
||||
|--SCAN TABLE t1
|
||||
`--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m
|
||||
`--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:M1
|
||||
}
|
||||
|
||||
do_eqp_test 1.2 {
|
||||
@ -46,7 +46,7 @@ do_eqp_test 1.3 {
|
||||
SELECT * FROM f1 WHERE f1 MATCH ? ORDER BY ff
|
||||
} {
|
||||
QUERY PLAN
|
||||
|--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:m
|
||||
|--SCAN TABLE f1 VIRTUAL TABLE INDEX 0:M1
|
||||
`--USE TEMP B-TREE FOR ORDER BY
|
||||
}
|
||||
|
||||
|
||||
121
ext/fts5/test/fts5trigram.test
Normal file
121
ext/fts5/test/fts5trigram.test
Normal file
@ -0,0 +1,121 @@
|
||||
# 2020 September 30
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
#
|
||||
# Tests for the fts5 "trigram" tokenizer.
|
||||
#
|
||||
|
||||
source [file join [file dirname [info script]] fts5_common.tcl]
|
||||
ifcapable !fts5 { finish_test ; return }
|
||||
set ::testprefix fts5trigram
|
||||
|
||||
do_execsql_test 1.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram);
|
||||
INSERT INTO t1 VALUES('abcdefghijklm');
|
||||
INSERT INTO t1 VALUES('กรุงเทพมหานคร');
|
||||
}
|
||||
|
||||
foreach {tn s res} {
|
||||
1 abc "(abc)defghijklm"
|
||||
2 defgh "abc(defgh)ijklm"
|
||||
3 abcdefghijklm "(abcdefghijklm)"
|
||||
4 กรุ "(กรุ)งเทพมหานคร"
|
||||
5 งเทพมห "กรุ(งเทพมห)านคร"
|
||||
6 กรุงเทพมหานคร "(กรุงเทพมหานคร)"
|
||||
7 Abc "(abc)defghijklm"
|
||||
8 deFgh "abc(defgh)ijklm"
|
||||
9 aBcdefGhijKlm "(abcdefghijklm)"
|
||||
} {
|
||||
do_execsql_test 1.1.$tn {
|
||||
SELECT highlight(t1, 0, '(', ')') FROM t1($s)
|
||||
} $res
|
||||
}
|
||||
|
||||
do_execsql_test 1.2.0 {
|
||||
SELECT fts5_expr('ABCD', 'tokenize=trigram')
|
||||
} {{"abc" + "bcd"}}
|
||||
|
||||
do_execsql_test 1.2.1 {
|
||||
SELECT * FROM t1 WHERE y LIKE ? ESCAPE 'a'
|
||||
}
|
||||
|
||||
foreach {tn like res} {
|
||||
1 {%cDef%} 1
|
||||
2 {cDef%} {}
|
||||
3 {%f%} 1
|
||||
4 {%f_h%} 1
|
||||
5 {%f_g%} {}
|
||||
6 {abc%klm} 1
|
||||
7 {ABCDEFG%} 1
|
||||
8 {%รุงเ%} 2
|
||||
} {
|
||||
do_execsql_test 1.3.$tn {
|
||||
SELECT rowid FROM t1 WHERE y LIKE $like
|
||||
} $res
|
||||
}
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
reset_db
|
||||
do_execsql_test 2.0 {
|
||||
CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize="trigram case_sensitive 1");
|
||||
INSERT INTO t1 VALUES('abcdefghijklm');
|
||||
INSERT INTO t1 VALUES('กรุงเทพมหานคร');
|
||||
}
|
||||
|
||||
foreach {tn s res} {
|
||||
1 abc "(abc)defghijklm"
|
||||
2 defgh "abc(defgh)ijklm"
|
||||
3 abcdefghijklm "(abcdefghijklm)"
|
||||
4 กรุ "(กรุ)งเทพมหานคร"
|
||||
5 งเทพมห "กรุ(งเทพมห)านคร"
|
||||
6 กรุงเทพมหานคร "(กรุงเทพมหานคร)"
|
||||
7 Abc ""
|
||||
8 deFgh ""
|
||||
9 aBcdefGhijKlm ""
|
||||
} {
|
||||
do_execsql_test 2.1.$tn {
|
||||
SELECT highlight(t1, 0, '(', ')') FROM t1($s)
|
||||
} $res
|
||||
}
|
||||
foreach {tn like res} {
|
||||
1 {%cDef%} 1
|
||||
2 {cDef%} {}
|
||||
3 {%f%} 1
|
||||
4 {%f_h%} 1
|
||||
5 {%f_g%} {}
|
||||
6 {abc%klm} 1
|
||||
7 {ABCDEFG%} 1
|
||||
8 {%รุงเ%} 2
|
||||
} {
|
||||
do_execsql_test 2.2.$tn {
|
||||
SELECT rowid FROM t1 WHERE y LIKE $like
|
||||
} $res
|
||||
}
|
||||
foreach {tn like res} {
|
||||
1 {*cdef*} 1
|
||||
2 {cdef*} {}
|
||||
3 {*f*} 1
|
||||
4 {*f?h*} 1
|
||||
5 {*f?g*} {}
|
||||
6 {abc*klm} 1
|
||||
7 {abcdefg*} 1
|
||||
8 {*รุงเ*} 2
|
||||
9 {abc[d]efg*} 1
|
||||
10 {abc[]d]efg*} 1
|
||||
11 {abc[^]d]efg*} {}
|
||||
12 {abc[^]XYZ]efg*} 1
|
||||
} {
|
||||
do_execsql_test 2.3.$tn {
|
||||
SELECT rowid FROM t1 WHERE y GLOB $like
|
||||
} $res
|
||||
}
|
||||
|
||||
finish_test
|
||||
|
||||
29
manifest
29
manifest
@ -1,5 +1,5 @@
|
||||
C Improved\squery\soptimization\sfor\smulti-column\sindexes\swhere\sthe\ssecond\sor\nlater\scolumns\sare\sconstrained\sby\san\sIN\soperator\sand\sthe\searlier\sindex\scolumns\nlimit\sthe\ssearch\sto\sa\ssmall\snumber\sof\srows.\s\sUse\sthe\snew\sOP_SeekScan\sopcode\nwhich\sdoes\sscanning\sof\sthe\srelevant\srange\sof\sthe\sindex\sbut\sgives\sup\sand\nfalls\sback\sto\sdoing\sa\sseek\sif\sthe\snumber\sof\srows\sscanned\sgrows\sto\slarge,\nin\sorder\sto\sguard\sagainst\spathological\scases\swhere\sthe\sestimated\snumber\nof\srows\sto\sbe\sscanned\sis\sfar\stoo\ssmall.
|
||||
D 2020-09-30T18:06:51.119
|
||||
C Add\sexperimental\sunicode-aware\strigram\stokenizer\sto\sfts5.\sAnd\ssupport\sfor\sLIKE\sand\sGLOB\soptimizations\sfor\sfts5\stables\sthat\suse\ssaid\stokenizer.
|
||||
D 2020-09-30T20:35:37.594
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -112,19 +112,19 @@ F ext/fts3/unicode/mkunicode.tcl bf7fcaa6d68e6d38223467983785d054f1cff4d9e3905dd
|
||||
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
|
||||
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
|
||||
F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
|
||||
F ext/fts5/fts5Int.h ba835c165bb87650fc806008969799a7a1fbe3e221fd5a850dd044eb6a87b243
|
||||
F ext/fts5/fts5Int.h 928aed51dbeb4acc0d2e3ceeebb5f6918d64c9ad5c4e7634a238895abea40350
|
||||
F ext/fts5/fts5_aux.c dcc627d8b6e3fc773db528ff67b39955dab7b51628f9dba8e15849e5bedfd7fa
|
||||
F ext/fts5/fts5_buffer.c 5a5fe0159752c0fb0a5a93c722e9db2662822709490769d482b76a6dc8aaca70
|
||||
F ext/fts5/fts5_config.c b447948f35ad3354e8fe5e242e0a7e7b5b941555400b9404259944e3aa570037
|
||||
F ext/fts5/fts5_expr.c b7b28ed203a3140f2fc503507d2c614a6cf1bd2e8987497f8894abc3f1477ec4
|
||||
F ext/fts5/fts5_config.c be54f44fca491e96c6923a4b9a736f2da2b13811600eb6e38d1bcc91c4ea2e61
|
||||
F ext/fts5/fts5_expr.c e1f548de5e7f146e55e1a34c2745d1893510c0766baa55d33aa05c0643398534
|
||||
F ext/fts5/fts5_hash.c 15bffa734fbdca013b2289c6f8827a3b935ef14bd4dde5837d31a75434c00627
|
||||
F ext/fts5/fts5_index.c 255d3ce3fec28be11c533451e5b23bd79e71a13a1b120f3658b34fff6b097816
|
||||
F ext/fts5/fts5_main.c 30969e4e14e720e9c603e66714bd1905a63defd4492d5a16d2671398a664fcfd
|
||||
F ext/fts5/fts5_main.c 65c5d579cabaecab478f4bd159ad5c040590f6a75e5afd4ad43c8b92ac65f7f2
|
||||
F ext/fts5/fts5_storage.c 58ba71e6cd3d43a5735815e7956ee167babb4d2cbfe206905174792af4d09d75
|
||||
F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
|
||||
F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
|
||||
F ext/fts5/fts5_test_tok.c f96c6e193c466711d6d7828d5f190407fe7ab897062d371426dd3036f01258e7
|
||||
F ext/fts5/fts5_tokenize.c 2e508c6a3bd8ee56c48e98a38052e1a650e49b32a484cce9b189984114bc3b88
|
||||
F ext/fts5/fts5_tokenize.c be911fbd2f9c9ef0db2b4b492d62628820567eb83521466250fd6df27858fb74
|
||||
F ext/fts5/fts5_unicode2.c 8bd0cd07396b74c1a05590e4070d635bccfc849812c305619f109e6c0485e250
|
||||
F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
|
||||
F ext/fts5/fts5_vocab.c 7a071833064dc8bca236c3c323e56aac36f583aa2c46ce916d52e31ce87462c9
|
||||
@ -198,7 +198,7 @@ F ext/fts5/test/fts5near.test 211477940142d733ac04fad97cb24095513ab2507073a99c27
|
||||
F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618834bf1fcc3e7b84da
|
||||
F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
|
||||
F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
|
||||
F ext/fts5/test/fts5plan.test 771b999d161e24fd803ce0290adb7c6e7c9b9cc2c6a0adb344813fb89473aa32
|
||||
F ext/fts5/test/fts5plan.test 79d35b5e83bbdcba48d946a7f008df161f6b0ede1a966892d0aa6c8dd0b6e773
|
||||
F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
|
||||
F ext/fts5/test/fts5porter2.test 0d251a673f02fa13ca7f011654873b3add20745f7402f108600a23e52d8c7457
|
||||
F ext/fts5/test/fts5prefix.test a0fa67b06650f2deaa7bf27745899d94e0fb547ad9ecbd08bfad98c04912c056
|
||||
@ -216,6 +216,7 @@ F ext/fts5/test/fts5synonym2.test b54cce5c34ec08ed616f646635538ae82e34a0e28f947e
|
||||
F ext/fts5/test/fts5tok1.test ce6551e41ff56f30b69963577324624733bed0d1753589f06120d664d9cd45c9
|
||||
F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
|
||||
F ext/fts5/test/fts5tokenizer.test ac3c9112b263a639fb0508ae73a3ee886bf4866d2153771a8e8a20c721305a43
|
||||
F ext/fts5/test/fts5trigram.test 442b9e0c0f64838e1fad8d3d9e4ebb96f53a3033498e6e80b15d97081b320b0c
|
||||
F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
|
||||
F ext/fts5/test/fts5unicode.test 17056f4efe6b0a5d4f41fdf7a7dc9af2873004562eaa899d40633b93dc95f5a9
|
||||
F ext/fts5/test/fts5unicode2.test 9b3df486de05fb4bde4aa7ee8de2e6dae1df6eb90e3f2e242c9383b95d314e3e
|
||||
@ -1880,8 +1881,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 7395e96b8cc370c8ac2657fb805915b0992a15d80f8bf256d277b423fec64675 dc4172e6b8e1f62dc7329a3adb2223f290bc4c8055c265e88182ef432f4bcf10
|
||||
R 51c03f3e654933edf1a61c7fc932246b
|
||||
T +closed dc4172e6b8e1f62dc7329a3adb2223f290bc4c8055c265e88182ef432f4bcf10
|
||||
U drh
|
||||
Z bbfb93b0824db2f472432135890f2a72
|
||||
P 4a43430fd23f88352c33b29c4c105b72f6dc821f94bf362040c41a1648c402e5
|
||||
R 1f094ba3b91e26d2f277be832160094c
|
||||
T *branch * fts5-trigram
|
||||
T *sym-fts5-trigram *
|
||||
T -sym-trunk *
|
||||
U dan
|
||||
Z f75cc113cdfae84e1f09152a573517c5
|
||||
|
||||
@ -1 +1 @@
|
||||
4a43430fd23f88352c33b29c4c105b72f6dc821f94bf362040c41a1648c402e5
|
||||
0d7810c1aea93c0a3da1ccc4911dbce8a1b6e1dbfe1ab7e800289a0c783b5985
|
||||
Loading…
Reference in New Issue
Block a user