mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add the "detail" option to fts5. Used to reduce the amount of information stored in an fts5 index.

Browse Source 
FossilOrigin-Name: a220e85fe535af5ef2da6ef5fb76abe5a96b5abf
This commit is contained in:
dan 2016-01-12 19:45:01 +00:00
commit 8656b24144
37 changed files with 3264 additions and 1026 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ext/fts5/fts5.h
View File

@ -110,6 +110,12 @@ struct Fts5PhraseIter {
** should be greater than or equal to zero and smaller than the value
** output by xInstCount().
**
** Usually, output parameter *piPhrase is set to the phrase number, *piCol
** to the column in which it occurs and *piOff the token offset of the
** first token of the phrase. The exception is if the table was created
** with the offsets=0 option specified. In this case *piOff is always
** set to -1.
**
** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM)
** if an error occurs.
**
@ -196,7 +202,7 @@ struct Fts5PhraseIter {
** Fts5PhraseIter iter;
** int iCol, iOff;
** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff);
** iOff>=0;
** iCol>=0;
** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
** ){
** // An instance of phrase iPhrase at offset iOff of column iCol
@ -210,7 +216,7 @@ struct Fts5PhraseIter {
** See xPhraseFirst above.
*/
struct Fts5ExtensionApi {
int iVersion; /* Currently always set to 1 */
int iVersion; /* Currently always set to 3 */
void *(*xUserData)(Fts5Context*);
@ -240,8 +246,11 @@ struct Fts5ExtensionApi {
int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*));
void *(*xGetAuxdata)(Fts5Context*, int bClear);
void (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*);
void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff);
int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
};
/*

25
ext/fts5/fts5Int.h
View File

@ -151,6 +151,7 @@ struct Fts5Config {
char *zContent; /* content table */
char *zContentRowid; /* "content_rowid=" option value */
int bColumnsize; /* "columnsize=" option value (dflt==1) */
int eDetail; /* FTS5_DETAIL_XXX value */
char *zContentExprlist;
Fts5Tokenizer *pTok;
fts5_tokenizer *pTokApi;
@ -179,6 +180,9 @@ struct Fts5Config {
#define FTS5_CONTENT_NONE 1
#define FTS5_CONTENT_EXTERNAL 2
#define FTS5_DETAIL_FULL 0
#define FTS5_DETAIL_NONE 1
#define FTS5_DETAIL_COLUMNS 2
@ -292,6 +296,13 @@ char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn);
/* Character set tests (like isspace(), isalpha() etc.) */
int sqlite3Fts5IsBareword(char t);
/* Bucket of terms object used by the integrity-check in offsets=0 mode. */
typedef struct Fts5Termset Fts5Termset;
int sqlite3Fts5TermsetNew(Fts5Termset**);
int sqlite3Fts5TermsetAdd(Fts5Termset*, int, const char*, int, int *pbPresent);
void sqlite3Fts5TermsetFree(Fts5Termset*);
/*
** End of interface to code in fts5_buffer.c.
**************************************************************************/
@ -436,6 +447,8 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
int sqlite3Fts5IterCollist(Fts5IndexIter*, const u8 **, int*);
/*
** End of interface to code in fts5_index.c.
**************************************************************************/
@ -492,7 +505,7 @@ typedef struct Fts5Hash Fts5Hash;
/*
** Create a hash table, free a hash table.
*/
int sqlite3Fts5HashNew(Fts5Hash**, int *pnSize);
int sqlite3Fts5HashNew(Fts5Config*, Fts5Hash**, int *pnSize);
void sqlite3Fts5HashFree(Fts5Hash*);
int sqlite3Fts5HashWrite(
@ -629,8 +642,18 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
typedef struct Fts5PoslistPopulator Fts5PoslistPopulator;
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr*, int);
int sqlite3Fts5ExprPopulatePoslists(
Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int
);
void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64);
void sqlite3Fts5ExprClearEof(Fts5Expr*);
int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
** C code in this module. The interfaces below this point are called by

83
ext/fts5/fts5_buffer.c
View File

@ -292,3 +292,86 @@ int sqlite3Fts5IsBareword(char t){
}
/*************************************************************************
*/
typedef struct Fts5TermsetEntry Fts5TermsetEntry;
struct Fts5TermsetEntry {
char *pTerm;
int nTerm;
int iIdx; /* Index (main or aPrefix[] entry) */
Fts5TermsetEntry *pNext;
};
struct Fts5Termset {
Fts5TermsetEntry *apHash[512];
};
int sqlite3Fts5TermsetNew(Fts5Termset **pp){
int rc = SQLITE_OK;
*pp = sqlite3Fts5MallocZero(&rc, sizeof(Fts5Termset));
return rc;
}
int sqlite3Fts5TermsetAdd(
Fts5Termset *p,
int iIdx,
const char *pTerm, int nTerm,
int *pbPresent
){
int rc = SQLITE_OK;
*pbPresent = 0;
if( p ){
int i;
int hash;
Fts5TermsetEntry *pEntry;
/* Calculate a hash value for this term */
hash = 104 + iIdx;
for(i=0; i<nTerm; i++){
hash += (hash << 3) + (int)pTerm[i];
}
hash = hash % ArraySize(p->apHash);
for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){
if( pEntry->iIdx==iIdx
&& pEntry->nTerm==nTerm
&& memcmp(pEntry->pTerm, pTerm, nTerm)==0
){
*pbPresent = 1;
break;
}
}
if( pEntry==0 ){
pEntry = sqlite3Fts5MallocZero(&rc, sizeof(Fts5TermsetEntry) + nTerm);
if( pEntry ){
pEntry->pTerm = (char*)&pEntry[1];
pEntry->nTerm = nTerm;
pEntry->iIdx = iIdx;
memcpy(pEntry->pTerm, pTerm, nTerm);
pEntry->pNext = p->apHash[hash];
p->apHash[hash] = pEntry;
}
}
}
return rc;
}
void sqlite3Fts5TermsetFree(Fts5Termset *p){
if( p ){
int i;
for(i=0; i<ArraySize(p->apHash); i++){
Fts5TermsetEntry *pEntry = p->apHash[i];
while( pEntry ){
Fts5TermsetEntry *pDel = pEntry;
pEntry = pEntry->pNext;
sqlite3_free(pDel);
}
}
sqlite3_free(p);
}
}

43
ext/fts5/fts5_config.c
View File

@ -14,7 +14,6 @@
*/
#include "fts5Int.h"
#define FTS5_DEFAULT_PAGE_SIZE 4050
@ -195,6 +194,33 @@ void sqlite3Fts5Dequote(char *z){
}
}
struct Fts5Enum {
const char *zName;
int eVal;
};
typedef struct Fts5Enum Fts5Enum;
static int fts5ConfigSetEnum(
const Fts5Enum *aEnum,
const char *zEnum,
int *peVal
){
int nEnum = strlen(zEnum);
int i;
int iVal = -1;
for(i=0; aEnum[i].zName; i++){
if( sqlite3_strnicmp(aEnum[i].zName, zEnum, nEnum)==0 ){
if( iVal>=0 ) return SQLITE_ERROR;
iVal = aEnum[i].eVal;
}
}
*peVal = iVal;
return iVal<0 ? SQLITE_ERROR : SQLITE_OK;
}
/*
** Parse a "special" CREATE VIRTUAL TABLE directive and update
** configuration object pConfig as appropriate.
@ -345,6 +371,20 @@ static int fts5ConfigParseSpecial(
return rc;
}
if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
const Fts5Enum aDetail[] = {
{ "none", FTS5_DETAIL_NONE },
{ "full", FTS5_DETAIL_FULL },
{ "columns", FTS5_DETAIL_COLUMNS },
{ 0, 0 }
};
if( rc = fts5ConfigSetEnum(aDetail, zArg, &pConfig->eDetail) ){
*pzErr = sqlite3_mprintf("malformed detail=... directive");
}
return rc;
}
*pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
return SQLITE_ERROR;
}
@ -500,6 +540,7 @@ int sqlite3Fts5ConfigParse(
pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1);
pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1);
pRet->bColumnsize = 1;
pRet->eDetail = FTS5_DETAIL_FULL;
#ifdef SQLITE_DEBUG
pRet->bPrefixIndex = 1;
#endif

342
ext/fts5/fts5_expr.c
View File

@ -40,6 +40,7 @@ void sqlite3Fts5ParserTrace(FILE*, char*);
struct Fts5Expr {
Fts5Index *pIndex;
Fts5Config *pConfig;
Fts5ExprNode *pRoot;
int bDesc; /* Iterate in descending rowid order */
int nPhrase; /* Number of phrases in expression */
@ -235,6 +236,7 @@ int sqlite3Fts5ExprNew(
}else{
pNew->pRoot = sParse.pExpr;
pNew->pIndex = 0;
pNew->pConfig = pConfig;
pNew->apExprPhrase = sParse.apPhrase;
pNew->nPhrase = sParse.nPhrase;
sParse.apPhrase = 0;
@ -299,8 +301,9 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
/*
** Argument pTerm must be a synonym iterator.
*/
static int fts5ExprSynonymPoslist(
static int fts5ExprSynonymList(
Fts5ExprTerm *pTerm,
int bCollist,
Fts5Colset *pColset,
i64 iRowid,
int *pbDel, /* OUT: Caller should sqlite3_free(*pa) */
@ -319,9 +322,16 @@ static int fts5ExprSynonymPoslist(
if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
const u8 *a;
int n;
i64 dummy;
rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
if( bCollist ){
rc = sqlite3Fts5IterCollist(pIter, &a, &n);
}else{
i64 dummy;
rc = sqlite3Fts5IterPoslist(pIter, pColset, &a, &n, &dummy);
}
if( rc!=SQLITE_OK ) goto synonym_poslist_out;
if( n==0 ) continue;
if( nIter==nAlloc ){
int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
@ -422,8 +432,8 @@ static int fts5ExprPhraseIsMatch(
int bFlag = 0;
const u8 *a = 0;
if( pTerm->pSynonym ){
rc = fts5ExprSynonymPoslist(
pTerm, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
rc = fts5ExprSynonymList(
pTerm, 0, pColset, pNode->iRowid, &bFlag, (u8**)&a, &n
);
}else{
rc = sqlite3Fts5IterPoslist(pTerm->pIter, pColset, &a, &n, &dummy);
@ -757,30 +767,51 @@ static int fts5ExprNearTest(
){
Fts5ExprNearset *pNear = pNode->pNear;
int rc = *pRc;
int i;
/* Check that each phrase in the nearset matches the current row.
** Populate the pPhrase->poslist buffers at the same time. If any
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
pPhrase->aTerm[0].pIter, &pPhrase->poslist
);
if( pExpr->pConfig->eDetail!=FTS5_DETAIL_FULL ){
Fts5ExprTerm *pTerm;
Fts5ExprPhrase *pPhrase = pNear->apPhrase[0];
pPhrase->poslist.n = 0;
for(pTerm=&pPhrase->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
Fts5IndexIter *pIter = pTerm->pIter;
if( sqlite3Fts5IterEof(pIter)==0 ){
int n;
i64 iRowid;
rc = sqlite3Fts5IterPoslist(pIter, pNear->pColset, 0, &n, &iRowid);
if( rc!=SQLITE_OK ){
*pRc = rc;
return 0;
}else if( iRowid==pNode->iRowid && n>0 ){
pPhrase->poslist.n = 1;
}
}
}
}
return pPhrase->poslist.n;
}else{
int i;
*pRc = rc;
if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
return 1;
}
/* Check that each phrase in the nearset matches the current row.
** Populate the pPhrase->poslist buffers at the same time. If any
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && i<pNear->nPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
int bMatch = 0;
rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
pPhrase->aTerm[0].pIter, &pPhrase->poslist
);
}
}
return 0;
*pRc = rc;
if( i==pNear->nPhrase && (i==1 || fts5ExprNearIsMatch(pRc, pNear)) ){
return 1;
}
return 0;
}
}
static int fts5ExprTokenTest(
@ -1218,6 +1249,9 @@ static int fts5ExprNodeNextMatch(
}
pNode->bEof = p1->bEof;
pNode->iRowid = p1->iRowid;
if( p1->bEof ){
fts5ExprNodeZeroPoslist(p2);
}
break;
}
}
@ -1603,6 +1637,7 @@ int sqlite3Fts5ExprClonePhrase(
if( rc==SQLITE_OK ){
/* All the allocations succeeded. Put the expression object together. */
pNew->pIndex = pExpr->pIndex;
pNew->pConfig = pExpr->pConfig;
pNew->nPhrase = 1;
pNew->apExprPhrase[0] = sCtx.pPhrase;
pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
@ -1744,6 +1779,15 @@ void sqlite3Fts5ParseSetColset(
Fts5ExprNearset *pNear,
Fts5Colset *pColset
){
if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){
pParse->rc = SQLITE_ERROR;
pParse->zErr = sqlite3_mprintf(
"fts5: column queries are not supported (detail=none)"
);
sqlite3_free(pColset);
return;
}
if( pNear ){
pNear->pColset = pColset;
}else{
@ -1805,11 +1849,20 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
for(iPhrase=0; iPhrase<pNear->nPhrase; iPhrase++){
pNear->apPhrase[iPhrase]->pNode = pRet;
}
if( pNear->nPhrase==1
&& pNear->apPhrase[0]->nTerm==1
&& pNear->apPhrase[0]->aTerm[0].pSynonym==0
){
pRet->eType = FTS5_TERM;
if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
if( pNear->apPhrase[0]->aTerm[0].pSynonym==0 ){
pRet->eType = FTS5_TERM;
}
}else if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){
assert( pParse->rc==SQLITE_OK );
pParse->rc = SQLITE_ERROR;
assert( pParse->zErr==0 );
pParse->zErr = sqlite3_mprintf(
"fts5: %s queries are not supported (detail!=full)",
pNear->nPhrase==1 ? "phrase": "NEAR"
);
sqlite3_free(pRet);
pRet = 0;
}
}else{
fts5ExprAddChildren(pRet, pLeft);
@ -1923,6 +1976,9 @@ static char *fts5ExprPrintTcl(
for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
char *zTerm = pPhrase->aTerm[iTerm].zTerm;
zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
if( pPhrase->aTerm[iTerm].bPrefix ){
zRet = fts5PrintfAppend(zRet, "*");
}
}
if( zRet ) zRet = fts5PrintfAppend(zRet, "}");
@ -2235,3 +2291,229 @@ int sqlite3Fts5ExprPoslist(Fts5Expr *pExpr, int iPhrase, const u8 **pa){
}
return nRet;
}
struct Fts5PoslistPopulator {
Fts5PoslistWriter writer;
int bOk; /* True if ok to populate */
int bMiss;
};
Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){
Fts5PoslistPopulator *pRet;
pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
if( pRet ){
int i;
memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase);
for(i=0; i<pExpr->nPhrase; i++){
Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist;
Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
assert( pExpr->apExprPhrase[i]->nTerm==1 );
if( bLive &&
(pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof)
){
pRet[i].bMiss = 1;
}else{
pBuf->n = 0;
}
}
}
return pRet;
}
struct Fts5ExprCtx {
Fts5Expr *pExpr;
Fts5PoslistPopulator *aPopulator;
i64 iOff;
};
typedef struct Fts5ExprCtx Fts5ExprCtx;
/*
** TODO: Make this more efficient!
*/
static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
int i;
for(i=0; i<pColset->nCol; i++){
if( pColset->aiCol[i]==iCol ) return 1;
}
return 0;
}
static int fts5ExprPopulatePoslistsCb(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
int iEnd /* Byte offset of end of token within input text */
){
Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
Fts5Expr *pExpr = p->pExpr;
int i;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprTerm *pTerm;
if( p->aPopulator[i].bOk==0 ) continue;
for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
int nTerm = strlen(pTerm->zTerm);
if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
&& memcmp(pTerm->zTerm, pToken, nTerm)==0
){
int rc = sqlite3Fts5PoslistWriterAppend(
&pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
);
if( rc ) return rc;
break;
}
}
}
return SQLITE_OK;
}
int sqlite3Fts5ExprPopulatePoslists(
Fts5Config *pConfig,
Fts5Expr *pExpr,
Fts5PoslistPopulator *aPopulator,
int iCol,
const char *z, int n
){
int i;
Fts5ExprCtx sCtx;
sCtx.pExpr = pExpr;
sCtx.aPopulator = aPopulator;
sCtx.iOff = (((i64)iCol) << 32) - 1;
for(i=0; i<pExpr->nPhrase; i++){
Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode;
Fts5Colset *pColset = pNode->pNear->pColset;
if( (pColset && 0==fts5ExprColsetTest(pColset, iCol))
|| aPopulator[i].bMiss
){
aPopulator[i].bOk = 0;
}else{
aPopulator[i].bOk = 1;
}
}
return sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_AUX, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb
);
}
static void fts5ExprClearPoslists(Fts5ExprNode *pNode){
if( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING ){
pNode->pNear->apPhrase[0]->poslist.n = 0;
}else{
int i;
for(i=0; i<pNode->nChild; i++){
fts5ExprClearPoslists(pNode->apChild[i]);
}
}
}
static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
if( pNode ){
pNode->iRowid = iRowid;
pNode->bEof = 0;
switch( pNode->eType ){
case FTS5_TERM:
case FTS5_STRING:
return (pNode->pNear->apPhrase[0]->poslist.n>0);
case FTS5_AND: {
int i;
for(i=0; i<pNode->nChild; i++){
if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid)==0 ){
fts5ExprClearPoslists(pNode);
return 0;
}
}
return 1;
}
case FTS5_OR: {
int i;
int bRet = 0;
for(i=0; i<pNode->nChild; i++){
if( fts5ExprCheckPoslists(pNode->apChild[i], iRowid) ){
bRet = 1;
}
}
if( bRet==0 ){
fts5ExprClearPoslists(pNode);
}
return bRet;
}
default: {
assert( pNode->eType==FTS5_NOT );
if( 0==fts5ExprCheckPoslists(pNode->apChild[0], iRowid)
|| 0!=fts5ExprCheckPoslists(pNode->apChild[1], iRowid)
){
fts5ExprClearPoslists(pNode);
return 0;
}
return 1;
}
}
}
}
void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){
fts5ExprCheckPoslists(pExpr->pRoot, iRowid);
}
static void fts5ExprClearEof(Fts5ExprNode *pNode){
int i;
for(i=0; i<pNode->nChild; i++){
fts5ExprClearEof(pNode->apChild[i]);
}
pNode->bEof = 0;
}
void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){
fts5ExprClearEof(pExpr->pRoot);
}
/*
** This function is only called for detail=columns tables.
*/
int sqlite3Fts5ExprPhraseCollist(
Fts5Expr *pExpr,
int iPhrase,
const u8 **ppCollist,
int *pnCollist
){
Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase];
Fts5ExprNode *pNode = pPhrase->pNode;
int rc = SQLITE_OK;
assert( iPhrase>=0 && iPhrase<pExpr->nPhrase );
if( pNode->bEof==0
&& pNode->iRowid==pExpr->pRoot->iRowid
&& pPhrase->poslist.n>0
){
Fts5ExprTerm *pTerm = &pPhrase->aTerm[0];
if( pTerm->pSynonym ){
int bDel = 0;
u8 *a;
rc = fts5ExprSynonymList(
pTerm, 1, 0, pNode->iRowid, &bDel, &a, pnCollist
);
if( bDel ){
sqlite3Fts5BufferSet(&rc, &pPhrase->poslist, *pnCollist, a);
*ppCollist = pPhrase->poslist.p;
sqlite3_free(a);
}else{
*ppCollist = a;
}
}else{
sqlite3Fts5IterCollist(pPhrase->aTerm[0].pIter, ppCollist, pnCollist);
}
}else{
*ppCollist = 0;
*pnCollist = 0;
}
return rc;
}

170
ext/fts5/fts5_hash.c
View File

@ -26,6 +26,7 @@ typedef struct Fts5HashEntry Fts5HashEntry;
struct Fts5Hash {
int eDetail; /* Copy of Fts5Config.eDetail */
int *pnByte; /* Pointer to bytes counter */
int nEntry; /* Number of entries currently in hash */
int nSlot; /* Size of aSlot[] array */
@ -62,6 +63,7 @@ struct Fts5HashEntry {
int iSzPoslist; /* Offset of space for 4-byte poslist size */
int nData; /* Total bytes of data (incl. structure) */
u8 bDel; /* Set delete-flag @ iSzPoslist */
u8 bContent; /* Set content-flag (detail=none mode) */
int iCol; /* Column of last value written */
int iPos; /* Position of last value written */
@ -79,7 +81,7 @@ struct Fts5HashEntry {
/*
** Allocate a new hash table.
*/
int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte){
int rc = SQLITE_OK;
Fts5Hash *pNew;
@ -90,6 +92,7 @@ int sqlite3Fts5HashNew(Fts5Hash **ppNew, int *pnByte){
int nByte;
memset(pNew, 0, sizeof(Fts5Hash));
pNew->pnByte = pnByte;
pNew->eDetail = pConfig->eDetail;
pNew->nSlot = 1024;
nByte = sizeof(Fts5HashEntry*) * pNew->nSlot;
@ -182,26 +185,46 @@ static int fts5HashResize(Fts5Hash *pHash){
return SQLITE_OK;
}
static void fts5HashAddPoslistSize(Fts5HashEntry *p){
static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){
if( p->iSzPoslist ){
u8 *pPtr = (u8*)p;
int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
int nPos = nSz*2 + p->bDel; /* Value of nPos field */
assert( p->bDel==0 || p->bDel==1 );
if( nPos<=127 ){
pPtr[p->iSzPoslist] = (u8)nPos;
if( pHash->eDetail==FTS5_DETAIL_NONE ){
assert( p->nData==p->iSzPoslist );
if( p->bDel ){
pPtr[p->nData++] = 0x00;
if( p->bContent ){
pPtr[p->nData++] = 0x00;
}
}
}else{
int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
p->nData += (nByte-1);
int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */
int nPos = nSz*2 + p->bDel; /* Value of nPos field */
assert( p->bDel==0 || p->bDel==1 );
if( nPos<=127 ){
pPtr[p->iSzPoslist] = (u8)nPos;
}else{
int nByte = sqlite3Fts5GetVarintLen((u32)nPos);
memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz);
sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos);
p->nData += (nByte-1);
}
}
p->bDel = 0;
p->iSzPoslist = 0;
p->bDel = 0;
p->bContent = 0;
}
}
/*
** Add an entry to the in-memory hash table. The key is the concatenation
** of bByte and (pToken/nToken). The value is (iRowid/iCol/iPos).
**
** (bByte || pToken) -> (iRowid,iCol,iPos)
**
** Or, if iCol is negative, then the value is a delete marker.
*/
int sqlite3Fts5HashWrite(
Fts5Hash *pHash,
i64 iRowid, /* Rowid for this entry */
@ -214,6 +237,9 @@ int sqlite3Fts5HashWrite(
Fts5HashEntry *p;
u8 *pPtr;
int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */
int bNew; /* If non-delete entry should be written */
bNew = (pHash->eDetail==FTS5_DETAIL_FULL);
/* Attempt to locate an existing hash entry */
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
@ -228,15 +254,18 @@ int sqlite3Fts5HashWrite(
/* If an existing hash entry cannot be found, create a new one. */
if( p==0 ){
/* Figure out how much space to allocate */
int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64;
if( nByte<128 ) nByte = 128;
/* Grow the Fts5Hash.aSlot[] array if necessary. */
if( (pHash->nEntry*2)>=pHash->nSlot ){
int rc = fts5HashResize(pHash);
if( rc!=SQLITE_OK ) return rc;
iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken);
}
/* Allocate new Fts5HashEntry and add it to the hash table. */
p = (Fts5HashEntry*)sqlite3_malloc(nByte);
if( !p ) return SQLITE_NOMEM;
memset(p, 0, FTS5_HASHENTRYSIZE);
@ -246,70 +275,95 @@ int sqlite3Fts5HashWrite(
assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) );
p->zKey[nToken+1] = '\0';
p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE;
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
p->iSzPoslist = p->nData;
p->nData += 1;
p->iRowid = iRowid;
p->pHashNext = pHash->aSlot[iHash];
pHash->aSlot[iHash] = p;
pHash->nEntry++;
nIncr += p->nData;
}
/* Check there is enough space to append a new entry. Worst case scenario
** is:
**
** + 9 bytes for a new rowid,
** + 4 byte reserved for the "poslist size" varint.
** + 1 byte for a "new column" byte,
** + 3 bytes for a new column number (16-bit max) as a varint,
** + 5 bytes for the new position offset (32-bit max).
*/
if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
int nNew = p->nAlloc * 2;
Fts5HashEntry *pNew;
Fts5HashEntry **pp;
pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
if( pNew==0 ) return SQLITE_NOMEM;
pNew->nAlloc = nNew;
for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
*pp = pNew;
p = pNew;
/* Add the first rowid field to the hash-entry */
p->nData += sqlite3Fts5PutVarint(&((u8*)p)[p->nData], iRowid);
p->iRowid = iRowid;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
}
nIncr += p->nData;
}else{
/* Appending to an existing hash-entry. Check that there is enough
** space to append the largest possible new entry. Worst case scenario
** is:
**
** + 9 bytes for a new rowid,
** + 4 byte reserved for the "poslist size" varint.
** + 1 byte for a "new column" byte,
** + 3 bytes for a new column number (16-bit max) as a varint,
** + 5 bytes for the new position offset (32-bit max).
*/
if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){
int nNew = p->nAlloc * 2;
Fts5HashEntry *pNew;
Fts5HashEntry **pp;
pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew);
if( pNew==0 ) return SQLITE_NOMEM;
pNew->nAlloc = nNew;
for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext);
*pp = pNew;
p = pNew;
}
nIncr -= p->nData;
}
assert( (p->nAlloc - p->nData) >= (9 + 4 + 1 + 3 + 5) );
pPtr = (u8*)p;
nIncr -= p->nData;
/* If this is a new rowid, append the 4-byte size field for the previous
** entry, and the new rowid for this entry. */
if( iRowid!=p->iRowid ){
fts5HashAddPoslistSize(p);
fts5HashAddPoslistSize(pHash, p);
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid);
p->iSzPoslist = p->nData;
p->nData += 1;
p->iCol = 0;
p->iPos = 0;
p->iRowid = iRowid;
bNew = 1;
p->iSzPoslist = p->nData;
if( pHash->eDetail!=FTS5_DETAIL_NONE ){
p->nData += 1;
p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1);
p->iPos = 0;
}
}
if( iCol>=0 ){
/* Append a new column value, if necessary */
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = iCol;
p->iPos = 0;
}
if( pHash->eDetail==FTS5_DETAIL_NONE ){
p->bContent = 1;
}else{
/* Append a new column value, if necessary */
assert( iCol>=p->iCol );
if( iCol!=p->iCol ){
if( pHash->eDetail==FTS5_DETAIL_FULL ){
pPtr[p->nData++] = 0x01;
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iCol);
p->iCol = iCol;
p->iPos = 0;
}else{
bNew = 1;
p->iCol = iPos = iCol;
}
}
/* Append the new position offset */
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->iPos = iPos;
/* Append the new position offset, if necessary */
if( bNew ){
p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iPos - p->iPos + 2);
p->iPos = iPos;
}
}
}else{
/* This is a delete. Set the delete flag. */
p->bDel = 1;
}
nIncr += p->nData;
nIncr += p->nData;
*pHash->pnByte += nIncr;
return SQLITE_OK;
}
@ -423,7 +477,7 @@ int sqlite3Fts5HashQuery(
}
if( p ){
fts5HashAddPoslistSize(p);
fts5HashAddPoslistSize(pHash, p);
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);
}else{
@ -459,7 +513,7 @@ void sqlite3Fts5HashScanEntry(
Fts5HashEntry *p;
if( (p = pHash->pScan) ){
int nTerm = (int)strlen(p->zKey);
fts5HashAddPoslistSize(p);
fts5HashAddPoslistSize(pHash, p);
*pzTerm = p->zKey;
*ppDoclist = (const u8*)&p->zKey[nTerm+1];
*pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1);

811
ext/fts5/fts5_index.c
View File

File diff suppressed because it is too large Load Diff

320
ext/fts5/fts5_main.c
View File

@ -226,6 +226,7 @@ struct Fts5Cursor {
#define FTS5CSR_EOF 0x08
#define FTS5CSR_FREE_ZRANK 0x10
#define FTS5CSR_REQUIRE_RESEEK 0x20
#define FTS5CSR_REQUIRE_POSLIST 0x40
#define BitFlagAllTest(x,y) (((x) & (y))==(y))
#define BitFlagTest(x,y) (((x) & (y))!=0)
@ -309,6 +310,13 @@ static int fts5IsContentless(Fts5Table *pTab){
return pTab->pConfig->eContent==FTS5_CONTENT_NONE;
}
/*
** Return true if pTab is an offsetless table.
*/
static int fts5IsOffsetless(Fts5Table *pTab){
return pTab->pConfig->eDetail!=FTS5_DETAIL_FULL;
}
/*
** Delete a virtual table handle allocated by fts5InitVtab().
*/
@ -639,6 +647,7 @@ static void fts5CsrNewrow(Fts5Cursor *pCsr){
FTS5CSR_REQUIRE_CONTENT
| FTS5CSR_REQUIRE_DOCSIZE
| FTS5CSR_REQUIRE_INST
| FTS5CSR_REQUIRE_POSLIST
);
}
@ -721,15 +730,18 @@ static int fts5SorterNext(Fts5Cursor *pCsr){
nBlob = sqlite3_column_bytes(pSorter->pStmt, 1);
aBlob = a = sqlite3_column_blob(pSorter->pStmt, 1);
for(i=0; i<(pSorter->nIdx-1); i++){
int iVal;
a += fts5GetVarint32(a, iVal);
iOff += iVal;
pSorter->aIdx[i] = iOff;
/* nBlob==0 in detail=none mode. */
if( nBlob>0 ){
for(i=0; i<(pSorter->nIdx-1); i++){
int iVal;
a += fts5GetVarint32(a, iVal);
iOff += iVal;
pSorter->aIdx[i] = iOff;
}
pSorter->aIdx[i] = &aBlob[nBlob] - a;
pSorter->aPoslist = a;
}
pSorter->aIdx[i] = &aBlob[nBlob] - a;
pSorter->aPoslist = a;
fts5CsrNewrow(pCsr);
}
@ -1167,6 +1179,7 @@ static int fts5FilterMethod(
pCsr->ePlan = FTS5_PLAN_SOURCE;
pCsr->pExpr = pTab->pSortCsr->pExpr;
rc = fts5CursorFirst(pTab, pCsr, bDesc);
sqlite3Fts5ExprClearEof(pCsr->pExpr);
}else if( pMatch ){
const char *zExpr = (const char*)sqlite3_value_text(apVal[0]);
if( zExpr==0 ) zExpr = "";
@ -1596,6 +1609,8 @@ static int fts5RollbackMethod(sqlite3_vtab *pVtab){
return rc;
}
static int fts5CsrPoslist(Fts5Cursor*, int, const u8**, int*);
static void *fts5ApiUserData(Fts5Context *pCtx){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
return pCsr->pAux->pUserData;
@ -1645,17 +1660,72 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){
return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase);
}
static int fts5CsrPoslist(Fts5Cursor *pCsr, int iPhrase, const u8 **pa){
int n;
if( pCsr->pSorter ){
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
const char **pz,
int *pn
){
int rc = SQLITE_OK;
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
*pz = 0;
*pn = 0;
}else{
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
*pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
*pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
}
}
return rc;
}
static int fts5CsrPoslist(
Fts5Cursor *pCsr,
int iPhrase,
const u8 **pa,
int *pn
){
Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
int rc = SQLITE_OK;
int bLive = (pCsr->pSorter==0);
if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
Fts5PoslistPopulator *aPopulator;
int i;
aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive);
if( aPopulator==0 ) rc = SQLITE_NOMEM;
for(i=0; i<pConfig->nCol && rc==SQLITE_OK; i++){
int n; const char *z;
rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n);
if( rc==SQLITE_OK ){
rc = sqlite3Fts5ExprPopulatePoslists(
pConfig, pCsr->pExpr, aPopulator, i, z, n
);
}
}
sqlite3_free(aPopulator);
if( pCsr->pSorter ){
sqlite3Fts5ExprCheckPoslists(pCsr->pExpr, pCsr->pSorter->iRowid);
}
}
CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
}
if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
Fts5Sorter *pSorter = pCsr->pSorter;
int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
n = pSorter->aIdx[iPhrase] - i1;
*pn = pSorter->aIdx[iPhrase] - i1;
*pa = &pSorter->aPoslist[i1];
}else{
n = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
*pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
}
return n;
return rc;
}
/*
@ -1680,43 +1750,46 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){
int i;
/* Initialize all iterators */
for(i=0; i<nIter; i++){
for(i=0; i<nIter && rc==SQLITE_OK; i++){
const u8 *a;
int n = fts5CsrPoslist(pCsr, i, &a);
int n;
rc = fts5CsrPoslist(pCsr, i, &a, &n);
sqlite3Fts5PoslistReaderInit(a, n, &aIter[i]);
}
while( 1 ){
int *aInst;
int iBest = -1;
for(i=0; i<nIter; i++){
if( (aIter[i].bEof==0)
&& (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
){
iBest = i;
if( rc==SQLITE_OK ){
while( 1 ){
int *aInst;
int iBest = -1;
for(i=0; i<nIter; i++){
if( (aIter[i].bEof==0)
&& (iBest<0 || aIter[i].iPos<aIter[iBest].iPos)
){
iBest = i;
}
}
}
if( iBest<0 ) break;
if( iBest<0 ) break;
nInst++;
if( nInst>=pCsr->nInstAlloc ){
pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
aInst = (int*)sqlite3_realloc(
pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
);
if( aInst ){
pCsr->aInst = aInst;
}else{
rc = SQLITE_NOMEM;
break;
nInst++;
if( nInst>=pCsr->nInstAlloc ){
pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32;
aInst = (int*)sqlite3_realloc(
pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3
);
if( aInst ){
pCsr->aInst = aInst;
}else{
rc = SQLITE_NOMEM;
break;
}
}
}
aInst = &pCsr->aInst[3 * (nInst-1)];
aInst[0] = iBest;
aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
aInst = &pCsr->aInst[3 * (nInst-1)];
aInst[0] = iBest;
aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos);
aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos);
sqlite3Fts5PoslistReaderNext(&aIter[iBest]);
}
}
pCsr->nInstCount = nInst;
@ -1749,6 +1822,12 @@ static int fts5ApiInst(
){
if( iIdx<0 || iIdx>=pCsr->nInstCount ){
rc = SQLITE_RANGE;
#if 0
}else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
*piPhrase = pCsr->aInst[iIdx*3];
*piCol = pCsr->aInst[iIdx*3 + 2];
*piOff = -1;
#endif
}else{
*piPhrase = pCsr->aInst[iIdx*3];
*piCol = pCsr->aInst[iIdx*3 + 1];
@ -1762,27 +1841,6 @@ static sqlite3_int64 fts5ApiRowid(Fts5Context *pCtx){
return fts5CursorRowid((Fts5Cursor*)pCtx);
}
static int fts5ApiColumnText(
Fts5Context *pCtx,
int iCol,
const char **pz,
int *pn
){
int rc = SQLITE_OK;
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){
*pz = 0;
*pn = 0;
}else{
rc = fts5SeekCursor(pCsr, 0);
if( rc==SQLITE_OK ){
*pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1);
*pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1);
}
}
return rc;
}
static int fts5ColumnSizeCb(
void *pContext, /* Pointer to int */
int tflags,
@ -1927,20 +1985,91 @@ static void fts5ApiPhraseNext(
}
}
static void fts5ApiPhraseFirst(
static int fts5ApiPhraseFirst(
Fts5Context *pCtx,
int iPhrase,
Fts5PhraseIter *pIter,
int *piCol, int *piOff
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
int n = fts5CsrPoslist(pCsr, iPhrase, &pIter->a);
pIter->b = &pIter->a[n];
*piCol = 0;
*piOff = 0;
fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
int n;
int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
if( rc==SQLITE_OK ){
pIter->b = &pIter->a[n];
*piCol = 0;
*piOff = 0;
fts5ApiPhraseNext(pCtx, pIter, piCol, piOff);
}
return rc;
}
static void fts5ApiPhraseNextColumn(
Fts5Context *pCtx,
Fts5PhraseIter *pIter,
int *piCol
){
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
if( pIter->a>=pIter->b ){
*piCol = -1;
}else{
int iIncr;
pIter->a += fts5GetVarint32(&pIter->a[0], iIncr);
*piCol += (iIncr-2);
}
}else{
while( 1 ){
int dummy;
if( pIter->a>=pIter->b ){
*piCol = -1;
return;
}
if( pIter->a[0]==0x01 ) break;
pIter->a += fts5GetVarint32(pIter->a, dummy);
}
pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
}
}
static int fts5ApiPhraseFirstColumn(
Fts5Context *pCtx,
int iPhrase,
Fts5PhraseIter *pIter,
int *piCol
){
int rc = SQLITE_OK;
Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
int n;
rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n);
if( rc==SQLITE_OK ){
pIter->b = &pIter->a[n];
*piCol = 0;
fts5ApiPhraseNextColumn(pCtx, pIter, piCol);
}
}else{
int n;
rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n);
if( rc==SQLITE_OK ){
pIter->b = &pIter->a[n];
if( n<=0 ){
*piCol = -1;
}else if( pIter->a[0]==0x01 ){
pIter->a += 1 + fts5GetVarint32(&pIter->a[1], *piCol);
}else{
*piCol = 0;
}
}
}
return rc;
}
static int fts5ApiQueryPhrase(Fts5Context*, int, void*,
int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
);
@ -1964,9 +2093,10 @@ static const Fts5ExtensionApi sFts5Api = {
fts5ApiGetAuxdata,
fts5ApiPhraseFirst,
fts5ApiPhraseNext,
fts5ApiPhraseFirstColumn,
fts5ApiPhraseNextColumn,
};
/*
** Implementation of API function xQueryPhrase().
*/
@ -2098,20 +2228,46 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){
Fts5Buffer val;
memset(&val, 0, sizeof(Fts5Buffer));
switch( ((Fts5Table*)(pCsr->base.pVtab))->pConfig->eDetail ){
case FTS5_DETAIL_FULL:
/* Append the varints */
for(i=0; i<(nPhrase-1); i++){
const u8 *dummy;
int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
}
/* Append the varints */
for(i=0; i<(nPhrase-1); i++){
const u8 *dummy;
int nByte = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &dummy);
sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
}
/* Append the position lists */
for(i=0; i<nPhrase; i++){
const u8 *pPoslist;
int nPoslist;
nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
/* Append the position lists */
for(i=0; i<nPhrase; i++){
const u8 *pPoslist;
int nPoslist;
nPoslist = sqlite3Fts5ExprPoslist(pCsr->pExpr, i, &pPoslist);
sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
}
break;
case FTS5_DETAIL_COLUMNS:
/* Append the varints */
for(i=0; rc==SQLITE_OK && i<(nPhrase-1); i++){
const u8 *dummy;
int nByte;
rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &dummy, &nByte);
sqlite3Fts5BufferAppendVarint(&rc, &val, nByte);
}
/* Append the position lists */
for(i=0; rc==SQLITE_OK && i<nPhrase; i++){
const u8 *pPoslist;
int nPoslist;
rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, i, &pPoslist, &nPoslist);
sqlite3Fts5BufferAppendBlob(&rc, &val, nPoslist, pPoslist);
}
break;
default:
break;
}
sqlite3_result_blob(pCtx, val.p, val.n, sqlite3_free);

86
ext/fts5/fts5_storage.c
View File

@ -825,14 +825,16 @@ struct Fts5IntegrityCtx {
int iCol;
int szCol;
u64 cksum;
Fts5Termset *pTermset;
Fts5Config *pConfig;
};
/*
** Tokenization callback used by integrity check.
*/
static int fts5StorageIntegrityCallback(
void *pContext, /* Pointer to Fts5InsertCtx object */
void *pContext, /* Pointer to Fts5IntegrityCtx object */
int tflags,
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
@ -840,13 +842,56 @@ static int fts5StorageIntegrityCallback(
int iEnd /* End offset of token */
){
Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
Fts5Termset *pTermset = pCtx->pTermset;
int bPresent;
int ii;
int rc = SQLITE_OK;
int iPos;
int iCol;
if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
pCtx->szCol++;
}
pCtx->cksum ^= sqlite3Fts5IndexCksum(
pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken
);
return SQLITE_OK;
switch( pCtx->pConfig->eDetail ){
case FTS5_DETAIL_FULL:
iPos = pCtx->szCol-1;
iCol = pCtx->iCol;
break;
case FTS5_DETAIL_COLUMNS:
iPos = pCtx->iCol;
iCol = 0;
break;
default:
assert( pCtx->pConfig->eDetail==FTS5_DETAIL_NONE );
iPos = 0;
iCol = 0;
break;
}
rc = sqlite3Fts5TermsetAdd(pTermset, 0, pToken, nToken, &bPresent);
if( rc==SQLITE_OK && bPresent==0 ){
pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
pCtx->iRowid, iCol, iPos, 0, pToken, nToken
);
}
for(ii=0; rc==SQLITE_OK && ii<pCtx->pConfig->nPrefix; ii++){
const int nChar = pCtx->pConfig->aPrefix[ii];
int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar);
if( nByte ){
rc = sqlite3Fts5TermsetAdd(pTermset, ii+1, pToken, nByte, &bPresent);
if( bPresent==0 ){
pCtx->cksum ^= sqlite3Fts5IndexEntryCksum(
pCtx->iRowid, iCol, iPos, ii+1, pToken, nByte
);
}
}
}
return rc;
}
/*
@ -882,22 +927,37 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
if( pConfig->bColumnsize ){
rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
}
if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5Tokenize(pConfig,
FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
}
if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
rc = FTS5_CORRUPT;
}
aTotalSize[i] += ctx.szCol;
if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
}
}
sqlite3Fts5TermsetFree(ctx.pTermset);
ctx.pTermset = 0;
if( rc!=SQLITE_OK ) break;
}
rc2 = sqlite3_reset(pScan);

58
ext/fts5/fts5_tcl.c
View File

@ -235,6 +235,8 @@ static int xF5tApi(
{ "xGetAuxdata", 1, "CLEAR" }, /* 13 */
{ "xSetAuxdataInt", 1, "INTEGER" }, /* 14 */
{ "xGetAuxdataInt", 1, "CLEAR" }, /* 15 */
{ "xPhraseForeach", 4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
{ "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
{ 0, 0, 0}
};
@ -431,6 +433,62 @@ static int xF5tApi(
break;
}
CASE(16, "xPhraseForeach") {
int iPhrase;
int iCol;
int iOff;
const char *zColvar;
const char *zOffvar;
Tcl_Obj *pScript = objv[5];
Fts5PhraseIter iter;
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
zColvar = Tcl_GetString(objv[3]);
zOffvar = Tcl_GetString(objv[4]);
for(p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
iCol>=0;
p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff)
){
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
Tcl_SetVar2Ex(interp, zOffvar, 0, Tcl_NewIntObj(iOff), 0);
rc = Tcl_EvalObjEx(interp, pScript, 0);
if( rc==TCL_CONTINUE ) rc = TCL_OK;
if( rc!=TCL_OK ){
if( rc==TCL_BREAK ) rc = TCL_OK;
break;
}
}
break;
}
CASE(17, "xPhraseColumnForeach") {
int iPhrase;
int iCol;
const char *zColvar;
Tcl_Obj *pScript = objv[4];
Fts5PhraseIter iter;
if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
zColvar = Tcl_GetString(objv[3]);
for(p->pApi->xPhraseFirstColumn(p->pFts, iPhrase, &iter, &iCol);
iCol>=0;
p->pApi->xPhraseNextColumn(p->pFts, &iter, &iCol)
){
Tcl_SetVar2Ex(interp, zColvar, 0, Tcl_NewIntObj(iCol), 0);
rc = Tcl_EvalObjEx(interp, pScript, 0);
if( rc==TCL_CONTINUE ) rc = TCL_OK;
if( rc!=TCL_OK ){
if( rc==TCL_BREAK ) rc = TCL_OK;
break;
}
}
break;
}
default:
assert( 0 );
break;

29
ext/fts5/fts5_test_mi.c
View File

@ -134,7 +134,7 @@ static int fts5MatchinfoXCb(
int iPrev = -1;
for(pApi->xPhraseFirst(pFts, 0, &iter, &iCol, &iOff);
iOff>=0;
iCol>=0;
pApi->xPhraseNext(pFts, &iter, &iCol, &iOff)
){
aOut[iCol*3+1]++;
@ -211,18 +211,31 @@ static int fts5MatchinfoLocalCb(
int rc = SQLITE_OK;
switch( f ){
case 'b':
case 'b': {
int iPhrase;
int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
for(i=0; i<nInt; i++) aOut[i] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;
int iCol;
for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol);
iCol>=0;
pApi->xPhraseNextColumn(pFts, &iter, &iCol)
){
aOut[iPhrase * ((p->nCol+31)/32) + iCol/32] |= ((u32)1 << iCol%32);
}
}
break;
}
case 'x':
case 'y': {
int nMul = (f=='x' ? 3 : 1);
int iPhrase;
if( f=='b' ){
int nInt = ((p->nCol + 31) / 32) * p->nPhrase;
for(i=0; i<nInt; i++) aOut[i] = 0;
}else{
for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
}
for(i=0; i<(p->nCol*p->nPhrase); i++) aOut[i*nMul] = 0;
for(iPhrase=0; iPhrase<p->nPhrase; iPhrase++){
Fts5PhraseIter iter;

86
ext/fts5/fts5_vocab.c
View File

@ -379,7 +379,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
if( pTab->eType==FTS5_VOCAB_COL ){
for(pCsr->iCol++; pCsr->iCol<nCol; pCsr->iCol++){
if( pCsr->aCnt[pCsr->iCol] ) break;
if( pCsr->aDoc[pCsr->iCol] ) break;
}
}
@ -412,24 +412,52 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
i64 iPos = 0; /* 64-bit position read from poslist */
int iOff = 0; /* Current offset within position list */
rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
if( rc==SQLITE_OK ){
if( pTab->eType==FTS5_VOCAB_ROW ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
}
pCsr->aDoc[0]++;
}else{
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
pCsr->aDoc[ii]++;
iCol = ii;
switch( pCsr->pConfig->eDetail ){
case FTS5_DETAIL_FULL:
rc = sqlite3Fts5IterPoslist(pCsr->pIter, 0, &pPos, &nPos, &dummy);
if( rc==SQLITE_OK ){
if( pTab->eType==FTS5_VOCAB_ROW ){
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
pCsr->aCnt[0]++;
}
pCsr->aDoc[0]++;
}else{
int iCol = -1;
while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
int ii = FTS5_POS2COLUMN(iPos);
pCsr->aCnt[ii]++;
if( iCol!=ii ){
pCsr->aDoc[ii]++;
iCol = ii;
}
}
}
}
}
break;
case FTS5_DETAIL_COLUMNS:
if( pTab->eType==FTS5_VOCAB_ROW ){
pCsr->aDoc[0]++;
}else{
Fts5Buffer buf = {0, 0, 0};
rc = sqlite3Fts5IterPoslistBuffer(pCsr->pIter, &buf);
if( rc==SQLITE_OK ){
while( 0==sqlite3Fts5PoslistNext64(buf.p, buf.n, &iOff,&iPos) ){
assert_nc( iPos>=0 && iPos<nCol );
if( iPos<nCol ) pCsr->aDoc[iPos]++;
}
}
sqlite3Fts5BufferFree(&buf);
}
break;
default:
assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE );
pCsr->aDoc[0]++;
break;
}
if( rc==SQLITE_OK ){
rc = sqlite3Fts5IterNextScan(pCsr->pIter);
}
@ -445,7 +473,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
}
if( pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){
while( pCsr->aCnt[pCsr->iCol]==0 ) pCsr->iCol++;
while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++;
assert( pCsr->iCol<pCsr->pConfig->nCol );
}
return rc;
@ -525,30 +553,36 @@ static int fts5VocabColumnMethod(
int iCol /* Index of column to read value from */
){
Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor;
int eDetail = pCsr->pConfig->eDetail;
int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType;
i64 iVal = 0;
if( iCol==0 ){
sqlite3_result_text(
pCtx, (const char*)pCsr->term.p, pCsr->term.n, SQLITE_TRANSIENT
);
}
else if( ((Fts5VocabTable*)(pCursor->pVtab))->eType==FTS5_VOCAB_COL ){
}else if( eType==FTS5_VOCAB_COL ){
assert( iCol==1 || iCol==2 || iCol==3 );
if( iCol==1 ){
const char *z = pCsr->pConfig->azCol[pCsr->iCol];
sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
if( eDetail!=FTS5_DETAIL_NONE ){
const char *z = pCsr->pConfig->azCol[pCsr->iCol];
sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC);
}
}else if( iCol==2 ){
sqlite3_result_int64(pCtx, pCsr->aDoc[pCsr->iCol]);
iVal = pCsr->aDoc[pCsr->iCol];
}else{
sqlite3_result_int64(pCtx, pCsr->aCnt[pCsr->iCol]);
iVal = pCsr->aCnt[pCsr->iCol];
}
}else{
assert( iCol==1 || iCol==2 );
if( iCol==1 ){
sqlite3_result_int64(pCtx, pCsr->aDoc[0]);
iVal = pCsr->aDoc[0];
}else{
sqlite3_result_int64(pCtx, pCsr->aCnt[0]);
iVal = pCsr->aCnt[0];
}
}
if( iVal>0 ) sqlite3_result_int64(pCtx, iVal);
return SQLITE_OK;
}

272
ext/fts5/test/fts5_common.tcl
View File

@ -28,6 +28,28 @@ proc fts5_test_poslist {cmd} {
set res
}
proc fts5_test_poslist2 {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseForeach $i c o {
lappend res $i.$c.$o
}
}
set res
}
proc fts5_test_collist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xPhraseCount]} {incr i} {
$cmd xPhraseColumnForeach $i c { lappend res $i.$c }
}
set res
}
proc fts5_test_columnsize {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@ -113,6 +135,8 @@ proc fts5_aux_test_functions {db} {
fts5_test_columntext
fts5_test_columntotalsize
fts5_test_poslist
fts5_test_poslist2
fts5_test_collist
fts5_test_tokenize
fts5_test_rowcount
fts5_test_all
@ -178,15 +202,24 @@ proc fts5_rnddoc {n} {
# -near N (NEAR distance. Default 10)
# -col C (List of column indexes to match against)
# -pc VARNAME (variable in caller frame to use for phrase numbering)
# -dict VARNAME (array in caller frame to use for synonyms)
#
proc nearset {aCol args} {
# Process the command line options.
#
set O(-near) 10
set O(-col) {}
set O(-pc) ""
set O(-dict) ""
set nOpt [lsearch -exact $args --]
if {$nOpt<0} { error "no -- option" }
# Set $lPhrase to be a list of phrases. $nPhrase its length.
set lPhrase [lrange $args [expr $nOpt+1] end]
set nPhrase [llength $lPhrase]
foreach {k v} [lrange $args 0 [expr $nOpt-1]] {
if {[info exists O($k)]==0} { error "unrecognized option $k" }
set O($k) $v
@ -198,9 +231,7 @@ proc nearset {aCol args} {
upvar $O(-pc) counter
}
# Set $phraselist to be a list of phrases. $nPhrase its length.
set phraselist [lrange $args [expr $nOpt+1] end]
set nPhrase [llength $phraselist]
if {$O(-dict)!=""} { upvar $O(-dict) aDict }
for {set j 0} {$j < [llength $aCol]} {incr j} {
for {set i 0} {$i < $nPhrase} {incr i} {
@ -208,27 +239,54 @@ proc nearset {aCol args} {
}
}
set iCol -1
foreach col $aCol {
incr iCol
if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
set nToken [llength $col]
# Loop through each column of the current row.
for {set iCol 0} {$iCol < [llength $aCol]} {incr iCol} {
set iFL [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
for { } {$iFL < $nToken} {incr iFL} {
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set B($iPhrase) [list]
}
# If there is a column filter, test whether this column is excluded. If
# so, skip to the next iteration of this loop. Otherwise, set zCol to the
# column value and nToken to the number of tokens that comprise it.
if {$O(-col)!="" && [lsearch $O(-col) $iCol]<0} continue
set zCol [lindex $aCol $iCol]
set nToken [llength $zCol]
# Each iteration of the following loop searches a substring of the
# column value for phrase matches. The last token of the substring
# is token $iLast of the column value. The first token is:
#
# iFirst = ($iLast - $O(-near) - 1)
#
# where $sz is the length of the phrase being searched for. A phrase
# counts as matching the substring if its first token lies on or before
# $iLast and its last token on or after $iFirst.
#
# For example, if the query is "NEAR(a+b c, 2)" and the column value:
#
# "x x x x A B x x C x"
# 0 1 2 3 4 5 6 7 8 9"
#
# when (iLast==8 && iFirst=5) the range will contain both phrases and
# so both instances can be added to the output poslists.
#
set iLast [expr $O(-near) >= $nToken ? $nToken - 1 : $O(-near)]
for { } {$iLast < $nToken} {incr iLast} {
catch { array unset B }
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
set p [lindex $phraselist $iPhrase]
set p [lindex $lPhrase $iPhrase]
set nPm1 [expr {[llength $p] - 1}]
set iFirst [expr $iFL - $O(-near) - [llength $p]]
set iFirst [expr $iLast - $O(-near) - [llength $p]]
for {set i $iFirst} {$i <= $iFL} {incr i} {
if {[lrange $col $i [expr $i+$nPm1]] == $p} { lappend B($iPhrase) $i }
for {set i $iFirst} {$i <= $iLast} {incr i} {
set lCand [lrange $zCol $i [expr $i+$nPm1]]
set bMatch 1
foreach tok $p term $lCand {
if {[nearset_match aDict $tok $term]==0} { set bMatch 0 ; break }
}
if {$bMatch} { lappend B($iPhrase) $i }
}
if {[llength $B($iPhrase)] == 0} break
if {![info exists B($iPhrase)]} break
}
if {$iPhrase==$nPhrase} {
@ -252,10 +310,22 @@ proc nearset {aCol args} {
incr counter
}
#puts $res
#puts "$aCol -> $res"
sort_poslist $res
}
proc nearset_match {aDictVar tok term} {
if {[string match $tok $term]} { return 1 }
upvar $aDictVar aDict
if {[info exists aDict($tok)]} {
foreach s $aDict($tok) {
if {[string match $s $term]} { return 1 }
}
}
return 0;
}
#-------------------------------------------------------------------------
# Usage:
#
@ -327,3 +397,167 @@ proc fts5_tokenize_split {text} {
set ret
}
#-------------------------------------------------------------------------
#
proc foreach_detail_mode {prefix script} {
set saved $::testprefix
foreach d [list full col none] {
set s [string map [list %DETAIL% $d] $script]
set ::detail $d
set ::testprefix "$prefix-$d"
reset_db
uplevel $s
unset ::detail
}
set ::testprefix $saved
}
proc detail_check {} {
if {$::detail != "none" && $::detail!="full" && $::detail!="col"} {
error "not in foreach_detail_mode {...} block"
}
}
proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
proc detail_is_col {} { detail_check ; expr {$::detail == "col" } }
proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
#-------------------------------------------------------------------------
# Convert a poslist of the type returned by fts5_test_poslist() to a
# collist as returned by fts5_test_collist().
#
proc fts5_poslist2collist {poslist} {
set res [list]
foreach h $poslist {
regexp {(.*)\.[1234567890]+} $h -> cand
lappend res $cand
}
set res [lsort -command fts5_collist_elem_compare -unique $res]
return $res
}
# Comparison function used by fts5_poslist2collist to sort collist entries.
proc fts5_collist_elem_compare {a b} {
foreach {a1 a2} [split $a .] {}
foreach {b1 b2} [split $b .] {}
if {$a1==$b1} { return [expr $a2 - $b2] }
return [expr $a1 - $b1]
}
#--------------------------------------------------------------------------
# Construct and return a tcl list equivalent to that returned by the SQL
# query executed against database handle [db]:
#
# SELECT
# rowid,
# fts5_test_poslist($tbl),
# fts5_test_collist($tbl)
# FROM $tbl('$expr')
# ORDER BY rowid $order;
#
proc fts5_query_data {expr tbl {order ASC} {aDictVar ""}} {
# Figure out the set of columns in the FTS5 table. This routine does
# not handle tables with UNINDEXED columns, but if it did, it would
# have to be here.
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set d ""
if {$aDictVar != ""} {
upvar $aDictVar aDict
set d aDict
}
set cols ""
foreach e $lCols { append cols ", '$e'" }
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, 'nearset $cols -dict $d -pc ::pc' [set cols] )
}]]
set res [list]
db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
set cols [list]
foreach col $lCols { lappend cols $x($col) }
set ::pc 0
set rowdata [eval $tclexpr]
if {$rowdata != ""} {
lappend res $x(rowid) $rowdata [fts5_poslist2collist $rowdata]
}
}
set res
}
#-------------------------------------------------------------------------
# Similar to [fts5_query_data], but omit the collist field.
#
proc fts5_poslist_data {expr tbl {order ASC} {aDictVar ""}} {
set res [list]
if {$aDictVar!=""} {
upvar $aDictVar aDict
set dict aDict
} else {
set dict ""
}
foreach {rowid poslist collist} [fts5_query_data $expr $tbl $order $dict] {
lappend res $rowid $poslist
}
set res
}
#-------------------------------------------------------------------------
#
# This command will only work inside a [foreach_detail_mode] block. It tests
# whether or not expression $expr run on FTS5 table $tbl is supported by
# the current mode. If so, 1 is returned. If not, 0.
#
# detail=full (all queries supported)
# detail=col (all but phrase queries and NEAR queries)
# detail=none (all but phrase queries, NEAR queries, and column filters)
#
proc fts5_expr_ok {expr tbl} {
if {![detail_is_full]} {
set nearset "nearset_rc"
if {[detail_is_col]} { set nearset "nearset_rf" }
set ::expr_not_ok 0
db eval "PRAGMA table_info = $tbl" x { lappend lCols $x(name) }
set cols ""
foreach e $lCols { append cols ", '$e'" }
set ::pc 0
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl( $expr, '[set nearset] $cols -pc ::pc' [set cols] )
}]]
eval $tclexpr
if {$::expr_not_ok} { return 0 }
}
return 1
}
# Helper for [fts5_expr_ok]
proc nearset_rf {aCol args} {
set idx [lsearch -exact $args --]
if {$idx != [llength $args]-2 || [llength [lindex $args end]]!=1} {
set ::expr_not_ok 1
}
list
}
# Helper for [fts5_expr_ok]
proc nearset_rc {aCol args} {
nearset_rf $aCol {*}$args
if {[lsearch $args -col]>=0} {
set ::expr_not_ok 1
}
list
}

62
ext/fts5/test/fts5aa.test
View File

@ -21,6 +21,8 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $::testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
SELECT name, sql FROM sqlite_master;
@ -41,9 +43,9 @@ do_execsql_test 1.1 {
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('a b c', 'd e f');
@ -66,11 +68,12 @@ do_execsql_test 2.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
foreach {i x y} {
1 {g f d b f} {h h e i a}
@ -93,7 +96,7 @@ foreach {i x y} {
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@ -117,7 +120,7 @@ foreach {i x y} {
#
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
foreach {i x y} {
@ -141,7 +144,7 @@ foreach {i x y} {
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
@ -276,7 +279,7 @@ for {set i 1} {$i <= 10} {incr i} {
#
reset_db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x,y);
CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
}
set d10 {
1 {g f d b f} {h h e i a}
@ -309,19 +312,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
#-------------------------------------------------------------------------
#
do_catchsql_test 11.1 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
} {1 {reserved fts5 column name: rank}}
do_catchsql_test 11.2 {
CREATE VIRTUAL TABLE rank USING fts5(a, b, c);
CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
} {1 {reserved fts5 table name: rank}}
do_catchsql_test 11.3 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
} {1 {reserved fts5 column name: rowid}}
#-------------------------------------------------------------------------
#
do_execsql_test 12.1 {
CREATE VIRTUAL TABLE t2 USING fts5(x,y);
CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
} {}
do_catchsql_test 12.2 {
@ -337,7 +340,7 @@ do_test 12.3 {
#
reset_db
do_execsql_test 13.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
} {}
@ -361,7 +364,7 @@ do_execsql_test 13.6 {
#
reset_db
do_execsql_test 14.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@ -371,6 +374,10 @@ do_execsql_test 14.1 {
INSERT INTO t1 SELECT * FROM d LIMIT 200;
}
do_execsql_test 15.x {
INSERT INTO t1(t1) VALUES('integrity-check');
}
do_test 14.2 {
set nRow 0
db eval { SELECT * FROM t1 WHERE t1 MATCH 'xyz' } {
@ -417,11 +424,11 @@ do_execsql_test 16.1 {
}
proc funk {} {
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
set fd [db incrblob main n1_data block 10]
fconfigure $fd -encoding binary -translation binary
puts -nonewline $fd "\x44\x45"
close $fd
db eval { UPDATE n1_config SET v=50 WHERE k='version' }
}
db func funk funk
@ -433,7 +440,7 @@ do_catchsql_test 16.2 {
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE b2 USING fts5(x);
CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
INSERT INTO b2 VALUES('a');
INSERT INTO b2 VALUES('b');
INSERT INTO b2 VALUES('c');
@ -447,18 +454,20 @@ do_test 17.2 {
set res
} {{a b c} {a b c} {a b c}}
reset_db
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
if {[string match n* %DETAIL%]==0} {
reset_db
do_execsql_test 17.3 {
CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO c2 VALUES('x x x', 'x x x');
SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
} {1}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 17.1 {
CREATE VIRTUAL TABLE uio USING fts5(ttt);
CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
INSERT INTO uio VALUES(NULL);
INSERT INTO uio SELECT NULL FROM uio;
INSERT INTO uio SELECT NULL FROM uio;
@ -505,8 +514,8 @@ do_execsql_test 17.9 {
#--------------------------------------------------------------------
#
do_execsql_test 18.1 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
CREATE VIRTUAL TABLE t2 USING fts5(c, d);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
INSERT INTO t1 VALUES('abc*', NULL);
INSERT INTO t2 VALUES(1, 'abcdefg');
}
@ -522,7 +531,7 @@ do_execsql_test 18.3 {
#
reset_db
do_execsql_test 19.0 {
CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('x y z');
INSERT INTO t1 VALUES('w x 1');
SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@ -533,7 +542,7 @@ do_execsql_test 19.0 {
#
reset_db
do_execsql_test 20.0 {
CREATE VIRTUAL TABLE temp.tmp USING fts5(x);
CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
}
set ::ids [list \
0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@ -545,6 +554,7 @@ do_test 20.1 {
execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
} $::ids
}
finish_test

28
ext/fts5/test/fts5ab.test
View File

@ -22,8 +22,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1 VALUES('hello', 'world');
INSERT INTO t1 VALUES('one two', 'three four');
INSERT INTO t1(rowid, a, b) VALUES(45, 'forty', 'five');
@ -57,7 +59,7 @@ do_execsql_test 1.6 {
reset_db
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
INSERT INTO t1 VALUES('one');
INSERT INTO t1 VALUES('two');
@ -159,7 +161,7 @@ foreach {tn expr res} {
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE s1 USING fts5(x);
CREATE VIRTUAL TABLE s1 USING fts5(x, detail=%DETAIL%);
}
foreach {tn doc} [list \
1 [string repeat {a x } 1500000] \
@ -172,8 +174,14 @@ do_execsql_test 4.3 {
SELECT rowid FROM s1 WHERE s1 MATCH 'x'
} {1 2}
do_execsql_test 4.4 {
SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
if {[detail_is_full]} {
do_execsql_test 4.4 {
SELECT rowid FROM s1 WHERE s1 MATCH '"a x"'
} {1 2}
}
do_execsql_test 4.5 {
SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
} {1 2}
#-------------------------------------------------------------------------
@ -182,7 +190,7 @@ do_execsql_test 4.4 {
# (L-2) is larger than it.
#
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE s2 USING fts5(x);
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
@ -222,7 +230,7 @@ do_test 5.2 {
do_test 5.3 {
execsql {
DROP TABLE s2;
CREATE VIRTUAL TABLE s2 USING fts5(x);
CREATE VIRTUAL TABLE s2 USING fts5(x, detail=%DETAIL%);
INSERT INTO s2(s2, rank) VALUES('pgsz', 32);
INSERT INTO s2(s2, rank) VALUES('automerge', 0);
}
@ -241,7 +249,7 @@ do_test 5.4 {
#-------------------------------------------------------------------------
#
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE s3 USING fts5(x);
CREATE VIRTUAL TABLE s3 USING fts5(x, detail=%DETAIL%);
BEGIN;
INSERT INTO s3 VALUES('a b c');
INSERT INTO s3 VALUES('A B C');
@ -276,13 +284,13 @@ do_test 6.4 {
#-------------------------------------------------------------------------
#
set doc [string repeat "a b c " 500]
breakpoint
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE x1 USING fts5(x);
CREATE VIRTUAL TABLE x1 USING fts5(x, detail=%DETAIL%);
INSERT INTO x1(x1, rank) VALUES('pgsz', 32);
INSERT INTO x1 VALUES($doc);
}
} ;# foreach_detail_mode...
finish_test

321
ext/fts5/test/fts5ac.test
View File

@ -22,6 +22,8 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
set data {
0 {p o q e z k z p n f y u z y n y} {l o o l v v k}
1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w}
@ -125,80 +127,21 @@ set data {
99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o}
}
# Argument $expr is an FTS5 match expression designed to be executed against
# an FTS5 table with the following schema:
#
# CREATE VIRTUAL TABLE xy USING fts5(x, y);
#
# Assuming the table contains the same records as stored int the global
# $::data array (see above), this function returns a list containing one
# element for each match in the dataset. The elements are themselves lists
# formatted as follows:
#
# <rowid> {<phrase 0 matches> <phrase 1 matches>...}
#
# where each <phrase X matches> element is a list of phrase matches in the
# same form as returned by auxiliary scalar function fts5_test().
#
proc matchdata {bPos expr {bAsc 1}} {
set tclexpr [db one {
SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y')
}]
set res [list]
#puts $tclexpr
foreach {id x y} $::data {
set cols [list $x $y]
set ::pc 0
#set hits [lsort -command instcompare [eval $tclexpr]]
set hits [eval $tclexpr]
if {[llength $hits]>0} {
if {$bPos} {
lappend res [list $id $hits]
} else {
lappend res $id
}
}
}
if {$bAsc} {
set res [lsort -integer -increasing -index 0 $res]
} else {
set res [lsort -integer -decreasing -index 0 $res]
}
return [concat {*}$res]
}
#
# End of test code
#-------------------------------------------------------------------------
proc fts5_test_poslist {cmd} {
set res [list]
for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
lappend res [string map {{ } .} [$cmd xInst $i]]
}
set res
}
foreach {tn2 sql} {
1 {}
2 {BEGIN}
} {
reset_db
sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE xx USING fts5(x,y);
do_execsql_test 1.$tn2.0 {
CREATE VIRTUAL TABLE xx USING fts5(x,y, detail=%DETAIL%);
INSERT INTO xx(xx, rank) VALUES('pgsz', 32);
}
execsql $sql
do_test $tn2.1.1 {
do_test 1.$tn2.1.1 {
foreach {id x y} $data {
execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) }
}
@ -207,153 +150,131 @@ foreach {tn2 sql} {
#-------------------------------------------------------------------------
# Test phrase queries.
#
foreach {tn phrase} {
1 "o"
2 "b q"
3 "e a e"
4 "m d g q q b k b w f q q p p"
5 "l o o l v v k"
6 "a"
7 "b"
8 "c"
9 "no"
10 "L O O L V V K"
} {
set expr "\"$phrase\""
set res [matchdata 1 $expr]
do_execsql_test $tn2.1.2.$tn.[llength $res] {
SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
} $res
}
#-------------------------------------------------------------------------
# Test some AND and OR queries.
#
foreach {tn expr} {
1.1 "a AND b"
1.2 "a+b AND c"
1.3 "d+c AND u"
1.4 "d+c AND u+d"
2.1 "a OR b"
2.2 "a+b OR c"
2.3 "d+c OR u"
2.4 "d+c OR u+d"
3.1 { a AND b AND c }
} {
set res [matchdata 1 $expr]
do_execsql_test $tn2.2.$tn.[llength $res] {
SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
} $res
}
#-------------------------------------------------------------------------
# Queries on a specific column.
#
foreach {tn expr} {
1.1 "x:a"
1.2 "y:a"
1.3 "x:b"
1.4 "y:b"
2.1 "{x}:a"
2.2 "{y}:a"
2.3 "{x}:b"
2.4 "{y}:b"
3.1 "{x y}:a"
3.2 "{y x}:a"
3.3 "{x x}:b"
3.4 "{y y}:b"
4.1 {{"x" "y"}:a}
4.2 {{"y" x}:a}
4.3 {{x "x"}:b}
4.4 {{"y" y}:b}
} {
set res [matchdata 1 $expr]
do_execsql_test $tn2.3.$tn.[llength $res] {
SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
} $res
}
#-------------------------------------------------------------------------
# Some NEAR queries.
#
foreach {tn expr} {
1 "NEAR(a b)"
2 "NEAR(r c)"
2 { NEAR(r c, 5) }
3 { NEAR(r c, 3) }
4 { NEAR(r c, 2) }
5 { NEAR(r c, 0) }
6 { NEAR(a b c) }
7 { NEAR(a b c, 8) }
8 { x : NEAR(r c) }
9 { y : NEAR(r c) }
} {
set res [matchdata 1 $expr]
do_execsql_test $tn2.4.1.$tn.[llength $res] {
SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr
} $res
}
do_test $tn2.4.1 { nearset {{a b c}} -- a } {0.0.0}
do_test $tn2.4.2 { nearset {{a b c}} -- c } {0.0.2}
foreach {tn expr tclexpr} {
1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
} {
do_execsql_test $tn2.5.$tn {
SELECT fts5_expr_tcl($expr, 'N $x')
} [list $tclexpr]
}
#-------------------------------------------------------------------------
#
do_execsql_test $tn2.6.integrity {
do_execsql_test 1.$tn2.integrity {
INSERT INTO xx(xx) VALUES('integrity-check');
}
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r}
foreach {bAsc sql} {
1 {SELECT rowid FROM xx WHERE xx MATCH $expr}
0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC}
#-------------------------------------------------------------------------
#
foreach {tn expr} {
1.2 "a OR b"
1.1 "a AND b"
1.3 "o"
1.4 "b q"
1.5 "e a e"
1.6 "m d g q q b k b w f q q p p"
1.7 "l o o l v v k"
1.8 "a"
1.9 "b"
1.10 "c"
1.11 "no"
1.12 "L O O L V V K"
1.13 "a AND b AND c"
1.14 "x:a"
2.1 "x:a"
2.2 "y:a"
2.3 "x:b"
2.4 "y:b"
3.1 "{x}:a"
3.2 "{y}:a"
3.3 "{x}:b"
3.4 "{y}:b"
4.1 "{x y}:a"
4.2 "{y x}:a"
4.3 "{x x}:b"
4.4 "{y y}:b"
5.1 {{"x" "y"}:a}
5.2 {{"y" x}:a}
5.3 {{x "x"}:b}
5.4 {{"y" y}:b}
6.1 "b + q"
6.2 "e + a + e"
6.3 "m + d + g + q + q + b + k + b + w + f + q + q + p + p"
6.4 "l + o + o + l + v + v + k"
6.5 "L + O + O + L + V + V + K"
7.1 "a+b AND c"
7.2 "d+c AND u"
7.3 "d+c AND u+d"
7.4 "a+b OR c"
7.5 "d+c OR u"
7.6 "d+c OR u+d"
8.1 "NEAR(a b)"
8.2 "NEAR(r c)"
8.2 { NEAR(r c, 5) }
8.3 { NEAR(r c, 3) }
8.4 { NEAR(r c, 2) }
8.5 { NEAR(r c, 0) }
8.6 { NEAR(a b c) }
8.7 { NEAR(a b c, 8) }
8.8 { x : NEAR(r c) }
8.9 { y : NEAR(r c) }
9.1 { NEAR(r c) }
9.2 { NEAR(r c, 5) }
9.3 { NEAR(r c, 3) }
9.4 { NEAR(r c, 2) }
9.5 { NEAR(r c, 0) }
9.6 { NEAR(a b c) }
9.7 { NEAR(a b c, 8) }
9.8 { x : NEAR(r c) }
9.9 { y : NEAR(r c) }
9.10 { x : "r c" }
9.11 { y : "r c" }
9.12 { a AND b }
9.13 { a AND b AND c }
9.14a { a }
9.14b { a OR b }
9.15 { a OR b AND c }
9.16 { c AND b OR a }
9.17 { c AND (b OR a) }
9.18 { c NOT (b OR a) }
9.19 { (c NOT b) OR (a AND d) }
} {
foreach {tn expr} {
0.1 x
1 { NEAR(r c) }
2 { NEAR(r c, 5) }
3 { NEAR(r c, 3) }
4 { NEAR(r c, 2) }
5 { NEAR(r c, 0) }
6 { NEAR(a b c) }
7 { NEAR(a b c, 8) }
8 { x : NEAR(r c) }
9 { y : NEAR(r c) }
10 { x : "r c" }
11 { y : "r c" }
12 { a AND b }
13 { a AND b AND c }
14a { a }
14b { a OR b }
15 { a OR b AND c }
16 { c AND b OR a }
17 { c AND (b OR a) }
18 { c NOT (b OR a) }
19 { c NOT b OR a AND d }
} {
set res [matchdata 0 $expr $bAsc]
do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res
if {[fts5_expr_ok $expr xx]==0} {
do_test 1.$tn2.$tn.OMITTED { list } [list]
continue
}
set res [fts5_query_data $expr xx]
do_execsql_test 1.$tn2.$tn.[llength $res].asc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr
} $res
set res [fts5_query_data $expr xx DESC]
do_execsql_test 1.$tn2.$tn.[llength $res].desc {
SELECT rowid, fts5_test_poslist(xx), fts5_test_collist(xx)
FROM xx WHERE xx match $expr ORDER BY 1 DESC
} $res
}
}
do_execsql_test 3.1 {
}
do_execsql_test 2.1 {
SELECT fts5_expr_tcl('a AND b');
} {{AND [nearset -- {a}] [nearset -- {b}]}}
do_test 2.2.1 { nearset {{a b c}} -- a } {0.0.0}
do_test 2.2.2 { nearset {{a b c}} -- c } {0.0.2}
foreach {tn expr tclexpr} {
1 {a b} {AND [N $x -- {a}] [N $x -- {b}]}
} {
do_execsql_test 2.3.$tn {
SELECT fts5_expr_tcl($expr, 'N $x')
} [list $tclexpr]
}
finish_test

14
ext/fts5/test/fts5ad.test
View File

@ -22,8 +22,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE yy USING fts5(x, y);
CREATE VIRTUAL TABLE yy USING fts5(x, y, detail=%DETAIL%);
INSERT INTO yy VALUES('Changes the result to be', 'the list of all matching');
INSERT INTO yy VALUES('indices (or all matching', 'values if -inline is');
INSERT INTO yy VALUES('specified as well.) If', 'indices are returned, the');
@ -53,23 +55,23 @@ foreach {tn match res} {
foreach {T create} {
2 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
3 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
4 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
5 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix="1,2,3,4", detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
BEGIN;
}
@ -235,5 +237,7 @@ foreach {T create} {
catchsql COMMIT
}
}
finish_test

52
ext/fts5/test/fts5ae.test
View File

@ -22,8 +22,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
}
@ -55,7 +57,7 @@ fts5_aux_test_functions db
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(x, y);
CREATE VIRTUAL TABLE t2 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t2 VALUES('u t l w w m s', 'm f m o l t k o p e');
INSERT INTO t2 VALUES('f g q e l n d m z x q', 'z s i i i m f w w f n g p');
}
@ -76,31 +78,35 @@ do_execsql_test 2.2 {
2 {1.0.2 1.0.10}
}
do_execsql_test 2.3 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
1 {0.1.3 0.1.7}
if {[detail_is_full]} {
do_execsql_test 2.3 {
SELECT rowid, fts5_test_poslist(t2) FROM t2
WHERE t2 MATCH 'y:o' ORDER BY rowid;
} {
1 {0.1.3 0.1.7}
}
}
#-------------------------------------------------------------------------
#
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t3 USING fts5(x, y);
CREATE VIRTUAL TABLE t3 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t3 VALUES( 'j f h o x x a z g b a f a m i b', 'j z c z y x w t');
INSERT INTO t3 VALUES( 'r c', '');
}
do_execsql_test 3.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
}
if {[detail_is_full]} {
do_execsql_test 3.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(a b)';
} {
1 {0.0.6 1.0.9 0.0.10 0.0.12 1.0.15}
}
do_execsql_test 3.2 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
2 {0.0.0 1.0.1}
do_execsql_test 3.2 {
SELECT rowid, fts5_test_poslist(t3) FROM t3 WHERE t3 MATCH 'NEAR(r c)';
} {
2 {0.0.0 1.0.1}
}
}
do_execsql_test 3.3 {
@ -116,7 +122,7 @@ do_execsql_test 3.3 {
#-------------------------------------------------------------------------
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y);
CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t4
VALUES('k x j r m a d o i z j', 'r t t t f e b r x i v j v g o');
}
@ -134,7 +140,7 @@ reset_db
fts5_aux_test_functions db
do_execsql_test 5.1 {
CREATE VIRTUAL TABLE t5 USING fts5(x, y);
CREATE VIRTUAL TABLE t5 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t5 VALUES('a b c d', 'e f g h i j');
INSERT INTO t5 VALUES('', 'a');
INSERT INTO t5 VALUES('a', '');
@ -182,7 +188,7 @@ do_execsql_test 5.5 {
reset_db
fts5_aux_test_functions db
do_execsql_test 6.1 {
CREATE VIRTUAL TABLE t6 USING fts5(x, y);
CREATE VIRTUAL TABLE t6 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t6 VALUES('There are more', 'things in heaven and earth');
INSERT INTO t6 VALUES(', Horatio, Than are', 'dreamt of in your philosophy.');
}
@ -200,7 +206,7 @@ do_execsql_test 6.2 {
reset_db
fts5_aux_test_functions db
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t7 USING fts5(x, y);
CREATE VIRTUAL TABLE t7 USING fts5(x, y, detail=%DETAIL%);
}
do_test 7.2 {
foreach {x y} {
@ -240,7 +246,7 @@ do_execsql_test 7.4 {
#-------------------------------------------------------------------------
#
do_test 8.1 {
execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y) }
execsql { CREATE VIRTUAL TABLE t8 USING fts5(x, y, detail=%DETAIL%) }
foreach {rowid x y} {
0 {A o} {o o o C o o o o o o o o}
1 {o o B} {o o o C C o o o o o o o}
@ -300,5 +306,7 @@ foreach {tn q cnt} {
} $cnt
}
}
finish_test

56
ext/fts5/test/fts5af.test
View File

@ -24,9 +24,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
}
proc do_snippet_test {tn doc match res} {
@ -111,34 +112,37 @@ foreach {tn doc res} {
do_snippet_test 1.$tn $doc X $res
}
foreach {tn doc res} {
1.1 {X Y o o o o o} {[X Y] o o o o o}
1.2 {o X Y o o o o} {o [X Y] o o o o}
1.3 {o o X Y o o o} {o o [X Y] o o o}
1.4 {o o o X Y o o} {o o o [X Y] o o}
1.5 {o o o o X Y o} {o o o o [X Y] o}
1.6 {o o o o o X Y} {o o o o o [X Y]}
if {[detail_is_full]} {
foreach {tn doc res} {
1.1 {X Y o o o o o} {[X Y] o o o o o}
1.2 {o X Y o o o o} {o [X Y] o o o o}
1.3 {o o X Y o o o} {o o [X Y] o o o}
1.4 {o o o X Y o o} {o o o [X Y] o o}
1.5 {o o o o X Y o} {o o o o [X Y] o}
1.6 {o o o o o X Y} {o o o o o [X Y]}
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o X Y} {...o o o o o [X Y]}
2.1 {X Y o o o o o o} {[X Y] o o o o o...}
2.2 {o X Y o o o o o} {o [X Y] o o o o...}
2.3 {o o X Y o o o o} {o o [X Y] o o o...}
2.4 {o o o X Y o o o} {...o o [X Y] o o o}
2.5 {o o o o X Y o o} {...o o o [X Y] o o}
2.6 {o o o o o X Y o} {...o o o o [X Y] o}
2.7 {o o o o o o X Y} {...o o o o o [X Y]}
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_snippet_test 2.$tn $doc "X + Y" $res
3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
} {
do_snippet_test 2.$tn $doc "X + Y" $res
}
}
} ;# foreach_detail_mode
finish_test

21
ext/fts5/test/fts5ag.test
View File

@ -33,8 +33,10 @@ ifcapable !fts5 {
# ... WHERE fts MATCH ? ORDER BY rank [ASC|DESC]
#
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, z);
CREATE VIRTUAL TABLE t1 USING fts5(x, y, z, detail=%DETAIL%);
}
do_test 1.1 {
@ -119,19 +121,24 @@ foreach {tn expr} {
2.3 c
2.4 d
2.5 {"m m"}
2.6 {e + s}
3.0 {a AND b}
3.1 {a OR b}
3.2 {b OR c AND d}
3.3 {NEAR(c d)}
} {
do_fts5ag_test $tn $expr
if {[set_test_counter errors]} break
}
if {[detail_is_full]} {
foreach {tn expr} {
4.1 {"m m"}
4.2 {e + s}
4.3 {NEAR(c d)}
} {
do_fts5ag_test $tn $expr
}
}
} ;# foreach_detail_mode
finish_test

46
ext/fts5/test/fts5ah.test
View File

@ -21,12 +21,16 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
#-------------------------------------------------------------------------
# This file contains tests for very large doclists.
#
set Y [list]
set W [list]
do_test 1.0 {
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a) }
execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%) }
execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 128) }
set v {w w w w w w w w w w w w w w w w w w w w}
execsql { INSERT INTO t1(rowid, a) VALUES(0, $v) }
@ -70,7 +74,12 @@ do_test 1.4 {
set nRead [reads]
execsql { SELECT rowid FROM t1 WHERE t1 MATCH 'x' }
set nReadX [expr [reads] - $nRead]
expr $nReadX>1000
#puts -nonewline "(nReadX=$nReadX)"
if {[detail_is_full]} { set expect 1000 }
if {[detail_is_col]} { set expect 250 }
if {[detail_is_none]} { set expect 80 }
expr $nReadX>$expect
} {1}
do_test 1.5 {
@ -87,17 +96,22 @@ foreach {tn q res} "
3 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND w' } [list $W]
4 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND x' } [list $Y]
" {
if {[detail_is_full]==0 && ($tn==1 || $tn==2)} continue
if {[detail_is_full]} { set ratio 8 }
if {[detail_is_col]} { set ratio 4 }
if {[detail_is_none]} { set ratio 2 }
do_test 1.6.$tn.1 {
set n [execsql_reads $q]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / 8)}
expr {$n < ($nReadX / $ratio)}
} {1}
do_test 1.6.$tn.2 {
set n [execsql_reads "$q ORDER BY rowid DESC"]
#puts -nonewline "(n=$n nReadX=$nReadX)"
expr {$n < ($nReadX / 8)}
expr {$n < ($nReadX / $ratio)}
} {1}
do_execsql_test 1.6.$tn.3 $q [lsort -int -incr $res]
@ -109,21 +123,26 @@ foreach {tn q res} "
# number of pages loaded from disk.
#
foreach {tn fraction tail cnt} {
1 0.6 {rowid > 5000} 5000
2 0.2 {rowid > 9000} 1000
3 0.2 {rowid < 1000} 999
4 0.2 {rowid BETWEEN 4000 AND 5000} 1001
5 0.6 {rowid >= 5000} 5001
6 0.2 {rowid >= 9000} 1001
7 0.2 {rowid <= 1000} 1000
8 0.6 {rowid > '5000'} 5000
9 0.2 {rowid > '9000'} 1000
1 0.6 {rowid > 5000} 5000
2 0.2 {rowid > 9000} 1000
3 0.2 {rowid < 1000} 999
4 0.2 {rowid BETWEEN 4000 AND 5000} 1001
5 0.6 {rowid >= 5000} 5001
6 0.2 {rowid >= 9000} 1001
7 0.2 {rowid <= 1000} 1000
8 0.6 {rowid > '5000'} 5000
9 0.2 {rowid > '9000'} 1000
10 0.1 {rowid = 444} 1
} {
set q "SELECT rowid FROM t1 WHERE t1 MATCH 'x' AND $tail"
set n [execsql_reads $q]
set ret [llength [execsql $q]]
# Because the position lists for 'x' are quite long in this db, the
# advantage is a bit smaller in detail=none mode. Update $fraction to
# reflect this.
if {[detail_is_none] && $fraction<0.5} { set fraction [expr $fraction*2] }
do_test "1.7.$tn.asc.(n=$n ret=$ret)" {
expr {$n < ($fraction*$nReadX) && $ret==$cnt}
} {1}
@ -143,6 +162,7 @@ do_execsql_test 1.8.2 {
SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
} {10000}
} ;# foreach_detail_mode
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

5
ext/fts5/test/fts5ai.test
View File

@ -23,8 +23,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a);
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=%DETAIL%);
} {}
do_execsql_test 1.1 {
@ -49,6 +51,7 @@ do_execsql_test 1.1 {
do_execsql_test 1.2 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
}
finish_test

107
ext/fts5/test/fts5ak.test
View File

@ -23,8 +23,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE ft1 USING fts5(x);
CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
INSERT INTO ft1 VALUES('i d d a g i b g d d');
INSERT INTO ft1 VALUES('h d b j c c g a c a');
INSERT INTO ft1 VALUES('e j a e f h b f h h');
@ -35,6 +37,9 @@ do_execsql_test 1.1 {
INSERT INTO ft1 VALUES('i c c f a d g h j e');
INSERT INTO ft1 VALUES('i d i g c d c h b f');
INSERT INTO ft1 VALUES('g d a e h a b c f j');
CREATE VIRTUAL TABLE ft2 USING fts5(x, detail=%DETAIL%);
INSERT INTO ft2 VALUES('a b c d e f g h i j');
}
do_execsql_test 1.2 {
@ -49,19 +54,6 @@ do_execsql_test 1.2 {
}
do_execsql_test 1.3 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
} {
{[h d] b j c c g a c a}
{j f [h d] g h i b d f}
}
do_execsql_test 1.4 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 1.5 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'e e e'
} {
{[e] j a [e] f h b f h h}
@ -72,57 +64,71 @@ do_execsql_test 1.5 {
{g d a [e] h a b c f j}
}
do_execsql_test 1.6 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 2.1 {
CREATE VIRTUAL TABLE ft2 USING fts5(x);
INSERT INTO ft2 VALUES('a b c d e f g h i j');
}
do_execsql_test 2.2 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
} {{a [b c d e] f g h i j}}
do_execsql_test 2.3 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
} {
{a [b c d] [e f g] h i j}
}
do_execsql_test 2.4 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
} {
{a [b c d] e f g h i j}
}
do_execsql_test 2.5 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
} {
{a [b c d e] f g h i j}
}
do_execsql_test 2.6.1 {
do_execsql_test 1.4 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'f d'
} {
{a b c [d] e [f] g h i j}
}
do_execsql_test 2.6.2 {
do_execsql_test 1.5 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'd f'
} {
{a b c [d] e [f] g h i j}
}
#-------------------------------------------------------------------------
# Tests below this point require detail=full.
#-------------------------------------------------------------------------
if {[detail_is_full]==0} continue
do_execsql_test 2.1 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'h + d';
} {
{[h d] b j c c g a c a}
{j f [h d] g h i b d f}
}
do_execsql_test 2.2 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 2.3 {
SELECT highlight(ft1, 0, '[', ']') FROM ft1 WHERE ft1 MATCH 'd + d d + d';
} {
{i [d d] a g i b g [d d]}
}
do_execsql_test 2.4 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c+d+e'
} {{a [b c d e] f g h i j}}
do_execsql_test 2.5 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d e+f+g'
} {
{a [b c d] [e f g] h i j}
}
do_execsql_test 2.6 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c+d c'
} {
{a [b c d] e f g h i j}
}
do_execsql_test 2.7 {
SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'b+c c+d+e'
} {
{a [b c d e] f g h i j}
}
#-------------------------------------------------------------------------
# The example from the docs.
#
do_execsql_test 3.1 {
-- Assuming this:
CREATE VIRTUAL TABLE ft USING fts5(a);
CREATE VIRTUAL TABLE ft USING fts5(a, detail=%DETAIL%);
INSERT INTO ft VALUES('a b c x c d e');
INSERT INTO ft VALUES('a b c c d e');
INSERT INTO ft VALUES('a b c d e');
@ -138,6 +144,7 @@ do_execsql_test 3.1 {
{[a b c d e]}
}
}
finish_test

59
ext/fts5/test/fts5al.test
View File

@ -23,8 +23,10 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE ft1 USING fts5(x);
CREATE VIRTUAL TABLE ft1 USING fts5(x, detail=%DETAIL%);
SELECT * FROM ft1_config;
} {version 4}
@ -83,7 +85,7 @@ foreach {tn defn} {
#
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1 VALUES('q w e r t y');
INSERT INTO t1 VALUES('y t r e w q');
}
@ -122,11 +124,13 @@ do_execsql_test 3.4.1 {
{{0 0 5}}
}
do_execsql_test 3.4.2 {
SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
} {
{{1 0 1}}
{{0 0 2} {1 0 4}}
if {[detail_is_full]} {
do_execsql_test 3.4.2 {
SELECT insttest(t1) FROM t1 WHERE t1 MATCH 'r+e OR w'
} {
{{1 0 1}}
{{0 0 2} {1 0 4}}
}
}
proc coltest {cmd} {
@ -149,7 +153,7 @@ do_execsql_test 3.5.1 {
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, b);
CREATE VIRTUAL TABLE t2 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t2 VALUES('a s h g s b j m r h', 's b p a d b b a o e');
INSERT INTO t2 VALUES('r h n t a g r d d i', 'l d n j r c f t o q');
INSERT INTO t2 VALUES('q k n i k c a a e m', 'c h n j p g s c i t');
@ -218,24 +222,26 @@ proc rowidplus {cmd ival} {
}
sqlite3_fts5_create_function db rowidplus rowidplus
do_execsql_test 4.2.1 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 110
}
do_execsql_test 4.2.2 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 121
}
if {[detail_is_full]} {
do_execsql_test 4.2.1 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(100) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 110
}
do_execsql_test 4.2.2 {
INSERT INTO t2(t2, rank) VALUES('rank', 'rowidplus(111) ');
SELECT rowid, rank FROM t2 WHERE t2 MATCH 'o + q + g'
} {
10 121
}
do_execsql_test 4.2.3 {
SELECT rowid, rank FROM t2
WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
} {
10 122
do_execsql_test 4.2.3 {
SELECT rowid, rank FROM t2
WHERE t2 MATCH 'o + q + g' AND rank MATCH 'rowidplus(112)'
} {
10 122
}
}
proc rowidmod {cmd imod} {
@ -243,7 +249,7 @@ proc rowidmod {cmd imod} {
}
sqlite3_fts5_create_function db rowidmod rowidmod
do_execsql_test 4.3.1 {
CREATE VIRTUAL TABLE t3 USING fts5(x);
CREATE VIRTUAL TABLE t3 USING fts5(x, detail=%DETAIL%);
INSERT INTO t3 VALUES('a one');
INSERT INTO t3 VALUES('a two');
INSERT INTO t3 VALUES('a three');
@ -287,6 +293,7 @@ do_catchsql_test 4.4.4 {
SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
} {1 {parse error in rank function: }}
} ;# foreach_detail_mode
finish_test

42
ext/fts5/test/fts5auto.test
View File

@ -22,7 +22,6 @@ ifcapable !fts5 {
return
}
set data {
-4026076
{n x w k b p x b n t t d s} {f j j s p j o}
@ -232,37 +231,9 @@ do_execsql_test 1.0 {
fts5_aux_test_functions db
proc matchdata {expr tbl collist {order ASC}} {
set cols ""
foreach e $collist {
append cols ", '$e'"
}
set tclexpr [db one [subst -novar {
SELECT fts5_expr_tcl(
$expr, 'nearset $cols -pc ::pc' [set cols]
)
}]]
set res [list]
db eval "SELECT rowid, * FROM $tbl ORDER BY rowid $order" x {
set cols [list]
foreach col $x(*) {
if {$col != "rowid"} { lappend cols $x($col) }
}
# set cols [list $a $b $c $d $e $f]
set ::pc 0
set rowdata [eval $tclexpr]
if {$rowdata != ""} { lappend res $x(rowid) $rowdata }
}
set res
}
proc do_auto_test {tn tbl cols expr} {
proc do_auto_test {tn tbl expr} {
foreach order {asc desc} {
set res [matchdata $expr $tbl $cols $order]
set res [fts5_poslist_data $expr $tbl $order]
set testname "$tn.[string range $order 0 0].rows=[expr [llength $res]/2]"
set ::autotest_expr $expr
@ -271,8 +242,6 @@ proc do_auto_test {tn tbl cols expr} {
WHERE [set tbl] MATCH $::autotest_expr ORDER BY rowid [set order]
}] $res
}
}
#-------------------------------------------------------------------------
@ -332,7 +301,7 @@ for {set fold 0} {$fold < 3} {incr fold} {
C.1 { a OR (b AND "b c") }
C.2 { a OR (b AND "z c") }
} {
do_auto_test 3.$fold.$tn tt {a b c d e f} $expr
do_auto_test 3.$fold.$tn tt $expr
}
}
@ -366,11 +335,8 @@ foreach {tn expr} {
4 {c1 : x} 5 {c2 : x} 6 {c3 : x}
7 {c1 : y} 8 {c2 : y} 9 {c3 : y}
10 {c1 : z} 11 {c2 : z} 12 {c3 : z}
} {
breakpoint
do_auto_test 4.$tn yy {c1 c2 c3} $expr
do_auto_test 4.$tn yy $expr
}

240
ext/fts5/test/fts5detail.test Normal file
View File

@ -0,0 +1,240 @@
# 2015 December 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS5 module.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5detail
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
fts5_aux_test_functions db
#--------------------------------------------------------------------------
# Simple tests.
#
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, detail=col);
INSERT INTO t1 VALUES('h d g', 'j b b g b', 'i e i d h g g'); -- 1
INSERT INTO t1 VALUES('h j d', 'j h d a h', 'f d d g g f b'); -- 2
INSERT INTO t1 VALUES('j c i', 'f f h e f', 'c j i j c h f'); -- 3
INSERT INTO t1 VALUES('e g g', 'g e d h i', 'e d b e g d c'); -- 4
INSERT INTO t1 VALUES('b c c', 'd i h a f', 'd i j f a b c'); -- 5
INSERT INTO t1 VALUES('e d e', 'b c j g d', 'a i f d h b d'); -- 6
INSERT INTO t1 VALUES('g h e', 'b c d i d', 'e f c i f i c'); -- 7
INSERT INTO t1 VALUES('c f j', 'j j i e a', 'h a c f d h e'); -- 8
INSERT INTO t1 VALUES('a h i', 'c i a f a', 'c f d h g d g'); -- 9
INSERT INTO t1 VALUES('j g g', 'e f e f f', 'h j b i c g e'); -- 10
}
do_execsql_test 1.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
foreach {tn match res} {
1 "a:a" {9}
2 "b:g" {1 4 6}
3 "c:h" {1 3 6 8 9 10}
} {
do_execsql_test 1.2.$tn.1 {
SELECT rowid FROM t1($match);
} $res
do_execsql_test 1.2.$tn.2 {
SELECT rowid FROM t1($match || '*');
} $res
}
do_catchsql_test 1.3.1 {
SELECT rowid FROM t1('h + d');
} {1 {fts5: phrase queries are not supported (detail!=full)}}
do_catchsql_test 1.3.2 {
SELECT rowid FROM t1('NEAR(h d)');
} {1 {fts5: NEAR queries are not supported (detail!=full)}}
#-------------------------------------------------------------------------
# integrity-check with both detail= and prefix= options.
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t2 USING fts5(a, detail=col, prefix="1");
INSERT INTO t2(a) VALUES('aa ab');
}
#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t2_data} {puts $r}
do_execsql_test 2.1 {
INSERT INTO t2(t2) VALUES('integrity-check');
}
do_execsql_test 2.2 {
SELECT fts5_test_poslist(t2) FROM t2('aa');
} {0.0.0}
set ::pc 0
#puts [nearset {{ax bx cx}} -pc ::pc -near 10 -- b*]
#exit
#-------------------------------------------------------------------------
# Check that the xInstCount, xInst, xPhraseFirst and xPhraseNext APIs
# work with detail=col tables.
#
set data {
1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
2 {bca aca acb} {ccb bcc bca aab bcc} {bab aaa aac cbb bba aca abc}
3 {cca abc cab} {aab aba bcc cac baa} {bab cbb acb aba aab ccc cca}
4 {ccb bcb aba} {aba bbb bcc cac bbb} {cbb aaa bca bcc aab cac aca}
5 {bca bbc cac} {aba cbb cac cca aca} {cab acb cbc ccb cac bbb bcb}
6 {acc bba cba} {bab bbc bbb bcb aca} {bca ccc cbb aca bac ccc ccb}
7 {aba bab aaa} {abb bca aac bcb bcc} {bcb bbc aba aaa cba abc acc}
8 {cab aba aaa} {ccb aca caa bbc bcc} {aaa abc ccb bbb cac cca abb}
9 {bcb bab bac} {bcb cba cac bbb abc} {aba aca cbb acb abb ccc ccb}
10 {aba aab ccc} {abc ccc bcc cab bbb} {aab bcc cbb ccc aaa bac baa}
11 {bab acb cba} {aac cab cab bca cbc} {aab cbc aac baa ccb acc cac}
12 {ccc cbb cbc} {aaa aab bcc aac bbc} {cbc cbc bac bac ccc bbc acc}
13 {cab bbc abc} {bbb bab bba aca bab} {baa bbb aab bbb ccb bbb ccc}
14 {bbc cab caa} {acb aac abb cba acc} {cba bba bba acb abc abb baa}
15 {aba cca bcc} {aaa acb abc aab ccb} {cca bcb acc aaa caa cca cbc}
16 {bcb bba aba} {cbc acb cab caa ccb} {aac aaa bbc cab cca cba abc}
17 {caa cbb acc} {ccb bcb bca aaa bcc} {bbb aca bcb bca cbc cbc cca}
18 {cbb bbc aac} {ccc bbc aaa aab baa} {cab cab cac cca bbc abc bbc}
19 {ccc acc aaa} {aab cbb bca cca caa} {bcb aca aca cab acc bac bcc}
20 {aab ccc bcb} {bbc cbb bbc aaa bcc} {cbc aab ccc aaa bcb bac cbc}
21 {aba cab ccc} {bbc cbc cba acc bbb} {acc aab aac acb aca bca acb}
22 {bcb bca baa} {cca bbc aca ccb cbb} {aab abc bbc aaa cab bcc bcc}
23 {cac cbb caa} {bbc aba bbb bcc ccb} {bbc bbb cab bbc cac abb acc}
24 {ccb acb caa} {cab bba cac bbc aac} {aac bca abc cab bca cab bcb}
25 {bbb aca bca} {bcb acc ccc cac aca} {ccc acb acc cac cac bba bbc}
26 {bab acc caa} {caa cab cac bac aca} {aba cac caa acc bac ccc aaa}
27 {bca bca aaa} {ccb aca bca aaa baa} {bab acc aaa cca cba cca bac}
28 {ccb cac cac} {bca abb bba bbc baa} {aca ccb aac cab ccc cab caa}
29 {abc bca cab} {cac cbc cbb ccc bcc} {bcc aaa aaa acc aac cac aac}
30 {aca acc acb} {aab aac cbb caa acb} {acb bbc bbc acc cbb bbc aac}
31 {aba aca baa} {aca bcc cab bab acb} {bcc acb baa bcb bbc acc aba}
32 {abb cbc caa} {cba abb bbb cbb aca} {bac aca caa cac caa ccb bbc}
33 {bcc bcb bcb} {cca cab cbc abb bab} {caa bbc aac bbb cab cba aaa}
34 {caa cab acc} {ccc ccc bcc acb bcc} {bac bba aca bcb bba bcb cac}
35 {bac bcb cba} {bcc acb bbc cba bab} {abb cbb abc abc bac acc cbb}
36 {cab bab ccb} {bca bba bab cca acc} {acc aab bcc bac acb cbb caa}
37 {aca cbc cab} {bba aac aca aac aaa} {baa cbb cba aba cab bca bcb}
38 {acb aab baa} {baa bab bca bbc bbb} {abc baa acc aba cab baa cac}
39 {bcb aac cba} {bcb baa caa cac bbc} {cbc ccc bab ccb bbb caa aba}
40 {cba ccb abc} {cbb caa cba aac bab} {cbb bbb bca bbb bac cac bca}
}
set data {
1 {abb aca aca} {aba bab aab aac caa} {abc cbc ccb bcc bab ccb aca}
}
proc matchdata {expr {bAsc 1}} {
set tclexpr [db one {
SELECT fts5_expr_tcl($expr, 'nearset $cols -pc ::pc', 'x', 'y', 'z')
}]
set res [list]
#puts "$expr -> $tclexpr"
foreach {id x y z} $::data {
set cols [list $x $y $z]
set ::pc 0
#set hits [lsort -command instcompare [eval $tclexpr]]
set hits [eval $tclexpr]
if {[llength $hits]>0} {
lappend res [list $id $hits]
}
}
if {$bAsc} {
set res [lsort -integer -increasing -index 0 $res]
} else {
set res [lsort -integer -decreasing -index 0 $res]
}
return [concat {*}$res]
}
foreach {tn tbl} {
1 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=col) }
2 { CREATE VIRTUAL TABLE t3 USING fts5(x, y, z, detail=none) }
} {
reset_db
fts5_aux_test_functions db
execsql $tbl
foreach {id x y z} $data {
execsql { INSERT INTO t3(rowid, x, y, z) VALUES($id, $x, $y, $z) }
}
foreach {tn2 expr} {
1 aaa 2 ccc 3 bab 4 aac
5 aa* 6 cc* 7 ba* 8 aa*
9 a* 10 b* 11 c*
} {
set res [matchdata $expr]
do_execsql_test 3.$tn.$tn2.1 {
SELECT rowid, fts5_test_poslist(t3) FROM t3($expr)
} $res
do_execsql_test 3.$tn.$tn2.2 {
SELECT rowid, fts5_test_poslist2(t3) FROM t3($expr)
} $res
}
}
#-------------------------------------------------------------------------
# Simple tests for detail=none tables.
#
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t4 USING fts5(a, b, c, detail=none);
INSERT INTO t4 VALUES('a b c', 'b c d', 'e f g');
INSERT INTO t4 VALUES('1 2 3', '4 5 6', '7 8 9');
}
do_catchsql_test 4.1 {
SELECT * FROM t4('a:a')
} {1 {fts5: column queries are not supported (detail=none)}}
#-------------------------------------------------------------------------
# Test that for the same content detail=none uses less space than
# detail=col, and that detail=col uses less space than detail=full
#
reset_db
do_test 5.1 {
foreach {tbl detail} {t1 none t2 col t3 full} {
execsql "CREATE VIRTUAL TABLE $tbl USING fts5(x, y, z, detail=$detail)"
foreach {rowid x y z} $::data {
execsql "INSERT INTO $tbl (rowid, x, y, z) VALUES(\$rowid, \$x, \$y, \$z)"
}
}
} {}
do_execsql_test 5.2 {
SELECT
(SELECT sum(length(block)) from t1_data) <
(SELECT sum(length(block)) from t2_data)
} {1}
do_execsql_test 5.3 {
SELECT
(SELECT sum(length(block)) from t2_data) <
(SELECT sum(length(block)) from t3_data)
} {1}
finish_test

25
ext/fts5/test/fts5dlidx.test
View File

@ -26,7 +26,7 @@ if { $tcl_platform(wordSize)<8 } {
return
}
if 1 {
foreach_detail_mode $testprefix {
proc do_fb_test {tn sql res} {
set res2 [lsort -integer -decr $res]
@ -34,7 +34,7 @@ proc do_fb_test {tn sql res} {
uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
}
# This test populates the FTS5 table containing $nEntry entries. Rows are
# This test populates the FTS5 table with $nEntry entries. Rows are
# numbered from 0 to ($nEntry-1). The rowid for row $i is:
#
# ($iFirst + $i*$nStep)
@ -77,10 +77,12 @@ proc do_dlidx_test1 {tn spc1 spc2 nEntry iFirst nStep} {
do_fb_test $tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
do_fb_test $tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
do_fb_test $tn.5.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
do_fb_test $tn.5.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
if {[detail_is_full]} {
do_fb_test $tn.5.1 {
SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
do_fb_test $tn.5.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
}
}
@ -90,7 +92,7 @@ foreach {tn pgsz} {
} {
do_execsql_test $tn.0 {
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', $pgsz);
}
@ -107,7 +109,7 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
execsql {
BEGIN;
DROP TABLE IF EXISTS t1;
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
INSERT INTO t1 VALUES('b a');
@ -130,8 +132,6 @@ proc do_dlidx_test2 {tn nEntry iFirst nStep} {
do_dlidx_test2 2.1 [expr 20] [expr 1<<57] [expr (1<<57) + 128]
}
#--------------------------------------------------------------------
#
reset_db
@ -158,7 +158,7 @@ proc rnddoc {} {
db func rnddoc rnddoc
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE abc USING fts5(a);
CREATE VIRTUAL TABLE abc USING fts5(a, detail=%DETAIL%);
INSERT INTO abc(abc, rank) VALUES('pgsz', 32);
INSERT INTO abc VALUES ( rnddoc() );
@ -192,6 +192,9 @@ foreach v $vocab {
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}
} ;# foreach_detail_mode
finish_test

9
ext/fts5/test/fts5eb.test
View File

@ -42,6 +42,15 @@ foreach {tn expr res} {
10 {abc + "" + def} {"abc" + "def"}
11 {abc "" def} {"abc" AND "def"}
12 {r+e OR w} {"r" + "e" OR "w"}
13 {a AND b NOT c} {"a" AND ("b" NOT "c")}
14 {a OR b NOT c} {"a" OR ("b" NOT "c")}
15 {a NOT b AND c} {("a" NOT "b") AND "c"}
16 {a NOT b OR c} {("a" NOT "b") OR "c"}
17 {a AND b OR c} {("a" AND "b") OR "c"}
18 {a OR b AND c} {"a" OR ("b" AND "c")}
} {
do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}

49
ext/fts5/test/fts5fault8.test Normal file
View File

@ -0,0 +1,49 @@
# 2015 September 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
# This file is focused on OOM errors.
#
source [file join [file dirname [info script]] fts5_common.tcl]
source $testdir/malloc_common.tcl
set testprefix fts5fault8
# If SQLITE_ENABLE_FTS3 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
foreach_detail_mode $testprefix {
if {[detail_is_none]==0} continue
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
INSERT INTO t1 VALUES('a b c d', '1 2 3 4');
INSERT INTO t1 VALUES('a b a b', NULL);
INSERT INTO t1 VALUES(NULL, '1 2 1 2');
}
do_faultsim_test 1 -faults oom-t* -body {
execsql {
SELECT rowid, fts5_test_poslist(t1) FROM t1 WHERE t1 MATCH 'b OR 2'
}
} -test {
faultsim_test_result {0 {1 {0.0.1 1.1.1} 2 {0.0.1 0.0.3} 3 {1.1.1 1.1.3}}} \
{1 SQLITE_NOMEM}
}
}
finish_test

80
ext/fts5/test/fts5matchinfo.test
View File

@ -16,6 +16,8 @@ set testprefix fts5matchinfo
# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
ifcapable !fts5 { finish_test ; return }
foreach_detail_mode $testprefix {
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
@ -27,7 +29,7 @@ db func mit mit
sqlite3_fts5_register_matchinfo db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(content);
CREATE VIRTUAL TABLE t1 USING fts5(content, detail=%DETAIL%);
}
do_execsql_test 1.1 {
@ -41,7 +43,7 @@ do_execsql_test 1.1 {
# Now create an FTS4 table that does not specify matchinfo=fts3.
#
do_execsql_test 1.2 {
CREATE VIRTUAL TABLE t2 USING fts5(content);
CREATE VIRTUAL TABLE t2 USING fts5(content, detail=%DETAIL%);
INSERT INTO t2 SELECT * FROM t1;
SELECT mit(matchinfo(t2)) FROM t2 WHERE t2 MATCH 'I';
} {{1 1 1 2 2} {1 1 1 2 2}}
@ -149,9 +151,17 @@ proc normalize2 {list_of_lists} {
return $res
}
# Similar to [do_matchinfo_test], except that this is a no-op if the FTS5
# mode is not detail=full.
#
proc do_matchinfo_p_test {tn tbl expr results} {
if {[detail_is_full]} {
uplevel [list do_matchinfo_test $tn $tbl $expr $results]
}
}
do_execsql_test 4.1.0 {
CREATE VIRTUAL TABLE t4 USING fts5(x, y);
CREATE VIRTUAL TABLE t4 USING fts5(x, y, detail=%DETAIL%);
INSERT INTO t4 VALUES('a b c d e', 'f g h i j');
INSERT INTO t4 VALUES('f g h i j', 'a b c d e');
}
@ -185,7 +195,7 @@ do_matchinfo_test 4.1.1 t4 {t4 MATCH 'a b c'} {
xpxsscplax -
}
do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
do_matchinfo_p_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
p {1 1}
c {2 2}
x {
@ -203,8 +213,8 @@ do_matchinfo_test 4.1.2 t4 {t4 MATCH '"g h i"'} {
}
do_matchinfo_test 4.1.3 t4 {t4 MATCH 'a b'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
do_matchinfo_p_test 4.1.4 t4 {t4 MATCH '"a b" c'} { s {{2 0} {0 2}} }
do_matchinfo_p_test 4.1.5 t4 {t4 MATCH 'a "b c"'} { s {{2 0} {0 2}} }
do_matchinfo_test 4.1.6 t4 {t4 MATCH 'd d'} { s {{1 0} {0 1}} }
do_matchinfo_test 4.1.7 t4 {t4 MATCH 'f OR abcd'} {
x {
@ -220,7 +230,7 @@ do_matchinfo_test 4.1.8 t4 {t4 MATCH 'f NOT abcd'} {
}
do_execsql_test 4.2.0 {
CREATE VIRTUAL TABLE t5 USING fts5(content);
CREATE VIRTUAL TABLE t5 USING fts5(content, detail=%DETAIL%);
INSERT INTO t5 VALUES('a a a a a');
INSERT INTO t5 VALUES('a b a b a');
INSERT INTO t5 VALUES('c b c b c');
@ -233,7 +243,7 @@ do_matchinfo_test 4.2.1 t5 {t5 MATCH 'a a'} {
do_matchinfo_test 4.2.2 t5 {t5 MATCH 'a b'} { s {2} }
do_matchinfo_test 4.2.3 t5 {t5 MATCH 'a b a'} { s {3} }
do_matchinfo_test 4.2.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
do_matchinfo_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_p_test 4.2.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} }
do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
@ -250,7 +260,7 @@ if 0 {
do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} }
do_matchinfo_test 4.3.3 t5 {t5 MATCH 'a b a'} { s {3} }
do_matchinfo_test 4.3.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
do_matchinfo_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_p_test 4.3.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_test 4.3.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1 1} }
do_execsql_test 4.4.0.1 { INSERT INTO t5(t5) VALUES('optimize') }
@ -260,10 +270,10 @@ do_matchinfo_test 4.4.1 t5 {t5 MATCH 'a a'} { s {2 1} }
do_matchinfo_test 4.4.2 t5 {t5 MATCH 'a b'} { s {2} }
do_matchinfo_test 4.4.3 t5 {t5 MATCH 'a b a'} { s {3} }
do_matchinfo_test 4.4.4 t5 {t5 MATCH 'a a a'} { s {3 1} }
do_matchinfo_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_matchinfo_p_test 4.4.5 t5 {t5 MATCH '"a b" "a b"'} { s {2} }
do_execsql_test 4.5.0 {
CREATE VIRTUAL TABLE t6 USING fts5(a, b, c);
CREATE VIRTUAL TABLE t6 USING fts5(a, b, c, detail=%DETAIL%);
INSERT INTO t6 VALUES('a', 'b', 'c');
}
do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} }
@ -274,7 +284,7 @@ do_matchinfo_test 4.5.1 t6 {t6 MATCH 'a b c'} { s {{1 1 1}} }
# use the full-text index (i.e. lookup by rowid or full-table scan).
#
do_execsql_test 7.1 {
CREATE VIRTUAL TABLE t10 USING fts5(content);
CREATE VIRTUAL TABLE t10 USING fts5(content, detail=%DETAIL%);
INSERT INTO t10 VALUES('first record');
INSERT INTO t10 VALUES('second record');
}
@ -299,7 +309,7 @@ do_execsql_test 7.4 {
# SELECT sum(length(content)) < count(*) FROM fts4table;
#
do_execsql_test 8.1 {
CREATE VIRTUAL TABLE t11 USING fts5(content);
CREATE VIRTUAL TABLE t11 USING fts5(content, detail=%DETAIL%);
INSERT INTO t11(t11, rank) VALUES('pgsz', 32);
INSERT INTO t11 VALUES('quitealongstringoftext');
INSERT INTO t11 VALUES('anotherquitealongstringoftext');
@ -318,22 +328,24 @@ do_execsql_test 8.3 {
#-------------------------------------------------------------------------
do_execsql_test 9.1 {
CREATE VIRTUAL TABLE t12 USING fts5(content);
INSERT INTO t12 VALUES('a b c d');
SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {{0 1 1 0 1 1 1 1 1}}
do_execsql_test 9.2 {
INSERT INTO t12 VALUES('a d c d');
SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
{0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
}
do_execsql_test 9.3 {
INSERT INTO t12 VALUES('a d d a');
SELECT mit(matchinfo(t12, 'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
{0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
if {[detail_is_full]} {
do_execsql_test 9.1 {
CREATE VIRTUAL TABLE t12 USING fts5(content, detail=%DETAIL%);
INSERT INTO t12 VALUES('a b c d');
SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {{0 1 1 0 1 1 1 1 1}}
do_execsql_test 9.2 {
INSERT INTO t12 VALUES('a d c d');
SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
{0 2 2 0 3 2 1 2 2} {1 2 2 1 3 2 1 2 2}
}
do_execsql_test 9.3 {
INSERT INTO t12 VALUES('a d d a');
SELECT mit(matchinfo(t12,'x')) FROM t12 WHERE t12 MATCH 'NEAR(a d, 1) OR a';
} {
{0 4 3 0 5 3 1 4 3} {1 4 3 1 5 3 1 4 3} {2 4 3 2 5 3 2 4 3}
}
}
#---------------------------------------------------------------------------
@ -341,7 +353,7 @@ do_execsql_test 9.3 {
#
do_execsql_test 10.1 {
DROP TABLE t10;
CREATE VIRTUAL TABLE t10 USING fts5(idx, value);
CREATE VIRTUAL TABLE t10 USING fts5(idx, value, detail=%DETAIL%);
INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three');
SELECT t10.rowid, t10.*
FROM t10
@ -358,7 +370,7 @@ do_execsql_test 10.1 {
reset_db
sqlite3_fts5_register_matchinfo db
do_execsql_test 11.0 {
CREATE VIRTUAL TABLE tt USING fts5(x, y);
CREATE VIRTUAL TABLE tt USING fts5(x, y, detail=%DETAIL%);
INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1
INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2
INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3
@ -410,6 +422,8 @@ foreach {tn expr res} {
}
} {
if {[string match *:* $expr] && [detail_is_none]} continue
do_execsql_test 11.1.$tn.1 {
SELECT rowid, mit(matchinfo(tt, 'y')) FROM tt WHERE tt MATCH $expr
} $res
@ -443,7 +457,7 @@ db func mit mit
do_test 12.0 {
set cols [list]
for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])"
execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,], detail=%DETAIL%)"
} {}
do_execsql_test 12.1 {
@ -451,5 +465,7 @@ do_execsql_test 12.1 {
SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
} [list [list [expr 1<<4] [expr 1<<(45-32)]]]
} ;# foreach_detail_mode
finish_test

33
ext/fts5/test/fts5simple.test
View File

@ -18,7 +18,9 @@ ifcapable !fts5 {
finish_test
return
}
if 1 {
#-------------------------------------------------------------------------
#
set doc "x x [string repeat {y } 50]z z"
@ -350,6 +352,35 @@ do_execsql_test 4.1 {
SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'
} {0 {} 4}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 15.0 {
CREATE VIRTUAL TABLE x2 USING fts5(x, prefix=1);
INSERT INTO x2 VALUES('ab');
}
do_execsql_test 15.1 {
INSERT INTO x2(x2) VALUES('integrity-check');
}
}
#-------------------------------------------------------------------------
foreach_detail_mode $testprefix {
reset_db
fts5_aux_test_functions db
do_execsql_test 16.0 {
CREATE VIRTUAL TABLE x3 USING fts5(x, detail=%DETAIL%);
INSERT INTO x3 VALUES('a b c d e f');
}
do_execsql_test 16.1 {
SELECT fts5_test_poslist(x3) FROM x3('(a NOT b) OR c');
} {2.0.2}
do_execsql_test 16.1 {
SELECT fts5_test_poslist(x3) FROM x3('a OR c');
} {{0.0.0 1.0.2}}
}
finish_test

304
ext/fts5/test/fts5simple2.test Normal file
View File

@ -0,0 +1,304 @@
# 2015 September 05
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5simple2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
if 1 {
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a b c');
}
do_execsql_test 1.1 {
SELECT rowid FROM t1('c a b')
} {1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('b c d');
INSERT INTO t1 VALUES('b c d');
COMMIT;
}
do_execsql_test 2.1 {
SELECT rowid FROM t1('b c d')
} {1 2}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 3.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('b c d');
INSERT INTO t1 VALUES('b c d');
}
do_execsql_test 3.1 {
SELECT rowid FROM t1('b c d'); COMMIT;
} {1 2}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 4.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a3 b3 c3');
COMMIT;
}
do_execsql_test 4.1 {
SELECT rowid FROM t1('b*');
} {1 2 3}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 5.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
COMMIT;
}
do_execsql_test 5.1 { SELECT rowid FROM t1('b*') } {1 2 3}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 6.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=full);
BEGIN;
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a1 b1 c1');
COMMIT;
}
do_execsql_test 6.1 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 6.2 { SELECT rowid FROM t1('b1') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 6.3 { SELECT rowid FROM t1('c1') ORDER BY rowid DESC } {3 2 1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
BEGIN;
INSERT INTO t1 VALUES('a1 b1');
INSERT INTO t1 VALUES('a1 b2');
COMMIT;
}
do_execsql_test 7.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {2 1}
do_execsql_test 7.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {2 1}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 8.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 8.0.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.0.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.0.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 8.0.4 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC } {3 2 1}
do_execsql_test 8.0.5 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC } {3 1}
do_execsql_test 8.0.8 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC } {2}
do_execsql_test 8.1.0 { INSERT INTO t1(t1) VALUES('optimize') }
do_execsql_test 8.1.1 { SELECT rowid FROM t1('b*') } {1 2 3}
do_execsql_test 8.1.2 { SELECT rowid FROM t1('a1') } {1 3}
do_execsql_test 8.1.3 { SELECT rowid FROM t1('c2') } {2}
do_execsql_test 8.2.1 { SELECT rowid FROM t1('b*') ORDER BY rowid DESC} {3 2 1}
do_execsql_test 8.2.2 { SELECT rowid FROM t1('a1') ORDER BY rowid DESC} {3 1}
do_execsql_test 8.2.3 { SELECT rowid FROM t1('c2') ORDER BY rowid DESC} {2}
#--------------------------------------------------------------------------
#
reset_db
do_execsql_test 9.0.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1');
INSERT INTO t1 VALUES('a2 b2 c2');
INSERT INTO t1 VALUES('a1 b1 c1');
}
do_execsql_test 9.0.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
reset_db
do_execsql_test 9.1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=none);
INSERT INTO t1 VALUES('a1 b1 c1', 'x y z');
INSERT INTO t1 VALUES('a2 b2 c2', '1 2 3');
INSERT INTO t1 VALUES('a1 b1 c1', 'x 2 z');
}
do_execsql_test 9.2.1 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
#--------------------------------------------------------------------------
#
reset_db
do_execsql_test 10.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('b1');
INSERT INTO t1 VALUES('b1');
DELETE FROM t1 WHERE rowid=1;
}
do_execsql_test 10.1 {
SELECT rowid FROM t1('b1');
} {2}
do_execsql_test 10.2 {
SELECT rowid FROM t1('b1') ORDER BY rowid DESC;
} {2}
do_execsql_test 10.3 {
INSERT INTO t1(t1) VALUES('integrity-check');
} {}
#--------------------------------------------------------------------------
#
reset_db
do_execsql_test 11.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=none);
INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
WITH d(x,y) AS (
SELECT NULL, 'xyz' UNION ALL SELECT NULL, 'xyz' FROM d
)
INSERT INTO t1 SELECT * FROM d LIMIT 23;
}
#db eval { SELECT rowid AS r, quote(block) AS b FROM t1_data } { puts "$r: $b" }
do_execsql_test 11.2 {
SELECT rowid FROM t1;
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
do_execsql_test 11.3 {
SELECT rowid FROM t1('xyz');
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23}
do_execsql_test 11.4 {
INSERT INTO t1(t1) VALUES('integrity-check');
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 12.0 {
CREATE VIRTUAL TABLE yy USING fts5(x, detail=none);
INSERT INTO yy VALUES('in if');
INSERT INTO yy VALUES('if');
} {}
do_execsql_test 12.1 {
SELECT rowid FROM yy('i*');
} {1 2}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 13.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, prefix=1, detail=none);
} {}
foreach {rowid a} {
0 {f}
1 {u}
2 {k}
3 {a}
4 {a}
5 {u}
6 {u}
7 {u}
8 {f}
9 {f}
10 {a}
11 {p}
12 {f}
13 {u}
14 {a}
15 {a}
} {
do_execsql_test 13.1.$rowid {
INSERT INTO t1(rowid, a) VALUES($rowid, $a);
}
}
#-------------------------------------------------------------------------
#
reset_db
fts5_aux_test_functions db
do_execsql_test 14.0 {
CREATE VIRTUAL TABLE t1 USING fts5(a, detail=none);
INSERT INTO t1 VALUES('a b c d');
} {}
do_execsql_test 14.1 {
SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
} {0.0.1}
}
#-------------------------------------------------------------------------
#
reset_db
do_execsql_test 15.1 {
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=none);
BEGIN;
INSERT INTO t1(rowid, x) VALUES(1, 'sqlite');
INSERT INTO t1(rowid, x) VALUES(2, 'sqlite');
COMMIT;
} {}
do_test 15.1 {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
do_test 15.2 {
execsql { DELETE FROM t1 }
} {}
do_execsql_test 15.3.1 {
SELECT rowid FROM t1('sqlite');
} {}
do_execsql_test 15.3.2 {
SELECT rowid FROM t1('sqlite') ORDER BY rowid DESC;
} {}
do_test 15.4 {
execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
} {}
finish_test

154
ext/fts5/test/fts5synonym2.test Normal file
View File

@ -0,0 +1,154 @@
# 2014 Dec 20
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on custom tokenizers that support synonyms.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5synonym2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
#-------------------------------------------------------------------------
# Code for a simple Tcl tokenizer that supports synonyms at query time.
#
foreach SYNDICT {
{zero 0}
{one 1 i}
{two 2 ii}
{three 3 iii}
{four 4 iv}
{five 5 v}
{six 6 vi}
{seven 7 vii}
{eight 8 viii}
{nine 9 ix}
} {
foreach s $SYNDICT {
set o [list]
foreach x $SYNDICT {if {$x!=$s} {lappend o $x}}
set ::syn($s) $o
}
}
proc tcl_tokenize {tflags text} {
foreach {w iStart iEnd} [fts5_tokenize_split $text] {
sqlite3_fts5_token $w $iStart $iEnd
if {$tflags == "query"} {
foreach s $::syn($w) { sqlite3_fts5_token -colo $s $iStart $iEnd }
}
}
}
proc tcl_create {args} {
return "tcl_tokenize"
}
#
# End of tokenizer code.
#-------------------------------------------------------------------------
foreach_detail_mode $testprefix {
sqlite3_fts5_create_tokenizer db tcl tcl_create
fts5_aux_test_functions db
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE ss USING fts5(a, b, tokenize=tcl, detail=%DETAIL%);
INSERT INTO ss VALUES('5 5 five seven 3 seven i', '2 1 5 0 two 1 i');
INSERT INTO ss VALUES('six ix iii 7 i vii iii', 'one seven nine 4 9 1 vi');
INSERT INTO ss VALUES('6 viii i five six zero seven', '5 v iii iv iv 3');
INSERT INTO ss VALUES('9 ii six 8 1 6', 'six 4 iv iv 7');
INSERT INTO ss VALUES('1 5 4 eight ii iv iii', 'nine 2 eight ix v vii');
INSERT INTO ss VALUES('one 7 seven six 2 two', '1 2 four 7 4 3 4');
INSERT INTO ss VALUES('eight iv 4 nine vii six 1', '5 6 v one zero 4');
INSERT INTO ss VALUES('v 9 8 iii 4', '9 4 seven two vi vii');
INSERT INTO ss VALUES('3 ix two 9 0 nine i', 'five ii nine two viii i five');
INSERT INTO ss VALUES('six iii 9 two eight 2', 'nine i nine vii nine');
INSERT INTO ss VALUES('6 three zero seven vii five', '8 vii ix 0 7 seven');
INSERT INTO ss VALUES('8 vii 8 7 3 4', 'eight iii four viii nine iv three');
INSERT INTO ss VALUES('4 v 7 two 0 one 8', 'vii 1 two five i zero 9');
INSERT INTO ss VALUES('3 ii vii vi eight', '8 4 ix one three eight');
INSERT INTO ss VALUES('iv eight seven 6 9 seven', 'one vi two five seven');
INSERT INTO ss VALUES('i i 5 i v vii eight', '2 seven i 2 2 four');
INSERT INTO ss VALUES('0 i iii nine 3 ix five', '0 eight iv 0 six 2');
INSERT INTO ss VALUES('iv vii three 3 9 one 8', '2 ii 6 eight ii six six');
INSERT INTO ss VALUES('eight one two nine six', '8 9 3 viii vi');
INSERT INTO ss VALUES('one 0 four ii eight one 3', 'iii eight vi vi vi');
INSERT INTO ss VALUES('4 0 eight 0 0', '1 four one vii seven ii');
INSERT INTO ss VALUES('1 zero nine 2 2', 'viii iv two vi nine v iii');
INSERT INTO ss VALUES('5 five viii four four vi', '8 five 7 vii 6 4');
INSERT INTO ss VALUES('7 ix four 8 vii', 'nine three nine ii ix vii');
INSERT INTO ss VALUES('nine iv v i 0 v', 'two iv vii six i ix 4');
INSERT INTO ss VALUES('one v v one viii 3 8', '2 1 3 five iii');
INSERT INTO ss VALUES('six ii 5 nine 4 viii seven', 'eight i ix ix 7 four');
INSERT INTO ss VALUES('9 ii two seven three 7 0', 'six viii seven 7 five');
INSERT INTO ss VALUES('five two 4 viii nine', '9 7 nine zero 1 two one');
INSERT INTO ss VALUES('viii 8 iii i ii 8 3', '4 2 7 v 8 8');
INSERT INTO ss VALUES('four vii 4 iii zero 0 vii', '3 viii iii zero 9 i');
INSERT INTO ss VALUES('0 seven v five i five v', 'one 4 2 ix 9');
INSERT INTO ss VALUES('two 5 two two ix 4 1', '3 nine ii v nine 3 five');
INSERT INTO ss VALUES('five 5 7 4 6 vii', 'three 2 ix 2 8 6');
INSERT INTO ss VALUES('six iii vi iv seven eight', '8 six 7 0 4');
INSERT INTO ss VALUES('vi vi iv 3 0 one one', '9 6 eight ix iv');
INSERT INTO ss VALUES('7 2 2 iii 0', '0 0 seven 1 nine');
INSERT INTO ss VALUES('8 6 iv six ii', 'iv 6 3 4 ii five');
INSERT INTO ss VALUES('0 two two seven ii', 'vii ix four 4 zero vi vi');
INSERT INTO ss VALUES('2 one eight 8 9 7', 'vi 3 0 3 vii');
INSERT INTO ss VALUES('iii ii ix iv three', 'vi i 6 1 two');
INSERT INTO ss VALUES('eight four nine 8 seven', 'one three i nine iii one');
INSERT INTO ss VALUES('iii seven five ix 8', 'ii 7 seven 0 four ii');
INSERT INTO ss VALUES('four 0 1 5 two', 'iii 9 5 ii ii 2 4');
INSERT INTO ss VALUES('iii nine four vi 8 five six', 'i i ii seven vi vii');
INSERT INTO ss VALUES('eight vii eight six 3', 'i vii 1 six 9 vii');
INSERT INTO ss VALUES('9 0 viii viii five', 'i 1 viii ix 3 4');
INSERT INTO ss VALUES('three nine 5 nine viii four zero', 'ii i 1 5 2 viii');
INSERT INTO ss VALUES('5 vii three 9 four', 'three five one 7 2 eight one');
}
foreach {tn expr} {
1.1 "one" 1.2 "two" 1.3 "three" 1.4 "four"
1.5 "v" 1.6 "vi" 1.7 "vii" 1.8 "viii"
1.9 "9" 1.10 "0" 1.11 "1" 1.12 "2"
2.1 "one OR two OR three OR four"
2.2 "(one AND two) OR (three AND four)"
2.3 "(one AND two) OR (three AND four) NOT five"
2.4 "(one AND two) NOT 6"
3.1 "b:one AND a:two"
3.2 "b:one OR a:two"
3.3 "a:one OR b:1 OR {a b} : i"
4.1 "NEAR(one two, 2)"
4.2 "NEAR(one two three, 2)"
4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
} {
if {[fts5_expr_ok $expr ss]==0} {
do_test 1.$tn.OMITTED { list } [list]
continue
}
set res [fts5_query_data $expr ss ASC ::syn]
breakpoint
do_execsql_test 1.$tn.[llength $res].asc {
SELECT rowid, fts5_test_poslist(ss), fts5_test_collist(ss) FROM ss($expr)
} $res
}
}
finish_test

104
ext/fts5/test/fts5vocab.test
View File

@ -21,9 +21,44 @@ ifcapable !fts5 {
return
}
foreach_detail_mode $testprefix {
proc null_list_entries {iFirst nInterval L} {
for {set i $iFirst} {$i < [llength $L]} {incr i $nInterval} {
lset L $i {}
}
return $L
}
proc star_from_row {L} {
if {[detail_is_full]==0} {
set L [null_list_entries 2 3 $L]
}
return $L
}
proc star_from_col {L} {
if {[detail_is_col]} {
set L [null_list_entries 3 4 $L]
}
if {[detail_is_none]} {
set L [null_list_entries 1 4 $L]
set L [null_list_entries 3 4 $L]
}
return $L
}
proc row_to_col {L} {
if {[detail_is_none]==0} { error "this is for detail=none mode" }
set ret [list]
foreach {a b c} $L {
lappend ret $a {} $b {}
}
set ret
}
do_execsql_test 1.1.1 {
CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1);
CREATE VIRTUAL TABLE t1 USING fts5(one, prefix=1, detail=%DETAIL%);
CREATE VIRTUAL TABLE v1 USING fts5vocab(t1, 'row');
PRAGMA table_info = v1;
} {
@ -52,32 +87,32 @@ do_execsql_test 1.3 {
do_execsql_test 1.4.1 {
SELECT * FROM v1;
} {x 2 4 y 1 1 z 1 1}
} [star_from_row {x 2 4 y 1 1 z 1 1}]
do_execsql_test 1.4.2 {
SELECT * FROM v2;
} {x one 2 4 y one 1 1 z one 1 1}
} [star_from_col {x one 2 4 y one 1 1 z one 1 1}]
do_execsql_test 1.5.1 {
BEGIN;
INSERT INTO t1 VALUES('a b c');
SELECT * FROM v1 WHERE term<'d';
} {a 1 1 b 1 1 c 1 1}
} [star_from_row {a 1 1 b 1 1 c 1 1}]
do_execsql_test 1.5.2 {
SELECT * FROM v2 WHERE term<'d';
COMMIT;
} {a one 1 1 b one 1 1 c one 1 1}
} [star_from_col {a one 1 1 b one 1 1 c one 1 1}]
do_execsql_test 1.6 {
DELETE FROM t1 WHERE one = 'a b c';
SELECT * FROM v1;
} {x 2 4 y 1 1 z 1 1}
} [star_from_row {x 2 4 y 1 1 z 1 1}]
#-------------------------------------------------------------------------
#
do_execsql_test 2.0 {
CREATE VIRTUAL TABLE tt USING fts5(a, b);
CREATE VIRTUAL TABLE tt USING fts5(a, b, detail=%DETAIL%);
INSERT INTO tt VALUES('d g b f d f', 'f c e c d a');
INSERT INTO tt VALUES('f a e a a b', 'e d c f d d');
INSERT INTO tt VALUES('b c a a a b', 'f f c c b c');
@ -90,7 +125,12 @@ do_execsql_test 2.0 {
INSERT INTO tt VALUES('c c a a c f', 'd g a e b g');
}
set res_col {
set res_row [star_from_row {
a 10 20 b 9 14 c 9 20 d 9 19
e 8 13 f 10 20 g 7 14 x 1 1
y 1 1
}]
set res_col [star_from_col {
a a 6 11 a b 7 9
b a 6 7 b b 7 7
c a 6 12 c b 5 8
@ -99,11 +139,9 @@ set res_col {
f a 9 10 f b 7 10
g a 5 7 g b 5 7
x a 1 1 y b 1 1
}
set res_row {
a 10 20 b 9 14 c 9 20 d 9 19
e 8 13 f 10 20 g 7 14 x 1 1
y 1 1
}]
if {[detail_is_none]} {
set res_col [row_to_col $res_row]
}
foreach {tn tbl resname} {
@ -153,9 +191,9 @@ reset_db
forcedelete test.db2
do_execsql_test 5.0 {
ATTACH 'test.db2' AS aux;
CREATE VIRTUAL TABLE t1 USING fts5(x);
CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
CREATE VIRTUAL TABLE aux.t1 USING fts5(x);
CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
CREATE VIRTUAL TABLE aux.t1 USING fts5(x, detail=%DETAIL%);
INSERT INTO main.t1 VALUES('a b c');
INSERT INTO main.t1 VALUES('d e f');
@ -178,18 +216,18 @@ do_execsql_test 5.1 {
CREATE VIRTUAL TABLE temp.va USING fts5vocab(aux, t1, row);
}
do_execsql_test 5.2 { SELECT * FROM vm } {
do_execsql_test 5.2 { SELECT * FROM vm } [star_from_row {
a 2 2 b 1 1 c 2 2 d 1 1 e 2 2 f 1 1
}
do_execsql_test 5.3 { SELECT * FROM vt1 } {
}]
do_execsql_test 5.3 { SELECT * FROM vt1 } [star_from_row {
1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
}
do_execsql_test 5.4 { SELECT * FROM vt2 } {
}]
do_execsql_test 5.4 { SELECT * FROM vt2 } [star_from_row {
1 2 2 2 1 1 3 2 2 4 1 1 5 2 2 6 1 1
}
do_execsql_test 5.5 { SELECT * FROM va } {
}]
do_execsql_test 5.5 { SELECT * FROM va } [star_from_row {
m 1 1 n 2 2 o 1 1 x 2 2 y 1 1 z 2 2
}
}]
#-------------------------------------------------------------------------
#
@ -218,7 +256,7 @@ do_catchsql_test 6.2 {
# constraints in the WHERE clause).
#
do_execsql_test 7.0 {
CREATE VIRTUAL TABLE tx USING fts5(one, two);
CREATE VIRTUAL TABLE tx USING fts5(one, two, detail=%DETAIL%);
INSERT INTO tx VALUES('g a ggg g a b eee', 'cc d aa ff g ee');
INSERT INTO tx VALUES('dd fff i a i jjj', 'f fff hh jj e f');
INSERT INTO tx VALUES('ggg a f f fff dd aa', 'd ggg f f j gg ddd');
@ -276,6 +314,10 @@ foreach {term} {
if {[lindex $r2 2]==0} {set r2 [list]}
set resc [concat $r1 $r2]
set resc [star_from_col $resc]
set resr [star_from_row $resr]
if {[detail_is_none]} { set resc [row_to_col $resr] }
do_execsql_test 7.$term.1 {SELECT * FROM txc WHERE term=$term} $resc
do_execsql_test 7.$term.2 {SELECT * FROM txr WHERE term=$term} $resr
}
@ -340,10 +382,14 @@ do_execsql_test 7.3.1 {
SELECT count(*) FROM txr, txr_c WHERE txr.term = txr_c.term;
} {30}
do_execsql_test 7.3.2 {
SELECT count(*) FROM txc, txc_c
WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
} {57}
if {![detail_is_none]} {
do_execsql_test 7.3.2 {
SELECT count(*) FROM txc, txc_c
WHERE txc.term = txc_c.term AND txc.col=txc_c.col;
} {57}
}
}
finish_test

59
ext/fts5/tool/fts5speed.tcl Normal file
View File

@ -0,0 +1,59 @@
set Q {
{1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron'"}
{25 "SELECT count(*) FROM t1 WHERE t1 MATCH 'hours'"}
{300 "SELECT count(*) FROM t1 WHERE t1 MATCH 'acid'"}
{100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'loaned OR mobility OR popcore OR sunk'"}
{100 "SELECT count(*) FROM t1 WHERE t1 MATCH 'enron AND myapps'"}
{1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'en* AND my*'"}
{1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'c:t*'"}
{1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t* OR b:t* OR c:t* OR d:t* OR e:t* OR f:t* OR g:t*'"}
{1 "SELECT count(*) FROM t1 WHERE t1 MATCH 'a:t*'"}
}
proc usage {} {
global Q
puts stderr "Usage: $::argv0 DATABASE QUERY"
puts stderr ""
for {set i 1} {$i <= [llength $Q]} {incr i} {
puts stderr " $i. [lindex $Q [expr $i-1]]"
}
puts stderr ""
exit -1
}
set nArg [llength $argv]
if {$nArg!=2 && $nArg!=3} usage
set database [lindex $argv 0]
set iquery [lindex $argv 1]
if {$iquery<1 || $iquery>[llength $Q]} usage
set nRepeat 0
if {$nArg==3} { set nRepeat [lindex $argv 2] }
sqlite3 db $database
catch { load_static_extension db fts5 }
incr iquery -1
set sql [lindex $Q $iquery 1]
if {$nRepeat==0} {
set nRepeat [lindex $Q $iquery 0]
}
puts "sql: $sql"
puts "nRepeat: $nRepeat"
if {[regexp matchinfo $sql]} {
sqlite3_fts5_register_matchinfo db
db eval $sql
} else {
puts "result: [db eval $sql]"
}
for {set i 1} {$i < $nRepeat} {incr i} {
db eval $sql
}

78
manifest
View File

@ -1,5 +1,5 @@
C Add\sthe\s".vfslist"\scommand\sto\sthe\scommand-line\sshell.
D 2016-01-12T19:37:20.761
C Add\sthe\s"detail"\soption\sto\sfts5.\sUsed\sto\sreduce\sthe\samount\sof\sinformation\sstored\sin\san\sfts5\sindex.
D 2016-01-12T19:45:01.185
F Makefile.in 7c8cc4c2f0179efc6fa9492141d1fb65f4807054
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc e45d8b9b56dfa3f2cd860b2c28bd9d304513b042
@ -96,39 +96,39 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h 8b9a13b309b180e9fb88ea5666c0d8d73c6102d9
F ext/fts5/fts5Int.h acf968e43d57b6b1caf7554d34ec35d6ed3b4fe8
F ext/fts5/fts5.h 88fec577a2148f34df75930dc2b0c45b0bad72c3
F ext/fts5/fts5Int.h 3918045ebceb1d600c9c1c1b460489ff0c788e96
F ext/fts5/fts5_aux.c 2dafc3aee0c70d643140c77d8d70daffa51a9e9e
F ext/fts5/fts5_buffer.c 1e49512a535045e621246dc7f4f65f3593fa0fc2
F ext/fts5/fts5_config.c 0ee66188609a62342e9f9aeefa3c3e44518a4dd6
F ext/fts5/fts5_expr.c 8228aca3e9af626a1ca9b093d4d24b4f2488ad23
F ext/fts5/fts5_hash.c 25838d525e97f8662ff3504be94d0bad24f9a37e
F ext/fts5/fts5_index.c 578f46697080f11a1e26cd45a1c039c043a3111d
F ext/fts5/fts5_main.c e11b525778e6fce416d916bfa96926bd1ce4616d
F ext/fts5/fts5_storage.c 57c636d87cbb829d6147c8c8bf78fa53b9ebb526
F ext/fts5/fts5_tcl.c 3bf445e66de32137d4693694ff7b1fd6074e32bd
F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf
F ext/fts5/fts5_buffer.c 87204c8b3b8bc62b27376eab09b74d6d5acc41f1
F ext/fts5/fts5_config.c b0ed7b0ddd785fb4d4e6f9037d357f8aa95918e6
F ext/fts5/fts5_expr.c 6eba2220747ea1b20a358fb3b34b2ab78323e285
F ext/fts5/fts5_hash.c 1b113977296cf4212c6ec667d5e3f2bd18036955
F ext/fts5/fts5_index.c 5ab044a67919e2c9f42f0288a39778136154511e
F ext/fts5/fts5_main.c 03bd44e4bd0ba16213ca9259ad5df1d4d743fd7e
F ext/fts5/fts5_storage.c f7b2d330dd7b29a9f4da09f6d85879ca8c41b2e8
F ext/fts5/fts5_tcl.c 18e9382d8cdad4c05b49559c68494968b9b4a4fb
F ext/fts5/fts5_test_mi.c 1ec66ffdf7632077fbd773b7a6df5153272ec070
F ext/fts5/fts5_tokenize.c 504984ac6993323247221eebe3cd55bead01b5f8
F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
F ext/fts5/fts5_vocab.c 3742d0abfe8aa8c3cb4a7df56aa38f2e3c3fb1c2
F ext/fts5/fts5_vocab.c da64ecbd217625980a1721fbd588a1e4118a51b6
F ext/fts5/fts5parse.y 1647eba089b9b3fc058b4dc989d9da87d15b9580
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl 51f7ef3af444b89c6f6ce3896a0ac349ff4e996d
F ext/fts5/test/fts5aa.test 2c553eea4dab4bc5a75928f56729277c7bc1d206
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
F ext/fts5/test/fts5ad.test e3dfb150fce971b4fd832498c29f56924d451b63
F ext/fts5/test/fts5ae.test 0a9984fc3479f89f8c63d9848d6ed0c465dfcebe
F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a
F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505
F ext/fts5/test/fts5ah.test e592c4978622dbc4de552cd0f9395df60ac5d54c
F ext/fts5/test/fts5ai.test f20e53bbf0c55bc596f1fd47f2740dae028b8f37
F ext/fts5/test/fts5_common.tcl 393882afb225a21edf033043bbf936951e9198c1
F ext/fts5/test/fts5aa.test 7e814df4a0e6c22a6fe2d84f210fdc0b5068a084
F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
F ext/fts5/test/fts5ac.test d5073ca7bd2d9fe8aab0c82c6c75a7e4b0d70ced
F ext/fts5/test/fts5ad.test 0ddaa5b692ff220100ee396228838f4331399eaa
F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
F ext/fts5/test/fts5af.test be858a96b1f5de66ba6d64f0021bd8b2408e126c
F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
F ext/fts5/test/fts5aj.test 05b569f5c16ea3098fb1984eec5cf50dbdaae5d8
F ext/fts5/test/fts5ak.test 7b8c5df96df599293f920b7e5521ebc79f647592
F ext/fts5/test/fts5al.test a1b7b6393376bc2adc216527a28f5ae5594069df
F ext/fts5/test/fts5ak.test fb26389985407826f6076bb9f382c67d3db6b5d9
F ext/fts5/test/fts5al.test 18c277f5986df0a3d9071dfd7128afeb16fe9d5d
F ext/fts5/test/fts5alter.test 6022c61467a82aa11c70822ccad22b328dcf0d04
F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
F ext/fts5/test/fts5auto.test 401c20e89f1114d733b94809be1e6f893e16c09e
F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
@ -139,10 +139,11 @@ F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
F ext/fts5/test/fts5corrupt3.test a2b537c120bdd43c79c42fe2438d7b8c81fe5599
F ext/fts5/test/fts5dlidx.test ecba5e62ea8b26c33829961602069c546228046d
F ext/fts5/test/fts5detail.test 4e971d28e7336c61ab916fc287900355dab7054d
F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
F ext/fts5/test/fts5eb.test 021aa80b7ac09b964249aa32ced9ee908703e4aa
F ext/fts5/test/fts5fault1.test 4b39c47ca3544615daa8a2f733b911fa08022c77
F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
@ -150,10 +151,11 @@ F ext/fts5/test/fts5fault4.test 4864f2b5c2c083440dbe85aff60897bc1aa04603
F ext/fts5/test/fts5fault5.test f2b8645053d48982e8979749e93994c43011c118
F ext/fts5/test/fts5fault6.test 9682664d679643ac6736e90c225526cc84073cda
F ext/fts5/test/fts5fault7.test 01be274bfc8d9bf22451a3bf5892e9399d044f1b
F ext/fts5/test/fts5fault8.test aeb4717b7b293678bc4d2f3c0159206a525375d9
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
F ext/fts5/test/fts5hash.test 7cf4607b8657c383f0b520668a99971e95d8b139
F ext/fts5/test/fts5integrity.test 87db5d4e7da0ce04a1dcba5ba91658673c997a65
F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2
F ext/fts5/test/fts5matchinfo.test 86569026d20f1ed748236587ce798de8a96615f1
F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
F ext/fts5/test/fts5onepass.test 7ed9608e258132cb8d55e7c479b08676ad68810c
@ -168,15 +170,18 @@ F ext/fts5/test/fts5rank.test 7e9e64eac7245637f6f2033aec4b292aaf611aab
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test 400384798349d658eaf06aefa1e364957d5d4821
F ext/fts5/test/fts5simple.test 9bded45827b4ab8933c87b7b3bcc3cd47f7378a4
F ext/fts5/test/fts5simple.test 2bc6451cbe887a9215f5b14ae307c70d850344c9
F ext/fts5/test/fts5simple2.test 843f1f7fe439ff32bf74f4fd6430632f9636ef3a
F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
F ext/fts5/test/fts5synonym2.test d2d9099d9d105b55ea03fd52d61ae2847d534129
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
F ext/fts5/test/fts5version.test 978f59541d8cef7e8591f8be2115ec5ccb863e2e
F ext/fts5/test/fts5vocab.test c88a5554d0409494da95ba647bbdb4879b2624b0
F ext/fts5/test/fts5vocab.test e4b12f238f113795615ba6343b63fb326d6a360e
F ext/fts5/tool/fts5speed.tcl aaee41894b552df8fbf8616aad003b2ea9ba3221
F ext/fts5/tool/fts5txt2db.tcl c374c4c4797e8cdfadabdfaeeb5412dcd6686e84
F ext/fts5/tool/loadfts5.tcl 4cc2d6af43b58d4fac05bc4fdabd0e5862c3b2c1
F ext/fts5/tool/mkfts5c.tcl d1c2a9ab8e0ec690a52316f33dd9b1d379942f45
@ -1407,7 +1412,8 @@ F tool/vdbe_profile.tcl 246d0da094856d72d2c12efec03250d71639d19f
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 4bb2c1df105c2d21f5c4c7ef656ff1d6e15f78bc
R 74300fef6ffb708c0011be0eb0839258
U drh
Z 0c47ae55d5f3f2500ab8561940e3da5a
P 5727562b75edf25102cd72607c420d245379c96d bc5118f40a11f64ffb4e1c086277fa80b9764745
R 32b6994fdd76448ebd10466a9fb05893
T +closed bc5118f40a11f64ffb4e1c086277fa80b9764745
U dan
Z d6aecf5b74614ca62cb2aeb2f23739db

2
manifest.uuid
View File

@ -1 +1 @@
5727562b75edf25102cd72607c420d245379c96d
a220e85fe535af5ef2da6ef5fb76abe5a96b5abf