mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Have fts5 cache the decoded structure of fts5 indexes in memory. Use "PRAGMA data_version" to detect stale caches.

Browse Source 
FossilOrigin-Name: 33ef2210ef19e55c8d460bfe9d3dc146034c8acc
This commit is contained in:
dan 2016-03-16 19:48:10 +00:00
parent 32f57d4c37
commit f55fb6615b
8 changed files with 153 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ext/fts5/fts5Int.h
View File

@ -480,6 +480,7 @@ int sqlite3Fts5IndexReads(Fts5Index *p);
int sqlite3Fts5IndexReinit(Fts5Index *p);
int sqlite3Fts5IndexOptimize(Fts5Index *p);
int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge);
int sqlite3Fts5IndexReset(Fts5Index *p);
int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
@ -622,6 +623,7 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p);
int sqlite3Fts5StorageRebuild(Fts5Storage *p);
int sqlite3Fts5StorageOptimize(Fts5Storage *p);
int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
int sqlite3Fts5StorageReset(Fts5Storage *p);
/*
** End of interface to code in fts5_storage.c.

115
ext/fts5/fts5_index.c
View File

@ -304,6 +304,10 @@ struct Fts5Index {
sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */
sqlite3_stmt *pIdxSelect;
int nRead; /* Total number of blocks read */
sqlite3_stmt *pDataVersion;
i64 iStructVersion; /* data_version when pStruct read */
Fts5Structure *pStruct; /* Current db structure (or NULL) */
};
struct Fts5DoclistIter {
@ -959,6 +963,48 @@ static void fts5StructureExtendLevel(
}
}
static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){
Fts5Structure *pRet = 0;
Fts5Config *pConfig = p->pConfig;
int iCookie; /* Configuration cookie */
Fts5Data *pData;
pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
if( p->rc==SQLITE_OK ){
/* TODO: Do we need this if the leaf-index is appended? Probably... */
memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
}
fts5DataRelease(pData);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
}
}
return pRet;
}
static i64 fts5IndexDataVersion(Fts5Index *p){
i64 iVersion = 0;
if( p->pDataVersion==0 ){
p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion,
sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb)
);
if( p->rc ) return 0;
}
if( SQLITE_ROW==sqlite3_step(p->pDataVersion) ){
iVersion = sqlite3_column_int64(p->pDataVersion, 0);
}
p->rc = sqlite3_reset(p->pDataVersion);
return iVersion;
}
/*
** Read, deserialize and return the structure record.
**
@ -971,28 +1017,51 @@ static void fts5StructureExtendLevel(
** is called, it is a no-op.
*/
static Fts5Structure *fts5StructureRead(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
Fts5Structure *pRet = 0; /* Object to return */
int iCookie; /* Configuration cookie */
Fts5Data *pData;
Fts5Structure *pRet; /* Object to return */
pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
if( p->rc ) return 0;
/* TODO: Do we need this if the leaf-index is appended? Probably... */
memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING);
p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
if( p->pStruct ){
pRet = p->pStruct;
#ifdef SQLITE_DEBUG
{
Fts5Structure *pTest = fts5StructureReadUncached(p);
if( pTest ){
int i, j;
assert_nc( pRet->nSegment==pTest->nSegment );
assert_nc( pRet->nLevel==pTest->nLevel );
for(i=0; i<pTest->nLevel; i++){
assert_nc( pRet->aLevel[i].nMerge==pTest->aLevel[i].nMerge );
assert_nc( pRet->aLevel[i].nSeg==pTest->aLevel[i].nSeg );
for(j=0; j<pTest->aLevel[i].nSeg; j++){
Fts5StructureSegment *p1 = &pTest->aLevel[i].aSeg[j];
Fts5StructureSegment *p2 = &pRet->aLevel[i].aSeg[j];
assert_nc( p1->iSegid==p2->iSegid );
assert_nc( p1->pgnoFirst==p2->pgnoFirst );
assert_nc( p1->pgnoLast==p2->pgnoLast );
}
}
fts5StructureRelease(pTest);
}
}
#endif
}else{
pRet = fts5StructureReadUncached(p);
}
fts5DataRelease(pData);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
if( pRet ){
fts5StructureRef(pRet);
p->pStruct = pRet;
p->iStructVersion = fts5IndexDataVersion(p);
}
return pRet;
}
static void fts5StructureInvalidate(Fts5Index *p){
if( p->pStruct ){
fts5StructureRelease(p->pStruct);
p->pStruct = 0;
}
}
/*
** Return the total number of segments in index structure pStruct. This
** function is only ever used as part of assert() conditions.
@ -4328,6 +4397,7 @@ static void fts5FlushOneHash(Fts5Index *p){
** for the new level-0 segment. */
pStruct = fts5StructureRead(p);
iSegid = fts5AllocateSegid(p, pStruct);
fts5StructureInvalidate(p);
if( iSegid ){
const int pgsz = p->pConfig->pgsz;
@ -4547,6 +4617,7 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){
assert( p->rc==SQLITE_OK );
fts5IndexFlush(p);
pStruct = fts5StructureRead(p);
fts5StructureInvalidate(p);
if( pStruct ){
pNew = fts5IndexOptimizeStruct(p, pStruct);
@ -4577,6 +4648,7 @@ int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){
Fts5Structure *pStruct = fts5StructureRead(p);
if( pStruct ){
int nMin = p->pConfig->nUsermerge;
fts5StructureInvalidate(p);
if( nMerge<0 ){
Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct);
fts5StructureRelease(pStruct);
@ -5004,6 +5076,7 @@ int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){
int sqlite3Fts5IndexRollback(Fts5Index *p){
fts5CloseReader(p);
fts5IndexDiscardData(p);
fts5StructureInvalidate(p);
/* assert( p->rc==SQLITE_OK ); */
return SQLITE_OK;
}
@ -5015,6 +5088,7 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){
*/
int sqlite3Fts5IndexReinit(Fts5Index *p){
Fts5Structure s;
fts5StructureInvalidate(p);
memset(&s, 0, sizeof(Fts5Structure));
fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
fts5StructureWrite(p, &s);
@ -5073,11 +5147,13 @@ int sqlite3Fts5IndexClose(Fts5Index *p){
int rc = SQLITE_OK;
if( p ){
assert( p->pReader==0 );
fts5StructureInvalidate(p);
sqlite3_finalize(p->pWriter);
sqlite3_finalize(p->pDeleter);
sqlite3_finalize(p->pIdxWriter);
sqlite3_finalize(p->pIdxDeleter);
sqlite3_finalize(p->pIdxSelect);
sqlite3_finalize(p->pDataVersion);
sqlite3Fts5HashFree(p->pHash);
sqlite3_free(p->zDataTbl);
sqlite3_free(p);
@ -6333,3 +6409,12 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
}
return rc;
}
int sqlite3Fts5IndexReset(Fts5Index *p){
assert( p->pStruct==0 || p->iStructVersion!=0 );
if( fts5IndexDataVersion(p)!=p->iStructVersion ){
fts5StructureInvalidate(p);
}
return fts5IndexReturn(p);
}

39
ext/fts5/fts5_main.c
View File

@ -597,27 +597,38 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
return SQLITE_OK;
}
static int fts5NewTransaction(Fts5Table *pTab){
Fts5Cursor *pCsr;
for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){
if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK;
}
return sqlite3Fts5StorageReset(pTab->pStorage);
}
/*
** Implementation of xOpen method.
*/
static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){
Fts5Table *pTab = (Fts5Table*)pVTab;
Fts5Config *pConfig = pTab->pConfig;
Fts5Cursor *pCsr; /* New cursor object */
Fts5Cursor *pCsr = 0; /* New cursor object */
int nByte; /* Bytes of space to allocate */
int rc = SQLITE_OK; /* Return code */
int rc; /* Return code */
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
memset(pCsr, 0, nByte);
pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
}else{
rc = SQLITE_NOMEM;
rc = fts5NewTransaction(pTab);
if( rc==SQLITE_OK ){
nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int);
pCsr = (Fts5Cursor*)sqlite3_malloc(nByte);
if( pCsr ){
Fts5Global *pGlobal = pTab->pGlobal;
memset(pCsr, 0, nByte);
pCsr->aColumnSize = (int*)&pCsr[1];
pCsr->pNext = pGlobal->pCsr;
pGlobal->pCsr = pCsr;
pCsr->iCsrId = ++pGlobal->iNextId;
}else{
rc = SQLITE_NOMEM;
}
}
*ppCsr = (sqlite3_vtab_cursor*)pCsr;
return rc;
@ -1578,8 +1589,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
** Implementation of xBegin() method.
*/
static int fts5BeginMethod(sqlite3_vtab *pVtab){
UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */
fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0);
fts5NewTransaction((Fts5Table*)pVtab);
return SQLITE_OK;
}

4
ext/fts5/fts5_storage.c
View File

@ -639,6 +639,10 @@ int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge){
return sqlite3Fts5IndexMerge(p->pIndex, nMerge);
}
int sqlite3Fts5StorageReset(Fts5Storage *p){
return sqlite3Fts5IndexReset(p->pIndex);
}
/*
** Allocate a new rowid. This is used for "external content" tables when
** a NULL value is inserted into the rowid column. The new rowid is allocated

4
ext/fts5/test/fts5corrupt3.test
View File

@ -179,6 +179,10 @@ for {set i 1} {1} {incr i} {
if {$end<=$i} break
lset var end [expr $end - $i]
set struct [binary format c* $var]
db close
sqlite3 db test.db
db eval {
BEGIN;
UPDATE t1_data SET block = $struct WHERE id=10;

4
ext/fts5/test/fts5dlidx.test
View File

@ -178,7 +178,7 @@ do_execsql_test 3.2 {
ORDER BY rowid DESC;
} {16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1}
do_execsql_test 3.2 {
do_execsql_test 3.3 {
INSERT INTO abc(abc) VALUES('integrity-check');
INSERT INTO abc(abc) VALUES('optimize');
INSERT INTO abc(abc) VALUES('integrity-check');
@ -187,7 +187,7 @@ do_execsql_test 3.2 {
set v [lindex $vocab 0]
set i 0
foreach v $vocab {
do_execsql_test 3.3.[incr i] {
do_execsql_test 3.4.[incr i] {
SELECT rowid FROM abc WHERE abc MATCH $v
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16}
}

27
manifest
View File

@ -1,5 +1,5 @@
C Add\sthe\sSQLITE_OMIT_CODEC_FROM_TCL\scompile-time\soption.
D 2016-03-16T01:03:10.251
C Have\sfts5\scache\sthe\sdecoded\sstructure\sof\sfts5\sindexes\sin\smemory.\sUse\s"PRAGMA\sdata_version"\sto\sdetect\sstale\scaches.
D 2016-03-16T19:48:10.873
F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@ -98,15 +98,15 @@ F ext/fts3/unicode/mkunicode.tcl 2debed3f582d77b3fdd0b8830880250021571fd8
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h ff9c2782e8ed890b0de2f697a8d63971939e70c7
F ext/fts5/fts5Int.h 4060504b7979601d99e1385c2b5713036854979a
F ext/fts5/fts5Int.h 4e507abebae0d7d3ac9b8daebf049d5153d00961
F ext/fts5/fts5_aux.c daa57fb45216491814520bbb587e97bf81ced458
F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
F ext/fts5/fts5_expr.c be309fb227003c931107bfcc12d5be4f2fd2bb8c
F ext/fts5/fts5_hash.c f3a7217c86eb8f272871be5f6aa1b6798960a337
F ext/fts5/fts5_index.c d4f0c12e4f04bbc3a06b6da052039f2ce3e45438
F ext/fts5/fts5_main.c b8501e1a6a11591c53b18ce7aea7e5386cfb0421
F ext/fts5/fts5_storage.c f8343db90d8c95a4d4b52f6676e354b4649ffd6e
F ext/fts5/fts5_index.c 317040cb17cdad05e973376b39239b29c75f90b5
F ext/fts5/fts5_main.c b4a0fc5bf17f2f1f056ee76cdd7d2af08b360f55
F ext/fts5/fts5_storage.c a3361410422e69639ca2bcd5a56a0933dadf84d2
F ext/fts5/fts5_tcl.c f8731e0508299bd43f1a2eff7dbeaac870768966
F ext/fts5/fts5_test_mi.c 783b86697ebf773c18fc109992426c0173a055bc
F ext/fts5/fts5_test_tok.c db08af63673c3a7d39f053b36fd6e065017706be
@ -141,9 +141,9 @@ F ext/fts5/test/fts5conflict.test 26f4e46c4d31e16221794832a990dc4e30e18de5
F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test c2ad090192708150d50d961278df10ae7a4b8b62
F ext/fts5/test/fts5corrupt2.test 26c0a39dd9ff73207e6229f83b50b21d37c7658c
F ext/fts5/test/fts5corrupt3.test b9558d5b0ca44a8b6247fbb5d4a47592a8976892
F ext/fts5/test/fts5corrupt3.test f77f65e386231daf62902466b40ff998b2c8ce4f
F ext/fts5/test/fts5detail.test ef5c690535a797413acaf5ad9b8ab5d49972df69
F ext/fts5/test/fts5dlidx.test 13871a14641017ae42f6f1055a8067bafd44cb3d
F ext/fts5/test/fts5dlidx.test 007e9390c94638760797dbec2990c97c3fa08dfe
F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
F ext/fts5/test/fts5eb.test c516ae0c934be6fd29ec95ea8b5f11f461311535
@ -1456,7 +1456,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P e0b116edd64a55c971c368685aa343cb6beed0f1
R 1b0c52bbc749be5b9a732d29d3d099e5
U drh
Z 5dddb7cbfc6ef15baab9797e73067d60
P 45f7f0c80bd91a0c7ff859c27fd9e82e551bd83e
R 7875ababc9def91b1b2c4626dd7ef0ad
T *branch * fts5-data-version
T *sym-fts5-data-version *
T -sym-trunk *
U dan
Z 015dc4987d82758ea8c0d324359dc366

2
manifest.uuid
View File

@ -1 +1 @@
45f7f0c80bd91a0c7ff859c27fd9e82e551bd83e
33ef2210ef19e55c8d460bfe9d3dc146034c8acc