mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Reduce the number of calls to malloc() made by fts5.

Browse Source 
FossilOrigin-Name: 898618ccf61d3ad166d9fc742e132d518338b5c3
This commit is contained in:
dan 2015-07-09 19:02:19 +00:00
parent a90579db99
commit 3dbd2397f4
8 changed files with 99 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ext/fts5/fts5Int.h
View File

@ -36,6 +36,7 @@ typedef sqlite3_uint64 u64;
#define NEVER(x) 0
#define MIN(x,y) (((x) < (y)) ? (x) : (y))
#define MAX(x,y) (((x) > (y)) ? (x) : (y))
#endif

81
ext/fts5/fts5_index.c
View File

@ -605,6 +605,21 @@ struct Fts5BtreeIter {
};
/*
** The first argument passed to this macro is a pointer to an Fts5Buffer
** object.
*/
#define fts5BufferSize(pBuf,n) { \
if( pBuf->nSpace<n ) { \
u8 *pNew = sqlite3_realloc(pBuf->p, n); \
if( pNew==0 ){ \
sqlite3_free(pBuf->p); \
} \
pBuf->nSpace = n; \
pBuf->p = pNew; \
} \
}
static void fts5PutU16(u8 *aOut, u16 iVal){
aOut[0] = (iVal>>8);
aOut[1] = (iVal&0xFF);
@ -724,16 +739,20 @@ static Fts5Data *fts5DataReadOrBuffer(
u8 *aOut; /* Read blob data into this buffer */
int nByte = sqlite3_blob_bytes(p->pReader);
if( pBuf ){
fts5BufferZero(pBuf);
fts5BufferGrow(&rc, pBuf, nByte);
aOut = pBuf->p;
fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20);
pBuf->n = nByte;
aOut = pBuf->p;
if( aOut==0 ){
rc = SQLITE_NOMEM;
}
}else{
int nSpace = nByte + FTS5_DATA_ZERO_PADDING;
pRet = (Fts5Data*)sqlite3Fts5MallocZero(&rc, nSpace+sizeof(Fts5Data));
pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
if( pRet ){
pRet->n = nByte;
aOut = pRet->p = (u8*)&pRet[1];
}else{
rc = SQLITE_NOMEM;
}
}
@ -1019,18 +1038,20 @@ static void fts5StructureExtendLevel(
static Fts5Structure *fts5StructureRead(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
Fts5Structure *pRet = 0; /* Object to return */
Fts5Data *pData; /* %_data entry containing structure record */
int iCookie; /* Configuration cookie */
Fts5Buffer buf = {0, 0, 0};
pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
if( !pData ) return 0;
p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);
fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID);
if( buf.p==0 ) return 0;
assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) );
memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING);
p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
}
fts5DataRelease(pData);
fts5BufferFree(&buf);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
@ -2028,7 +2049,7 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
#ifdef SQLITE_DEBUG
static void fts5AssertNodeSeekOk(
Fts5Data *pNode,
Fts5Buffer *pNode,
const u8 *pTerm, int nTerm, /* Term to search for */
int iExpectPg,
int bExpectDlidx
@ -2070,7 +2091,7 @@ static void fts5AssertNodeSeekOk(
** returned child page number has a doclist-index. Or left as is otherwise.
*/
static int fts5NodeSeek(
Fts5Data *pNode, /* Node to search */
Fts5Buffer *pNode, /* Node to search */
const u8 *pTerm, int nTerm, /* Term to search for */
int *pbDlidx /* OUT: True if dlidx flag is set */
){
@ -2094,6 +2115,7 @@ static int fts5NodeSeek(
*pbDlidx = (*pPtr==0x01);
pPtr++;
pPtr += fts5GetVarint32(pPtr, nEmpty);
if( pPtr>=pEnd ) break;
}
/* Read the next "term" pointer. Set nKeep to the number of bytes to
@ -2230,7 +2252,10 @@ static void fts5LeafSeek(
/* Skip past position list */
fts5IndexGetVarint32(a, iOff, nPos);
iOff += (nPos >> 1);
if( iOff>=n ) goto search_failed;
if( iOff>=(n-1) ){
iOff = n;
goto search_failed;
}
/* If this is the end of the doclist, break out of the loop */
if( a[iOff]==0x00 ){
@ -2287,6 +2312,7 @@ static void fts5LeafSeek(
*/
static void fts5SegIterSeekInit(
Fts5Index *p, /* FTS5 backend */
Fts5Buffer *pBuf, /* Buffer to use for loading pages */
const u8 *pTerm, int nTerm, /* Term to seek to */
int flags, /* Mask of FTS5INDEX_XXX flags */
Fts5StructureSegment *pSeg, /* Description of segment */
@ -2298,6 +2324,9 @@ static void fts5SegIterSeekInit(
int bDlidx = 0; /* True if there is a doclist-index */
Fts5Data *pLeaf;
static int nCall = 0;
nCall++;
assert( bGe==0 || (flags & FTS5INDEX_QUERY_DESC)==0 );
assert( pTerm && nTerm );
memset(pIter, 0, sizeof(*pIter));
@ -2307,11 +2336,9 @@ static void fts5SegIterSeekInit(
** contain term (pTerm/nTerm), if it is present in the segment. */
for(h=pSeg->nHeight-1; h>0; h--){
i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, h, iPg);
Fts5Data *pNode = fts5DataRead(p, iRowid);
if( pNode==0 ) break;
iPg = fts5NodeSeek(pNode, pTerm, nTerm, &bDlidx);
fts5DataRelease(pNode);
fts5DataBuffer(p, pBuf, iRowid);
if( p->rc ) break;
iPg = fts5NodeSeek(pBuf, pTerm, nTerm, &bDlidx);
}
if( iPg<pSeg->pgnoFirst ){
@ -2340,6 +2367,20 @@ static void fts5SegIterSeekInit(
}
}
}
/* Either:
**
** 1) an error has occurred, or
** 2) the iterator points to EOF, or
** 3) the iterator points to an entry with term (pTerm/nTerm), or
** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points
** to an entry with a term greater than or equal to (pTerm/nTerm).
*/
assert( p->rc!=SQLITE_OK /* 1 */
|| pIter->pLeaf==0 /* 2 */
|| fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */
|| (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */
);
}
/*
@ -2795,6 +2836,7 @@ static void fts5MultiIterNew(
int nSeg = 0; /* Number of segment-iters in use */
int iIter = 0; /* */
int iSeg; /* Used to iterate through segments */
Fts5Buffer buf = {0,0,0}; /* Buffer used by fts5SegIterSeekInit() */
Fts5StructureLevel *pLvl;
Fts5IndexIter *pNew;
@ -2832,7 +2874,7 @@ static void fts5MultiIterNew(
if( pTerm==0 ){
fts5SegIterInit(p, pSeg, pIter);
}else{
fts5SegIterSeekInit(p, pTerm, nTerm, flags, pSeg, pIter);
fts5SegIterSeekInit(p, &buf, pTerm, nTerm, flags, pSeg, pIter);
}
}
}
@ -2866,6 +2908,7 @@ static void fts5MultiIterNew(
fts5MultiIterFree(p, pNew);
*ppOut = 0;
}
fts5BufferFree(&buf);
}
/*
@ -5391,7 +5434,7 @@ static void fts5DebugStructure(
for(iLvl=0; iLvl<p->nLevel; iLvl++){
Fts5StructureLevel *pLvl = &p->aLevel[iLvl];
sqlite3Fts5BufferAppendPrintf(pRc, pBuf,
" {lvl=%d nMerge=%d", iLvl, pLvl->nMerge
" {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg
);
for(iSeg=0; iSeg<pLvl->nSeg; iSeg++){
Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg];

4
ext/fts5/test/fts5_common.tcl
View File

@ -124,7 +124,7 @@ proc fts5_level_segs {tbl} {
set sql "SELECT fts5_decode(rowid,block) aS r FROM ${tbl}_data WHERE rowid=10"
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
lappend ret [expr [llength $L] - 2]
lappend ret [expr [llength $L] - 3]
}
set ret
}
@ -134,7 +134,7 @@ proc fts5_level_segids {tbl} {
set ret [list]
foreach L [lrange [db one $sql] 1 end] {
set lvl [list]
foreach S [lrange $L 2 end] {
foreach S [lrange $L 3 end] {
regexp {id=([1234567890]*)} $S -> segid
lappend lvl $segid
}

2
ext/fts5/test/fts5aa.test
View File

@ -49,7 +49,7 @@ do_execsql_test 2.1 {
}
do_test 2.2 {
execsql { SELECT fts5_decode(id, block) FROM t1_data WHERE id==10 }
} {/{\(structure\) {lvl=0 nMerge=0 {id=[0123456789]* h=1 leaves=1..1}}}/}
} {/{\(structure\) {lvl=0 nMerge=0 nSeg=1 {id=[0123456789]* h=1 leaves=1..1}}}/}
foreach w {a b c d e f} {
do_execsql_test 2.3.$w.asc {

13
ext/fts5/test/fts5rowid.test
View File

@ -71,7 +71,8 @@ do_execsql_test 2.3 {
} $res
do_execsql_test 2.4 {
UPDATE x1_data SET block = X'';
SELECT count(fts5_decode(rowid, block)) FROM x1_data;
-- SELECT count(fts5_decode(rowid, block)) FROM x1_data;
SELECT count(*) FROM x1_data;
} $res
do_execsql_test 2.5 {
@ -83,10 +84,12 @@ set res [db one {SELECT count(*) FROM x1_data}]
do_execsql_test 2.6 {
SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
do_execsql_test 2.7 {
UPDATE x1_data SET block = X'';
SELECT count(fts5_decode(rowid, block)) FROM x1_data;
} $res
# This is really a corruption test...
#do_execsql_test 2.7 {
# UPDATE x1_data SET block = X'';
# SELECT count(fts5_decode(rowid, block)) FROM x1_data;
#} $res
#-------------------------------------------------------------------------
# Tests with very large tokens.

22
ext/fts5/tool/showfts5.tcl
View File

@ -1,28 +1,32 @@
#-------------------------------------------------------------------------
# Process command line arguments.
#
proc usage {} {
puts stderr "usage: $::argv0 database table"
puts stderr ""
exit 1
}
set o(vtab) fts5
set o(tok) ""
set o(limit) 0
set o(automerge) -1
set o(crisismerge) -1
if {[llength $argv]!=2} usage
set database [lindex $argv 0]
set tbl [lindex $argv 1]
#-------------------------------------------------------------------------
# Start of main program.
#
sqlite3 db $database
catch { load_static_extension db fts5 }
db eval "SELECT fts5_decode(rowid, block) AS d FROM ${tbl}_data WHERE id=10" {
foreach lvl [lrange $d 1 end] {
puts $lvl
puts [lrange $lvl 0 2]
foreach seg [lrange $lvl 3 end] {
puts " $seg"
}
}
}

22
manifest
View File

@ -1,5 +1,5 @@
C Fix\stwo\sproblems\sthat\scould\scause\sfts3\sauxiliary\sfunctions\sto\soccasionally\smisbehave\sif\sused\swith\smatch\sexpressions\sthat\scontain\sboth\sOR\sand\sNEAR.
D 2015-07-08T17:59:08.894
C Reduce\sthe\snumber\sof\scalls\sto\smalloc()\smade\sby\sfts5.
D 2015-07-09T19:02:19.523
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 017bf0511d1b2dd1db5e16488fbf75a17b526cbc
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a
F ext/fts5/fts5.h 81d1a92fc2b4bd477af7e4e0b38b456f3e199fba
F ext/fts5/fts5Int.h c91773920639c01571d6870ac0c20e960798f79c
F ext/fts5/fts5Int.h 582eb73064b5a2837d44e72907f84f94e86a2f79
F ext/fts5/fts5_aux.c 7cd0e2858171dacf505fea4e2e84ee6126854c3d
F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
F ext/fts5/fts5_config.c b2456e9625bca41c51d54c363e369c6356895c90
F ext/fts5/fts5_expr.c d2e148345639c5a5583e0daa39a639bf298ae6a7
F ext/fts5/fts5_hash.c 219f4edd72e5cf95b19c33f1058809a18fad5229
F ext/fts5/fts5_index.c ab14b8a273ec72a4407eebfb787bbfb51acef565
F ext/fts5/fts5_index.c ba397fe86f0b5ffdafe8eff71d5dbc85cc1180da
F ext/fts5/fts5_main.c 37b0055cb4036c4b4bb4eb36e30ebd1c21c63939
F ext/fts5/fts5_storage.c 4cae85b5287b159d9d98174a4e70adf872b0930a
F ext/fts5/fts5_tcl.c 85eb4e0d0fefa9420b78151496ad4599a1783e20
@ -122,8 +122,8 @@ F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
F ext/fts5/fts5_vocab.c 4e268a3fcbc099e50e335a1135be985a41ff6f7f
F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
F ext/fts5/test/fts5_common.tcl 9553cce0757092d194307c2168d4edd100eab578
F ext/fts5/test/fts5aa.test f7f057811eb0113d3259e059218bc85cef444280
F ext/fts5/test/fts5_common.tcl e0b4a846a7670f6232a644ece69ef25a5c19c0e8
F ext/fts5/test/fts5aa.test 4e896b9154764fed48179a87ba0bdf3650d7f49d
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
@ -169,7 +169,7 @@ F ext/fts5/test/fts5prefix.test 552a462f0e8595676611f41643de217fb4ac2808
F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test f7674e19a40987bf59624d8db9827114cb7f7a3e
F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
F ext/fts5/test/fts5tokenizer.test 83e7e01a21ec7fdf814d51f6184cc26bb77d7695
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@ -179,7 +179,7 @@ F ext/fts5/test/fts5version.test c54a708236642bcc850d2aedc6f505fef1d9f9f1
F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
F ext/fts5/tool/loadfts5.tcl 95edf0b6b92a09f9ed85595038b1108127987556
F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
F ext/fts5/tool/showfts5.tcl 8478fbe6cb93a8ff37335c9b48a38c4197dcfe5e
F ext/fts5/tool/showfts5.tcl fb62e8eae6d862afdd22f367e286fb886d5e1ab6
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c b2732aef0b076e4276d9b39b5a33cec7a05e1413
F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@ -1365,7 +1365,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 5348ffc3fda5168c1e9e14aa88b0c6aedbda7c94
R bb6c5e63b90e2eefb82b18295e953873
P 372c1db2475f367d54270d5801aff0503745bff4
R cd5eb7bfe39c7cb521d188bde2e79a4e
U dan
Z bead1c8f07e0e547f66cbd142b1cf6b7
Z 5190374d0e2aa885aa84b71a9146c526

2
manifest.uuid
View File

@ -1 +1 @@
372c1db2475f367d54270d5801aff0503745bff4
898618ccf61d3ad166d9fc742e132d518338b5c3