From a11846b77af8e8ef0fc89409beeabe73439bd412 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 Jan 2004 18:52:56 +0000 Subject: [PATCH] Defer the {quote: MoveTo} opcode in VDBE until the data is actually needed. Sometimes the data is never needed, resulting in a performance increase. On an indexed order search with a large OFFSET, queries times can be an order of magnitude faster. (CVS 1165) FossilOrigin-Name: d3e96da20d269a068188915b3cc0eb02d330d316 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 68 +++++++++++++++++++++------------------------------ src/vdbeInt.h | 13 ++++++++++ src/vdbeaux.c | 48 ++++++++++++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index cc988a7ab4..7bff28b818 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sit\ssafe\sto\scall\ssqliteMalloc()\swith\sa\srequest\sfor\s0\sbytes.\s\sTicket\s#534.\s(CVS\s1164) -D 2004-01-07T03:41:04 +C Defer\sthe\s{quote:\sMoveTo}\r\nopcode\sin\sVDBE\suntil\sthe\sdata\sis\sactually\sneeded.\s\sSometimes\r\nthe\sdata\sis\snever\sneeded,\sresulting\sin\sa\sperformance\sincrease.\s\sOn\san\sindexed\r\norder\ssearch\swith\sa\slarge\sOFFSET,\squeries\stimes\scan\sbe\san\sorder\sof\smagnitude\r\nfaster.\s(CVS\s1165) +D 2004-01-07T18:52:57 F Makefile.in 0515ff9218ad8d5a8f6220f0494b8ef94c67013b F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd @@ -63,10 +63,10 @@ F src/trigger.c ce83e017b407d046e909d05373d7f8ee70f9f7f9 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397 F src/util.c 64995b5949a5d377629ffd2598747bc771cade1e F src/vacuum.c 77485a64a6e4e358170f150fff681c1624a092b0 -F src/vdbe.c a16a084ca40edeec3a2e490d6f672fc84f851dd9 +F src/vdbe.c 651f294e3a56baf50d56e11fed822b963f3bf41f F src/vdbe.h 3957844e46fea71fd030e78f6a3bd2f7e320fb43 -F src/vdbeInt.h 2824bf88895b901b3a8c9e44527c67530e1c0dcb -F src/vdbeaux.c 8f0df877ddbfc3f6500f78d2f8f1aeea24f1e964 +F src/vdbeInt.h eab39bc209b267271bc4afbcf4991d6c229bae9a +F src/vdbeaux.c 6f2d43643f83656b2555b7ee320397805db11d4c F src/where.c 724c7b82938b2b52602e583c1c3a719eec17003c F test/all.test 569a92a8ee88f5300c057cc4a8f50fbbc69a3242 F test/attach.test c26848402e7ac829e043e1fa5e0eb87032e5d81d @@ -179,7 +179,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1 -P 70df32b716b0d6a4f72bb3ae6496431e53733b6a -R 81967e931e92d936dc4d0dd66b067402 +P 6c858db2c099c7ba73d72e02b19bf6173620db13 +R 4cb210b9d1a4f8eeca46f83ab493882a U drh -Z 999146283487fc892d6fd412710898d3 +Z c064aa79b059a29becbc8ed82d605d9d diff --git a/manifest.uuid b/manifest.uuid index f42e51d97a..079c54afa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c858db2c099c7ba73d72e02b19bf6173620db13 \ No newline at end of file +d3e96da20d269a068188915b3cc0eb02d330d316 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 587a22dc1e..50f6f95cae 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -43,7 +43,7 @@ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. ** -** $Id: vdbe.c,v 1.246 2003/12/23 02:17:35 drh Exp $ +** $Id: vdbe.c,v 1.247 2004/01/07 18:52:57 drh Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -362,42 +362,6 @@ static Sorter *Merge(Sorter *pLeft, Sorter *pRight){ return sHead.pNext; } -/* -** Convert an integer in between the native integer format and -** the bigEndian format used as the record number for tables. -** -** The bigEndian format (most significant byte first) is used for -** record numbers so that records will sort into the correct order -** even though memcmp() is used to compare the keys. On machines -** whose native integer format is little endian (ex: i486) the -** order of bytes is reversed. On native big-endian machines -** (ex: Alpha, Sparc, Motorola) the byte order is the same. -** -** This function is its own inverse. In other words -** -** X == byteSwap(byteSwap(X)) -*/ -static int byteSwap(int x){ - union { - char zBuf[sizeof(int)]; - int i; - } ux; - ux.zBuf[3] = x&0xff; - ux.zBuf[2] = (x>>8)&0xff; - ux.zBuf[1] = (x>>16)&0xff; - ux.zBuf[0] = (x>>24)&0xff; - return ux.i; -} - -/* -** When converting from the native format to the key format and back -** again, in addition to changing the byte order we invert the high-order -** bit of the most significant byte. This causes negative numbers to -** sort before positive numbers in the memcmp() function. -*/ -#define keyToInt(X) (byteSwap(X) ^ 0x80000000) -#define intToKey(X) (byteSwap((X) ^ 0x80000000)) - /* ** Code contained within the VERIFY() macro is not needed for correct ** execution. It is there only to catch errors. So when we compile @@ -2597,8 +2561,15 @@ case OP_MoveTo: { pC = &p->aCsr[i]; if( pC->pCursor!=0 ){ int res, oc; + pC->nullRow = 0; if( aStack[tos].flags & STK_Int ){ int iKey = intToKey(aStack[tos].i); + if( pOp->p2==0 && pOp->opcode==OP_MoveTo ){ + pC->movetoTarget = iKey; + pC->deferredMoveto = 1; + POPSTACK; + break; + } sqliteBtreeMoveto(pC->pCursor, (char*)&iKey, sizeof(int), &res); pC->lastRecno = aStack[tos].i; pC->recnoIsValid = res==0; @@ -2607,7 +2578,7 @@ case OP_MoveTo: { sqliteBtreeMoveto(pC->pCursor, zStack[tos], aStack[tos].n, &res); pC->recnoIsValid = 0; } - pC->nullRow = 0; + pC->deferredMoveto = 0; sqlite_search_count++; oc = pOp->opcode; if( oc==OP_MoveTo && res<0 ){ @@ -2682,6 +2653,7 @@ case OP_Found: { Stringify(p, tos); rx = sqliteBtreeMoveto(pC->pCursor, zStack[tos], aStack[tos].n, &res); alreadyExists = rx==SQLITE_OK && res==0; + pC->deferredMoveto = 0; } if( pOp->opcode==OP_Found ){ if( alreadyExists ) pc = pOp->p2 - 1; @@ -2743,6 +2715,7 @@ case OP_IsUnique: { /* Search for an entry in P1 where all but the last four bytes match K. ** If there is no such entry, jump immediately to P2. */ + assert( p->aCsr[i].deferredMoveto==0 ); rc = sqliteBtreeMoveto(pCrsr, zKey, nKey-4, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; if( res<0 ){ @@ -2911,6 +2884,7 @@ case OP_NewRecno: { } } pC->recnoIsValid = 0; + pC->deferredMoveto = 0; } p->tos++; aStack[p->tos].i = v; @@ -2993,6 +2967,7 @@ case OP_PutStrKey: { zStack[tos], aStack[tos].n); } pC->recnoIsValid = 0; + pC->deferredMoveto = 0; } POPSTACK; POPSTACK; @@ -3019,6 +2994,7 @@ case OP_Delete: { assert( i>=0 && inCursor ); pC = &p->aCsr[i]; if( pC->pCursor!=0 ){ + sqliteVdbeCursorMoveto(pC); rc = sqliteBtreeDelete(pC->pCursor); pC->nextRowidValid = 0; } @@ -3061,6 +3037,7 @@ case OP_RowData: { aStack[tos].flags = STK_Null; }else if( pC->pCursor!=0 ){ BtCursor *pCrsr = pC->pCursor; + sqliteVdbeCursorMoveto(pC); if( pC->nullRow ){ aStack[tos].flags = STK_Null; break; @@ -3131,6 +3108,7 @@ case OP_Column: { zRec = zStack[tos+i]; payloadSize = aStack[tos+i].n; }else if( (pC = &p->aCsr[i])->pCursor!=0 ){ + sqliteVdbeCursorMoveto(pC); zRec = 0; pCrsr = pC->pCursor; if( pC->nullRow ){ @@ -3237,7 +3215,9 @@ case OP_Recno: { int v; assert( i>=0 && inCursor ); - if( (pC = &p->aCsr[i])->recnoIsValid ){ + pC = &p->aCsr[i]; + sqliteVdbeCursorMoveto(pC); + if( pC->recnoIsValid ){ v = pC->lastRecno; }else if( pC->pseudoTable ){ v = keyToInt(pC->iKey); @@ -3276,6 +3256,7 @@ case OP_FullKey: { int amt; char *z; + sqliteVdbeCursorMoveto(&p->aCsr[i]); sqliteBtreeKeySize(pCrsr, &amt); if( amt<=0 ){ rc = SQLITE_CORRUPT; @@ -3329,7 +3310,8 @@ case OP_Last: { if( (pCrsr = pC->pCursor)!=0 ){ int res; rc = sqliteBtreeLast(pCrsr, &res); - p->aCsr[i].nullRow = res; + pC->nullRow = res; + pC->deferredMoveto = 0; if( res && pOp->p2>0 ){ pc = pOp->p2 - 1; } @@ -3359,6 +3341,7 @@ case OP_Rewind: { rc = sqliteBtreeFirst(pCrsr, &res); pC->atFirst = res==0; pC->nullRow = res; + pC->deferredMoveto = 0; if( res && pOp->p2>0 ){ pc = pOp->p2 - 1; } @@ -3397,6 +3380,7 @@ case OP_Next: { if( pC->nullRow ){ res = 1; }else{ + assert( pC->deferredMoveto==0 ); rc = pOp->opcode==OP_Next ? sqliteBtreeNext(pCrsr, &res) : sqliteBtreePrevious(pCrsr, &res); pC->nullRow = res; @@ -3458,6 +3442,7 @@ case OP_IdxPut: { } } rc = sqliteBtreeInsert(pCrsr, zKey, nKey, "", 0); + assert( p->aCsr[i].deferredMoveto==0 ); } POPSTACK; break; @@ -3479,6 +3464,7 @@ case OP_IdxDelete: { if( rx==SQLITE_OK && res==0 ){ rc = sqliteBtreeDelete(pCrsr); } + assert( p->aCsr[i].deferredMoveto==0 ); } POPSTACK; break; @@ -3501,6 +3487,7 @@ case OP_IdxRecno: { if( VERIFY( i>=0 && inCursor && ) (pCrsr = p->aCsr[i].pCursor)!=0 ){ int v; int sz; + assert( p->aCsr[i].deferredMoveto==0 ); sqliteBtreeKeySize(pCrsr, &sz); if( szaCsr[i].deferredMoveto==0 ); rc = sqliteBtreeKeyCompare(pCrsr, zStack[tos], aStack[tos].n, 4, &res); if( rc!=SQLITE_OK ){ break; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index d168387cd2..d58523ad28 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -16,6 +16,15 @@ ** this header information was factored out. */ +/* +** When converting from the native format to the key format and back +** again, in addition to changing the byte order we invert the high-order +** bit of the most significant byte. This causes negative numbers to +** sort before positive numbers in the memcmp() function. +*/ +#define keyToInt(X) (sqliteVdbeByteSwap(X) ^ 0x80000000) +#define intToKey(X) (sqliteVdbeByteSwap((X) ^ 0x80000000)) + /* ** The makefile scans this source file and creates the following ** array of string constants which are the names of all VDBE opcodes. @@ -62,6 +71,8 @@ struct Cursor { Bool nullRow; /* True if pointing to a row with no data */ Bool nextRowidValid; /* True if the nextRowid field is valid */ Bool pseudoTable; /* This is a NEW or OLD pseudo-tables of a trigger */ + Bool deferredMoveto; /* A call to sqliteBtreeMoveto() is needed */ + int movetoTarget; /* Argument to the deferred sqliteBtreeMoveto() */ Btree *pBt; /* Separate file holding temporary table */ int nData; /* Number of bytes in pData */ char *pData; /* Data for a NEW or OLD pseudo-table */ @@ -294,6 +305,8 @@ void sqliteVdbeSorterReset(Vdbe*); void sqliteVdbeAggReset(Agg*); void sqliteVdbeKeylistFree(Keylist*); void sqliteVdbePopStack(Vdbe*,int); +int sqliteVdbeCursorMoveto(Cursor*); +int sqliteVdbeByteSwap(int); #if !defined(NDEBUG) || defined(VDBE_PROFILE) void sqliteVdbePrintOp(FILE*, int, Op*); #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c08db5c088..6249a29f0b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -992,3 +992,51 @@ void sqliteVdbeDelete(Vdbe *p){ p->magic = VDBE_MAGIC_DEAD; sqliteFree(p); } + +/* +** Convert an integer in between the native integer format and +** the bigEndian format used as the record number for tables. +** +** The bigEndian format (most significant byte first) is used for +** record numbers so that records will sort into the correct order +** even though memcmp() is used to compare the keys. On machines +** whose native integer format is little endian (ex: i486) the +** order of bytes is reversed. On native big-endian machines +** (ex: Alpha, Sparc, Motorola) the byte order is the same. +** +** This function is its own inverse. In other words +** +** X == byteSwap(byteSwap(X)) +*/ +int sqliteVdbeByteSwap(int x){ + union { + char zBuf[sizeof(int)]; + int i; + } ux; + ux.zBuf[3] = x&0xff; + ux.zBuf[2] = (x>>8)&0xff; + ux.zBuf[1] = (x>>16)&0xff; + ux.zBuf[0] = (x>>24)&0xff; + return ux.i; +} + +/* +** If a MoveTo operation is pending on the given cursor, then do that +** MoveTo now. Return an error code. If no MoveTo is pending, this +** routine does nothing and returns SQLITE_OK. +*/ +int sqliteVdbeCursorMoveto(Cursor *p){ + if( p->deferredMoveto ){ + int res; + extern int sqlite_search_count; + sqliteBtreeMoveto(p->pCursor, (char*)&p->movetoTarget, sizeof(int), &res); + p->lastRecno = keyToInt(p->movetoTarget); + p->recnoIsValid = res==0; + if( res<0 ){ + sqliteBtreeNext(p->pCursor, &res); + } + sqlite_search_count++; + p->deferredMoveto = 0; + } + return SQLITE_OK; +}