From 3d4435b234b53370989f59c56647c479eeb7e36f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Aug 2011 20:55:50 +0000 Subject: [PATCH 01/36] Fix several harmless compiler warnings and a documentation bug. FossilOrigin-Name: 5454d0fe227b7c1f0e7715b6c08f97019628fc4c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 24 ++++++++++++++---------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 91a0745b81..530096adf2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\scompiler\serror\smessage\sregarding\sthe\schoice\sof\smemory\sallocator\sdefines. -D 2011-08-26T11:25:02.455 +C Fix\sseveral\sharmless\scompiler\swarnings\sand\sa\sdocumentation\sbug. +D 2011-08-26T20:55:50.529 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 1a34ca3794ced80e4a4ebcc3ba1f4c516762e534 +F src/os_unix.c 3792524439a13af0464ba65baf4d647dad063604 F src/os_win.c 86bcb5bd0386c761c764c3383879469346da3a14 F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 5238a74987507f27af4080e2842f53e049b5b111 -R 68d004e4a18ddd9b15416c3d56993539 -U mistachkin -Z cf5bdd8db49b4512fd908f85b844bfec +P 1dada5158215d1816edb69ff2610f9d2259ce19d +R 16e082a02686bfb3de55c1b850240992 +U drh +Z 9f47e5dd82da8bb114d554ebcb62232c diff --git a/manifest.uuid b/manifest.uuid index 3413c7f093..07a8c25fca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dada5158215d1816edb69ff2610f9d2259ce19d \ No newline at end of file +5454d0fe227b7c1f0e7715b6c08f97019628fc4c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 868e029f82..446c6c5fb9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -2525,11 +2525,12 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; + afpLockingContext *context; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + context = (afpLockingContext *) pFile->lockingContext; if( context->reserved ){ *pResOut = 1; return SQLITE_OK; @@ -2669,7 +2670,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ ** operating system calls for the specified lock. */ if( eFileLock==SHARED_LOCK ){ - int lrc1, lrc2, lrc1Errno; + int lrc1, lrc2, lrc1Errno = 0; long lk, mask; assert( pInode->nShared==0 ); @@ -3439,11 +3440,9 @@ static int proxyFileControl(sqlite3_file*,int,void*); /* ** This function is called to handle the SQLITE_FCNTL_SIZE_HINT -** file-control operation. -** -** If the user has configured a chunk-size for this file, it could be -** that the file needs to be extended at this point. Otherwise, the -** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +** file-control operation. Enlarge the database to nBytes in size +** (rounded up to the next chunk-size). If the database is already +** nBytes or larger, this routine is a no-op. */ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ { /* preserve indentation of removed "if" */ @@ -4957,6 +4956,9 @@ static int unixOpen( #if SQLITE_ENABLE_LOCKING_STYLE int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); #endif +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; +#endif /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() @@ -5089,7 +5091,6 @@ static int unixOpen( #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE - struct statfs fsInfo; if( fstatfs(fd, &fsInfo) == -1 ){ ((unixFile*)pFile)->lastErrno = errno; robust_close(p, fd, __LINE__); @@ -5113,7 +5114,6 @@ static int unixOpen( if( envforce!=NULL ){ useProxy = atoi(envforce)>0; }else{ - struct statfs fsInfo; if( statfs(zPath, &fsInfo) == -1 ){ /* In theory, the close(fd) call is sub-optimal. If the file opened ** with fd is a database file, and there are other connections open @@ -5854,6 +5854,8 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ return SQLITE_IOERR; } } +#else + UNUSED_PARAMETER(pError); #endif #ifdef SQLITE_TEST /* simulate multiple hosts by creating unique hostid file paths */ @@ -5946,6 +5948,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ int nTries = 0; struct timespec conchModTime; + memset(&conchModTime, 0, sizeof(conchModTime)); do { rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); nTries ++; @@ -6177,11 +6180,12 @@ static int proxyTakeConch(unixFile *pFile){ end_takeconch: OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ + int fd; if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); } pFile->h = -1; - int fd = robust_open(pCtx->dbPath, pFile->openFlags, + fd = robust_open(pCtx->dbPath, pFile->openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ From a205a486d2e1d2d75e7641ca92dd243f91be651b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Aug 2011 18:48:57 +0000 Subject: [PATCH 02/36] Experimental changes to improve the performance of OP_Next. FossilOrigin-Name: 1a249845251199c00817893add300a1a654b4df9 --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/btree.c | 23 ++++++++++++++++------- src/vdbe.c | 45 ++++++++++++++++++++++----------------------- src/vdbe.h | 2 ++ src/vdbeaux.c | 6 ++++++ 6 files changed, 60 insertions(+), 41 deletions(-) diff --git a/manifest b/manifest index 530096adf2..21e42fba5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sseveral\sharmless\scompiler\swarnings\sand\sa\sdocumentation\sbug. -D 2011-08-26T20:55:50.529 +C Experimental\schanges\sto\simprove\sthe\sperformance\sof\sOP_Next. +D 2011-08-27T18:48:57.573 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ed13fdefdbe671d5777773dcfb3a162ddb4623ae +F src/btree.c bd89d604a532063da8ed1a095f1805db49896325 F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd @@ -238,11 +238,11 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 4a7191c0f8e918b74e8c84cbdd77746d6b7e3bcf -F src/vdbe.h 2bf6ec77d8b9980fc19da6e0b0a36d0dbf884ce4 +F src/vdbe.c d63854aef07d036987f768dae7ca5c852881ce28 +F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c 11b0df8822ecf61e543562247207df75e2ebb617 +F src/vdbeaux.c dd5d10ae523bbc6ed55ac73daa28a9ea1f2fa42a F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e @@ -961,7 +961,10 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 1dada5158215d1816edb69ff2610f9d2259ce19d -R 16e082a02686bfb3de55c1b850240992 -U drh -Z 9f47e5dd82da8bb114d554ebcb62232c +P 5454d0fe227b7c1f0e7715b6c08f97019628fc4c +R 06eac31c16dc3f7f5b4649f86c063bf4 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z 50e65d7380a3dc463f2c091e8241574b diff --git a/manifest.uuid b/manifest.uuid index 07a8c25fca..ae93ed9c60 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5454d0fe227b7c1f0e7715b6c08f97019628fc4c \ No newline at end of file +1a249845251199c00817893add300a1a654b4df9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0c5fa38e4a..d77fce4c8e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3479,7 +3479,8 @@ static int btreeCursor( return SQLITE_READONLY; } if( iTable==1 && btreePagecount(pBt)==0 ){ - return SQLITE_EMPTY; + assert( wrFlag==0 ); + iTable = 0; } /* Now that no other errors can occur, finish filling in the BtCursor @@ -4233,6 +4234,9 @@ static int moveToRoot(BtCursor *pCur){ releasePage(pCur->apPage[i]); } pCur->iPage = 0; + }else if( pCur->pgnoRoot==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; }else{ rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); if( rc!=SQLITE_OK ){ @@ -4342,7 +4346,7 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); *pRes = 1; }else{ assert( pCur->apPage[pCur->iPage]->nCell>0 ); @@ -4381,7 +4385,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); *pRes = 1; }else{ assert( pCur->eState==CURSOR_VALID ); @@ -4454,12 +4458,12 @@ int sqlite3BtreeMovetoUnpacked( if( rc ){ return rc; } - assert( pCur->apPage[pCur->iPage] ); - assert( pCur->apPage[pCur->iPage]->isInit ); - assert( pCur->apPage[pCur->iPage]->nCell>0 || pCur->eState==CURSOR_INVALID ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit ); + assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 ); if( pCur->eState==CURSOR_INVALID ){ *pRes = -1; - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } assert( pCur->apPage[0]->intKey || pIdxKey ); @@ -7376,6 +7380,11 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ i64 nEntry = 0; /* Value to return in *pnEntry */ int rc; /* Return code */ + + if( pCur->pgnoRoot==0 ){ + *pnEntry = 0; + return SQLITE_OK; + } rc = moveToRoot(pCur); /* Unless an error occurs, the following loop runs one iteration for each diff --git a/src/vdbe.c b/src/vdbe.c index 8491306507..133dc54b92 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2550,7 +2550,7 @@ case OP_Count: { /* out2-prerelease */ BtCursor *pCrsr; pCrsr = p->apCsr[pOp->p1]->pCursor; - if( pCrsr ){ + if( ALWAYS(pCrsr) ){ rc = sqlite3BtreeCount(pCrsr, &nEntry); }else{ nEntry = 0; @@ -3112,15 +3112,9 @@ case OP_OpenWrite: { rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; - /* Since it performs no memory allocation or IO, the only values that - ** sqlite3BtreeCursor() may return are SQLITE_EMPTY and SQLITE_OK. - ** SQLITE_EMPTY is only returned when attempting to open the table - ** rooted at page 1 of a zero-byte database. */ - assert( rc==SQLITE_EMPTY || rc==SQLITE_OK ); - if( rc==SQLITE_EMPTY ){ - pCur->pCursor = 0; - rc = SQLITE_OK; - } + /* Since it performs no memory allocation or IO, the only value that + ** sqlite3BtreeCursor() may return is SQLITE_OK. */ + assert( rc==SQLITE_OK ); /* Set the VdbeCursor.isTable and isIndex variables. Previous versions of ** SQLite used to check if the root-page flags were sane at this point @@ -3333,7 +3327,7 @@ case OP_SeekGt: { /* jump, in3 */ assert( OP_SeekGe == OP_SeekLt+2 ); assert( OP_SeekGt == OP_SeekLt+3 ); assert( pC->isOrdered ); - if( pC->pCursor!=0 ){ + if( ALWAYS(pC->pCursor!=0) ){ oc = pOp->opcode; pC->nullRow = 0; if( pC->isTable ){ @@ -3691,7 +3685,7 @@ case OP_NotExists: { /* jump, in3 */ assert( pC->isTable ); assert( pC->pseudoTableReg==0 ); pCrsr = pC->pCursor; - if( pCrsr!=0 ){ + if( ALWAYS(pCrsr!=0) ){ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); @@ -4218,6 +4212,7 @@ case OP_NullRow: { assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; + assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } @@ -4241,7 +4236,7 @@ case OP_Last: { /* jump */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->pCursor; - if( pCrsr==0 ){ + if( NEVER(pCrsr==0) ){ res = 1; }else{ rc = sqlite3BtreeLast(pCrsr, &res); @@ -4296,7 +4291,9 @@ case OP_Rewind: { /* jump */ res = 1; if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(db, pC, &res); - }else if( (pCrsr = pC->pCursor)!=0 ){ + }else{ + pCrsr = pC->pCursor; + assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->atFirst = res==0 ?1:0; pC->deferredMoveto = 0; @@ -4311,7 +4308,7 @@ case OP_Rewind: { /* jump */ break; } -/* Opcode: Next P1 P2 * * P5 +/* Opcode: Next P1 P2 * P4 P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through @@ -4320,6 +4317,9 @@ case OP_Rewind: { /* jump */ ** ** The P1 cursor must be for a real table, not a pseudo-table. ** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreeNext(). +** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** @@ -4334,13 +4334,15 @@ case OP_Rewind: { /* jump */ ** ** The P1 cursor must be for a real table, not a pseudo-table. ** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreePrevious(). +** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; - BtCursor *pCrsr; int res; CHECK_FOR_INTERRUPT; @@ -4354,15 +4356,12 @@ case OP_Next: { /* jump */ assert( pOp->opcode==OP_Next ); rc = sqlite3VdbeSorterNext(db, pC, &res); }else{ - pCrsr = pC->pCursor; - if( pCrsr==0 ){ - pC->nullRow = 1; - break; - } res = 1; assert( pC->deferredMoveto==0 ); - rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(pCrsr, &res) : - sqlite3BtreePrevious(pCrsr, &res); + assert( pC->pCursor ); + assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); + assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); + rc = pOp->p4.xAdvance(pC->pCursor, &res); } pC->nullRow = (u8)res; pC->cacheStatus = CACHE_STALE; diff --git a/src/vdbe.h b/src/vdbe.h index fe758d2dbd..9728548517 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -61,6 +61,7 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_DEBUG char *zComment; /* Comment to improve readability */ @@ -116,6 +117,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ +#define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ /* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure ** is made. That copy is freed when the Vdbe is finalized. But if the diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ffbb95ecac..a1e6a276f3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -433,6 +433,12 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; #endif + }else if( opcode==OP_Next ){ + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + }else if( opcode==OP_Prev ){ + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; } if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ From 4a6f3aabca04a12a7f7f5e54350ed4c52520d8ca Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 28 Aug 2011 00:19:26 +0000 Subject: [PATCH 03/36] Correctly display P4_ADVANCE values on opcode traces. Mark an always-taken branch in OP_Column as such. FossilOrigin-Name: 087dc96086fe4e45da93ab6a0d5dda34c932ce97 --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 2 +- src/vdbeaux.c | 4 ++++ 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 21e42fba5a..804c577387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\sto\simprove\sthe\sperformance\sof\sOP_Next. -D 2011-08-27T18:48:57.573 +C Correctly\sdisplay\sP4_ADVANCE\svalues\son\sopcode\straces.\s\sMark\san\salways-taken\nbranch\sin\sOP_Column\sas\ssuch. +D 2011-08-28T00:19:26.614 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,11 +238,11 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c d63854aef07d036987f768dae7ca5c852881ce28 +F src/vdbe.c 47713d4005ad7256b0b5c9744dded58d14f652ad F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c dd5d10ae523bbc6ed55ac73daa28a9ea1f2fa42a +F src/vdbeaux.c de1e4cab060a45df9ebee68dd63543d14559f0e7 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e @@ -961,10 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 5454d0fe227b7c1f0e7715b6c08f97019628fc4c -R 06eac31c16dc3f7f5b4649f86c063bf4 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * -U dan -Z 50e65d7380a3dc463f2c091e8241574b +P 1a249845251199c00817893add300a1a654b4df9 +R d6f033bc6e8a438086fa5960ee9d7157 +U drh +Z 47cc80ebc2025318e796ba82fe696b90 diff --git a/manifest.uuid b/manifest.uuid index ae93ed9c60..0860b8faf2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a249845251199c00817893add300a1a654b4df9 \ No newline at end of file +087dc96086fe4e45da93ab6a0d5dda34c932ce97 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 133dc54b92..bfc3cbd531 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2173,7 +2173,7 @@ case OP_Column: { rc = sqlite3BtreeDataSize(pCrsr, &payloadSize); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } - }else if( pC->pseudoTableReg>0 ){ + }else if( ALWAYS(pC->pseudoTableReg>0) ){ pReg = &aMem[pC->pseudoTableReg]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a1e6a276f3..053d89f3b5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -945,6 +945,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ sqlite3_snprintf(nTemp, zTemp, "program"); break; } + case P4_ADVANCE: { + zTemp[0] = 0; + break; + } default: { zP4 = pOp->p4.z; if( zP4==0 ){ From e6f43fc4f88b2d640227ea65b11dcb7d149bc50d Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 28 Aug 2011 02:15:34 +0000 Subject: [PATCH 04/36] About a 3% improvement in the performance of OP_Column. FossilOrigin-Name: edff9d4a995095e555fcc9aec4c56f4bcaa1557e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 804c577387..5d893103c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\sdisplay\sP4_ADVANCE\svalues\son\sopcode\straces.\s\sMark\san\salways-taken\nbranch\sin\sOP_Column\sas\ssuch. -D 2011-08-28T00:19:26.614 +C About\sa\s3%\simprovement\sin\sthe\sperformance\sof\sOP_Column. +D 2011-08-28T02:15:34.550 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 47713d4005ad7256b0b5c9744dded58d14f652ad +F src/vdbe.c 8c9e0508ce0fe16e45c7398b3042d62c2968d779 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 1a249845251199c00817893add300a1a654b4df9 -R d6f033bc6e8a438086fa5960ee9d7157 +P 087dc96086fe4e45da93ab6a0d5dda34c932ce97 +R 1c4d03777b493872eab90f1e3c1ffa05 U drh -Z 47cc80ebc2025318e796ba82fe696b90 +Z 518276ac7d503545f19a1d6a51270da1 diff --git a/manifest.uuid b/manifest.uuid index 0860b8faf2..c1149bf53e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -087dc96086fe4e45da93ab6a0d5dda34c932ce97 \ No newline at end of file +edff9d4a995095e555fcc9aec4c56f4bcaa1557e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index bfc3cbd531..891071ee7a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2129,7 +2129,6 @@ case OP_Column: { assert( pOp->p3>0 && pOp->p3<=p->nMem ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); - MemSetTypeFlag(pDest, MEM_Null); zRec = 0; /* This block sets the variable payloadSize to be the total number of @@ -2186,9 +2185,10 @@ case OP_Column: { payloadSize = 0; } - /* If payloadSize is 0, then just store a NULL */ + /* If payloadSize is 0, then just store a NULL. This can happen because of + ** nullRow or because of a corrupt database. */ if( payloadSize==0 ){ - assert( pDest->flags&MEM_Null ); + MemSetTypeFlag(pDest, MEM_Null); goto op_column_out; } assert( db->aLimit[SQLITE_LIMIT_LENGTH]>=0 ); @@ -2354,7 +2354,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - assert( pDest->flags&MEM_Null ); + MemSetTypeFlag(pDest, MEM_Null); } } From 5a077b741fbec498718c79f655c84d1e511949ad Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Aug 2011 02:16:18 +0000 Subject: [PATCH 05/36] Small performance improvement to OP_Column. FossilOrigin-Name: b6b73a747ad8d0f026074e41c2a4adc529ec2674 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5d893103c6..ddba4d7170 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C About\sa\s3%\simprovement\sin\sthe\sperformance\sof\sOP_Column. -D 2011-08-28T02:15:34.550 +C Small\sperformance\simprovement\sto\sOP_Column. +D 2011-08-29T02:16:18.855 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 8c9e0508ce0fe16e45c7398b3042d62c2968d779 +F src/vdbe.c dc84628c47e76ffa5dd945afea1460815323ad91 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 087dc96086fe4e45da93ab6a0d5dda34c932ce97 -R 1c4d03777b493872eab90f1e3c1ffa05 +P edff9d4a995095e555fcc9aec4c56f4bcaa1557e +R 2bdc90f10d7d2094fa6ddf41f90c24fd U drh -Z 518276ac7d503545f19a1d6a51270da1 +Z 9bbe13d88effafce7032c1658f4f21f3 diff --git a/manifest.uuid b/manifest.uuid index c1149bf53e..a29a8dd016 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edff9d4a995095e555fcc9aec4c56f4bcaa1557e \ No newline at end of file +b6b73a747ad8d0f026074e41c2a4adc529ec2674 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 891071ee7a..643f4b480f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2118,6 +2118,7 @@ case OP_Column: { u32 szField; /* Number of bytes in the content of a field */ int szHdr; /* Size of the header size field at start of record */ int avail; /* Number of bytes of available data */ + u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ @@ -2295,8 +2296,14 @@ case OP_Column: { for(i=0; i Date: Mon, 29 Aug 2011 02:49:41 +0000 Subject: [PATCH 06/36] About a 1% overall performance improvement by using a macro to avoid no-op calls to sqlite3MemReleaseExternal(). FossilOrigin-Name: ff71d20a9ed129bd1785a3f7a777ce62098735b7 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- src/vdbeInt.h | 3 +++ src/vdbemem.c | 36 +++++++++++++++--------------------- 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index ddba4d7170..9b75cb0bd1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sto\sOP_Column. -D 2011-08-29T02:16:18.855 +C About\sa\s1%\soverall\sperformance\simprovement\sby\susing\sa\smacro\sto\savoid\nno-op\scalls\sto\ssqlite3MemReleaseExternal(). +D 2011-08-29T02:49:41.731 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,13 +238,13 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c dc84628c47e76ffa5dd945afea1460815323ad91 +F src/vdbe.c 9165b35da939f4a66c7e68b0c6d3f017ca982cb1 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 +F src/vdbeInt.h 70767f6504aac4f0057ec2a55738470a890789ac F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c de1e4cab060a45df9ebee68dd63543d14559f0e7 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 -F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b +F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P edff9d4a995095e555fcc9aec4c56f4bcaa1557e -R 2bdc90f10d7d2094fa6ddf41f90c24fd +P b6b73a747ad8d0f026074e41c2a4adc529ec2674 +R 285e2022a0bbe1b677ddbb6dec295cd8 U drh -Z 9bbe13d88effafce7032c1658f4f21f3 +Z 74a42c07c09977852193d878c3468fe6 diff --git a/manifest.uuid b/manifest.uuid index a29a8dd016..f7d413534f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6b73a747ad8d0f026074e41c2a4adc529ec2674 \ No newline at end of file +ff71d20a9ed129bd1785a3f7a777ce62098735b7 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 643f4b480f..e7c7ed7fe2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -673,7 +673,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - sqlite3VdbeMemReleaseExternal(pOut); + MemReleaseExt(pOut); pOut->flags = MEM_Int; } @@ -2344,7 +2344,7 @@ case OP_Column: { if( aOffset[p2] ){ assert( rc==SQLITE_OK ); if( zRec ){ - sqlite3VdbeMemReleaseExternal(pDest); + MemReleaseExt(pDest); sqlite3VdbeSerialGet((u8 *)&zRec[aOffset[p2]], aType[p2], pDest); }else{ len = sqlite3VdbeSerialTypeLen(aType[p2]); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 11a22bc972..846d807075 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -384,6 +384,9 @@ int sqlite3VdbeMemNumerify(Mem*); int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); void sqlite3VdbeMemRelease(Mem *p); void sqlite3VdbeMemReleaseExternal(Mem *p); +#define MemReleaseExt(X) \ + if((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame)) \ + sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); diff --git a/src/vdbemem.c b/src/vdbemem.c index 882c686334..d512572825 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -271,24 +271,18 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ */ void sqlite3VdbeMemReleaseExternal(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); - testcase( p->flags & MEM_Agg ); - testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_RowSet ); - testcase( p->flags & MEM_Frame ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame) ){ - if( p->flags&MEM_Agg ){ - sqlite3VdbeMemFinalize(p, p->u.pDef); - assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn && p->xDel ){ - assert( (p->flags&MEM_RowSet)==0 ); - p->xDel((void *)p->z); - p->xDel = 0; - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); - } + if( p->flags&MEM_Agg ){ + sqlite3VdbeMemFinalize(p, p->u.pDef); + assert( (p->flags & MEM_Agg)==0 ); + sqlite3VdbeMemRelease(p); + }else if( p->flags&MEM_Dyn && p->xDel ){ + assert( (p->flags&MEM_RowSet)==0 ); + p->xDel((void *)p->z); + p->xDel = 0; + }else if( p->flags&MEM_RowSet ){ + sqlite3RowSetClear(p->u.pRowSet); + }else if( p->flags&MEM_Frame ){ + sqlite3VdbeMemSetNull(p); } } @@ -298,7 +292,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** (Mem.type==SQLITE_TEXT). */ void sqlite3VdbeMemRelease(Mem *p){ - sqlite3VdbeMemReleaseExternal(p); + MemReleaseExt(p); sqlite3DbFree(p->db, p->zMalloc); p->z = 0; p->zMalloc = 0; @@ -620,7 +614,7 @@ void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); + MemReleaseExt(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ @@ -638,7 +632,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); + MemReleaseExt(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; From 54bbe7f15e78a4c4fb2c1eb80a7935218ee730c8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Aug 2011 18:24:27 +0000 Subject: [PATCH 07/36] Fix a broken assert() statement in select.c. FossilOrigin-Name: ad78ef2b3a20e3c3191fba232cbae70656210cf5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 385ff9b82f..173dbf664c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Back\sout\s[05c9832e5f6eb]\ssince\sit\swas\scausing\sa\sperformance\sregression\swith\nno\sobvious\sbenefit. -D 2011-08-29T11:56:14.294 +C Fix\sa\sbroken\sassert()\sstatement\sin\sselect.c. +D 2011-08-29T18:24:27.787 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d +F src/select.c 14552e9ff4b27ec027a43fafb62ea5d049cd2809 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 5a00d24b279424d257781ec3f1227e72a5d56f57 08d0e8799e1441ef063b1cdf9e4107071a0f81ca -R e71350ad5d643f1597441507aa7bf97e -U drh -Z 50a26ca875ae5b915f6bf10b2da0f1e7 +P 639cc85a911454bffdcccb33f2976c683953ae64 +R 91692067377849e6b7529274128e3e95 +U dan +Z 23ba19994854bcc26c6e0981ff18a6fb diff --git a/manifest.uuid b/manifest.uuid index af5aad4f7e..e3526a403d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -639cc85a911454bffdcccb33f2976c683953ae64 \ No newline at end of file +ad78ef2b3a20e3c3191fba232cbae70656210cf5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index bd5e964e7c..2e44522c6d 100644 --- a/src/select.c +++ b/src/select.c @@ -3949,7 +3949,7 @@ int sqlite3Select( if( pWInfo->eDistinct ){ VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - assert( addrDistinctIndex>0 ); + assert( addrDistinctIndex>=0 ); pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); assert( isDistinct ); From bc2be0c770b7ea9a8969c7371e001b8c9cf000db Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Aug 2011 00:53:50 +0000 Subject: [PATCH 08/36] Fix a buffer overrun in test logic. No impact on the core SQLite. FossilOrigin-Name: 49cd60e38bd8df9d736ced95e0ace6efea95ca7d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test1.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 173dbf664c..b5a9fe675f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sstatement\sin\sselect.c. -D 2011-08-29T18:24:27.787 +C Fix\sa\sbuffer\soverrun\sin\stest\slogic.\s\sNo\simpact\son\sthe\score\sSQLite. +D 2011-08-30T00:53:50.943 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -188,7 +188,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3ef1dda2f1dc207c792eaadebf9d8adc44648581 -F src/test1.c 7439efb86c1022f19a39a8e61de2cbac23ffab03 +F src/test1.c 0f41b7c67719207a5de24b009e172c4dcf189827 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 639cc85a911454bffdcccb33f2976c683953ae64 -R 91692067377849e6b7529274128e3e95 -U dan -Z 23ba19994854bcc26c6e0981ff18a6fb +P ad78ef2b3a20e3c3191fba232cbae70656210cf5 +R 9fd102bca587a254d41deef9450d4a29 +U drh +Z 9655fc5ee46b247cbd13810f6ee3dd0d diff --git a/manifest.uuid b/manifest.uuid index e3526a403d..13a9ce5ab5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad78ef2b3a20e3c3191fba232cbae70656210cf5 \ No newline at end of file +49cd60e38bd8df9d736ced95e0ace6efea95ca7d \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 59b570c28a..26342522c5 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4395,7 +4395,7 @@ static u8 *sqlite3_stack_baseline = 0; static void prepStack(void){ int i; u32 bigBuf[65536]; - for(i=0; i Date: Tue, 30 Aug 2011 00:58:58 +0000 Subject: [PATCH 09/36] Fix a total unimportant file descriptor leak in lemon. This is to silence warning messages. FossilOrigin-Name: e95cf2c576dda656c0f31eeec3d98e911b9003a1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b5a9fe675f..fc14f9ba21 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbuffer\soverrun\sin\stest\slogic.\s\sNo\simpact\son\sthe\score\sSQLite. -D 2011-08-30T00:53:50.943 +C Fix\sa\stotal\sunimportant\sfile\sdescriptor\sleak\sin\slemon.\s\sThis\sis\sto\ssilence\nwarning\smessages. +D 2011-08-30T00:58:58.556 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -927,7 +927,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c d51c68d405ff7f9bad99268ca3c20a198eb983ed +F tool/lemon.c 949328f67cac94969d3112b105b8457edf27f44e F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P ad78ef2b3a20e3c3191fba232cbae70656210cf5 -R 9fd102bca587a254d41deef9450d4a29 +P 49cd60e38bd8df9d736ced95e0ace6efea95ca7d +R 864960f9c27e1685a9edda4031c320e3 U drh -Z 9655fc5ee46b247cbd13810f6ee3dd0d +Z 0c80443f2de345db03862d58a6ec74c5 diff --git a/manifest.uuid b/manifest.uuid index 13a9ce5ab5..72809be723 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49cd60e38bd8df9d736ced95e0ace6efea95ca7d \ No newline at end of file +e95cf2c576dda656c0f31eeec3d98e911b9003a1 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index bd2938be25..1fb0308bec 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -2522,6 +2522,7 @@ void Parse(struct lemon *gp) ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.", filesize+1); gp->errorcnt++; + fclose(fp); return; } if( fread(filebuf,1,filesize,fp)!=filesize ){ @@ -2529,6 +2530,7 @@ void Parse(struct lemon *gp) filesize); free(filebuf); gp->errorcnt++; + fclose(fp); return; } fclose(fp); From d589a5444b9a0eb316f06fbeb6fe04cf3049cc68 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 30 Aug 2011 01:23:34 +0000 Subject: [PATCH 10/36] Make sure SQLITE_FCNTL_SIZE_HINT a no-op if the chunk size is not greater than zero. FossilOrigin-Name: 88b763e8d73fafa1538b08af28b1c8b723b39c61 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/os_win.c | 25 ++++++++++++++----------- test/wal5.test | 9 +-------- 5 files changed, 26 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index fc14f9ba21..7e13b09011 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stotal\sunimportant\sfile\sdescriptor\sleak\sin\slemon.\s\sThis\sis\sto\ssilence\nwarning\smessages. -D 2011-08-30T00:58:58.556 +C Make\ssure\sSQLITE_FCNTL_SIZE_HINT\sa\sno-op\sif\sthe\schunk\ssize\sis\snot\sgreater\sthan\szero. +D 2011-08-30T01:23:34.101 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,8 +165,8 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 014cd466edc2b73e417f382c3213b6c72bc35280 -F src/os_win.c 86bcb5bd0386c761c764c3383879469346da3a14 +F src/os_unix.c ae82cf32c497d9a3a0f147de1b7219b636db4f53 +F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -887,7 +887,7 @@ F test/wal.test e11da8d5ea8a38a247339455098357e9adf63d76 F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 +F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 49cd60e38bd8df9d736ced95e0ace6efea95ca7d -R 864960f9c27e1685a9edda4031c320e3 -U drh -Z 0c80443f2de345db03862d58a6ec74c5 +P e95cf2c576dda656c0f31eeec3d98e911b9003a1 +R 23e2e1649d860cacfc7d885e0481975c +U mistachkin +Z 106f8d2e0fe01f69aa7179a809ac1874 diff --git a/manifest.uuid b/manifest.uuid index 72809be723..4613466421 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e95cf2c576dda656c0f31eeec3d98e911b9003a1 \ No newline at end of file +88b763e8d73fafa1538b08af28b1c8b723b39c61 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8abef8de8b..f273d04cb6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3445,7 +3445,7 @@ static int proxyFileControl(sqlite3_file*,int,void*); ** nBytes or larger, this routine is a no-op. */ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ - if( pFile->szChunk ){ + if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ diff --git a/src/os_win.c b/src/os_win.c index 02a7a0c622..2441398945 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1216,7 +1216,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ ** actual file size after the operation may be larger than the requested ** size). */ - if( pFile->szChunk ){ + if( pFile->szChunk>0 ){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } @@ -1603,18 +1603,21 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - winFile *pFile = (winFile*)id; - sqlite3_int64 oldSz; - int rc = winFileSize(id, &oldSz); - if( rc==SQLITE_OK ){ - sqlite3_int64 newSz = *(sqlite3_int64*)pArg; - if( newSz>oldSz ){ - SimulateIOErrorBenign(1); - rc = winTruncate(id, newSz); - SimulateIOErrorBenign(0); + if( pFile->szChunk>0 ){ + winFile *pFile = (winFile*)id; + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } } + return rc; } - return rc; + return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { int bPersist = *(int*)pArg; diff --git a/test/wal5.test b/test/wal5.test index 0c700dfc50..ad6bcfc7d8 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,14 +235,7 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 7 5} - if {$tcl_platform(platform) == "windows"} { - # on unix, the size_hint is a no-op if no chunk size is set. - # the windows implementation does not have a similar check, - # and because of this, the db file size has an extra page. - do_test 2.3.$tn.8 { file_page_counts } {2 7 2 7} - } { - do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} - } + do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} } # Check that checkpoints block on the correct locks. And respond correctly From 307ff30b54002e3634df904eff06176ddadb0df0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Aug 2011 01:29:04 +0000 Subject: [PATCH 11/36] Change the build process so that it does not require the unix "sort" command. This avoids confusion between ms-sort and mingw-sort on windows systems. FossilOrigin-Name: f1bd5bbae505068d24bfd9cc6bab6a8b8940bad6 --- Makefile.in | 2 +- Makefile.msc | 2 +- Makefile.vxworks | 2 +- main.mk | 2 +- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- mkopcodec.awk | 9 +++++++-- 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Makefile.in b/Makefile.in index 53ad95df70..8f1f65dcd0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -775,7 +775,7 @@ tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h diff --git a/Makefile.msc b/Makefile.msc index 022939f2fa..e139a745a1 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -809,7 +809,7 @@ tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c + $(NAWK) -f $(TOP)\mkopcodec.awk opcodes.h > opcodes.c opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h diff --git a/Makefile.vxworks b/Makefile.vxworks index 993e557959..8d57da7283 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -517,7 +517,7 @@ tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | \ diff --git a/main.mk b/main.mk index 4837d4e2fb..aab3706272 100644 --- a/main.mk +++ b/main.mk @@ -429,7 +429,7 @@ tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | \ diff --git a/manifest b/manifest index 7e13b09011..27ea6ae63d 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Make\ssure\sSQLITE_FCNTL_SIZE_HINT\sa\sno-op\sif\sthe\schunk\ssize\sis\snot\sgreater\sthan\szero. -D 2011-08-30T01:23:34.101 +C Change\sthe\sbuild\sprocess\sso\sthat\sit\sdoes\snot\srequire\sthe\sunix\s"sort"\ncommand.\s\sThis\savoids\sconfusion\sbetween\sms-sort\sand\smingw-sort\son\nwindows\ssystems. +D 2011-08-30T01:29:04.104 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 +F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 55fe94bf23b4c1ff035f19b0ae2ea486350f8d01 -F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 +F Makefile.msc c39cc7a7a9fe8f6304f37b88f3bca9aae5b83f30 +F Makefile.vxworks 1deb39c8bb047296c30161ffa10c1b5423e632f9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -104,11 +104,11 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 201b1d81aa4ac0af21b243151e79477c14ce3722 +F main.mk 8744cb76517817170f7fd2c78fbf0006fabb93c1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac -F mkopcodec.awk 3fb9bf077053c968451f4dd03d11661ac373f9d1 +F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F publish.sh 313c5b2425f2cf5e547db7549a9796acc4508f22 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P e95cf2c576dda656c0f31eeec3d98e911b9003a1 -R 23e2e1649d860cacfc7d885e0481975c -U mistachkin -Z 106f8d2e0fe01f69aa7179a809ac1874 +P 88b763e8d73fafa1538b08af28b1c8b723b39c61 +R b27c1f3a83b2f5462f0150a6100017fc +U drh +Z 704d1aefb0f6d4ce0005ee8d827376d9 diff --git a/manifest.uuid b/manifest.uuid index 4613466421..9ff339e8b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88b763e8d73fafa1538b08af28b1c8b723b39c61 \ No newline at end of file +f1bd5bbae505068d24bfd9cc6bab6a8b8940bad6 \ No newline at end of file diff --git a/mkopcodec.awk b/mkopcodec.awk index ec80953009..2ef77d4cca 100644 --- a/mkopcodec.awk +++ b/mkopcodec.awk @@ -17,13 +17,18 @@ BEGIN { print " || defined(SQLITE_DEBUG)" print "const char *sqlite3OpcodeName(int i){" print " static const char *const azName[] = { \"?\"," + mx = 0 } /define OP_/ { sub("OP_","",$2) - i++ - printf " /* %3d */ \"%s\",\n", $3, $2 + i = $3+0 + label[i] = $2 + if( mx Date: Tue, 30 Aug 2011 19:52:32 +0000 Subject: [PATCH 12/36] Enable the thread test logic to work with the SQLITE_HAS_CODEC compile-time option. FossilOrigin-Name: 20ddfb4780b87953718f3a8e67b777dcff0e3b5e --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/test_thread.c | 17 ++++++++++++++++- test/thread001.test | 4 ++-- test/thread002.test | 2 ++ test/thread003.test | 6 +++--- 6 files changed, 33 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 27ea6ae63d..fdf9ba1eff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sbuild\sprocess\sso\sthat\sit\sdoes\snot\srequire\sthe\sunix\s"sort"\ncommand.\s\sThis\savoids\sconfusion\sbetween\sms-sort\sand\smingw-sort\son\nwindows\ssystems. -D 2011-08-30T01:29:04.104 +C Enable\sthe\sthread\stest\slogic\sto\swork\swith\sthe\sSQLITE_HAS_CODEC\scompile-time\noption. +D 2011-08-30T19:52:32.227 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/test_stat.c f682704b5d1ba8e1d4e7e882a6d7922e2dcf066c F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa -F src/test_thread.c dc77f920d24f2f515bd315b87942b6396332a414 +F src/test_thread.c 35022393dd54d147b998b6b7f7e945b01114d666 F src/test_vfs.c b0baec983bd6f872715a4b44c8f39104fec333af F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 @@ -700,9 +700,9 @@ F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d F test/tester.tcl 0b2999b578964297663de4870babbbee29225622 -F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f -F test/thread002.test 716631b06cccf33b368ab7f6dd3cad92907b8928 -F test/thread003.test 33d2d46e6a53ccb2ff8dc4d0c4e3b3aaee36dcd1 +F test/thread001.test 7cc2ce08f9cde95964736d11e91f9ab610f82f91 +F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 +F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 F test/thread004.test f51dfc3936184aaf73ee85f315224baad272a87f F test/thread005.test 50d10b5684399676174bd96c94ad4250b1a2c8b6 F test/thread1.test df115faa10a4ba1d456e9d4d9ec165016903eae4 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 88b763e8d73fafa1538b08af28b1c8b723b39c61 -R b27c1f3a83b2f5462f0150a6100017fc +P f1bd5bbae505068d24bfd9cc6bab6a8b8940bad6 +R 6d1c7722e8d08f5c9ec39c32c435674d U drh -Z 704d1aefb0f6d4ce0005ee8d827376d9 +Z 883417057169f45a687263a717525500 diff --git a/manifest.uuid b/manifest.uuid index 9ff339e8b2..8b4b3fb8bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1bd5bbae505068d24bfd9cc6bab6a8b8940bad6 \ No newline at end of file +20ddfb4780b87953718f3a8e67b777dcff0e3b5e \ No newline at end of file diff --git a/src/test_thread.c b/src/test_thread.c index 08df14c2ce..aa89467071 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -282,6 +282,21 @@ static int sqlthread_open( zFilename = Tcl_GetString(objv[2]); rc = sqlite3_open(zFilename, &db); +#ifdef SQLITE_HAS_CODEC + if( db && objc>=4 ){ + const char *zKey; + int nKey; + zKey = Tcl_GetStringFromObj(objv[3], &nKey); + rc = sqlite3_key(db, zKey, nKey); + if( rc!=SQLITE_OK ){ + char *zErrMsg = sqlite3_mprintf("error %d: %s", rc, sqlite3_errmsg(db)); + sqlite3_close(db); + Tcl_AppendResult(interp, zErrMsg, (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + } +#endif Md5_Register(db); sqlite3_busy_handler(db, xBusy, 0); @@ -349,7 +364,7 @@ static int sqlthread_proc( if( rc!=TCL_OK ) return rc; pSub = &aSub[iIndex]; - if( objc!=(pSub->nArg+2) ){ + if( objc<(pSub->nArg+2) ){ Tcl_WrongNumArgs(interp, 2, objv, pSub->zUsage); return TCL_ERROR; } diff --git a/test/thread001.test b/test/thread001.test index 9b788aafaf..7e0893f29d 100644 --- a/test/thread001.test +++ b/test/thread001.test @@ -42,7 +42,7 @@ foreach {tn same_db shared_cache} [list \ sqlite3_enable_shared_cache $shared_cache sqlite3_enable_shared_cache $shared_cache } $shared_cache - sqlite3 db test.db -fullmutex 1 + sqlite3 db test.db -fullmutex 1 -key xyzzy set dbconfig "" if {$same_db} { @@ -77,7 +77,7 @@ foreach {tn same_db shared_cache} [list \ #sqlthread parent {puts STARTING..} set needToClose 0 if {![info exists ::DB]} { - set ::DB [sqlthread open test.db] + set ::DB [sqlthread open test.db xyzzy] #sqlthread parent "puts \"OPEN $::DB\"" set needToClose 1 } diff --git a/test/thread002.test b/test/thread002.test index 60071d9243..b39c9ae7cd 100644 --- a/test/thread002.test +++ b/test/thread002.test @@ -16,9 +16,11 @@ set testdir [file dirname $argv0] +set do_not_use_codec 1 source $testdir/tester.tcl if {[run_thread_tests]==0} { finish_test ; return } + db close set ::enable_shared_cache [sqlite3_enable_shared_cache 1] diff --git a/test/thread003.test b/test/thread003.test index 22cd534c88..8c7440c82e 100644 --- a/test/thread003.test +++ b/test/thread003.test @@ -80,7 +80,7 @@ do_test thread003.2 { foreach zFile {test.db test2.db} { set SCRIPT [format { set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open %s] + set ::DB [sqlthread open %s xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } @@ -117,7 +117,7 @@ do_test thread003.3 { set SCRIPT [format { set iStart [clock_seconds] set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open %s] + set ::DB [sqlthread open %s xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } @@ -156,7 +156,7 @@ unset -nocomplain finished(2) do_test thread003.4 { thread_spawn finished(1) $thread_procs [format { set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open test.db] + set ::DB [sqlthread open test.db xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } From e42a9b431b1a012a2b4e00acbeddac21de049711 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 13:27:19 +0000 Subject: [PATCH 13/36] Add checks to make sure cells in corrupt database files do not overflow a page when doing autovacuum. Problem detected by valgrind. FossilOrigin-Name: d0b347b412376d22e9f0770ac083dafb5e480dd0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 +++++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index fdf9ba1eff..4c460d652a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\sthread\stest\slogic\sto\swork\swith\sthe\sSQLITE_HAS_CODEC\scompile-time\noption. -D 2011-08-30T19:52:32.227 +C Add\schecks\sto\smake\ssure\scells\sin\scorrupt\sdatabase\sfiles\s\ndo\snot\soverflow\sa\spage\swhen\sdoing\sautovacuum.\nProblem\sdetected\sby\svalgrind. +D 2011-08-31T13:27:19.588 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bd89d604a532063da8ed1a095f1805db49896325 +F src/btree.c 4a2856b3bde9959986a7b9327841b3ff94023784 F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P f1bd5bbae505068d24bfd9cc6bab6a8b8940bad6 -R 6d1c7722e8d08f5c9ec39c32c435674d +P 20ddfb4780b87953718f3a8e67b777dcff0e3b5e +R 513927bc09bdb01972234dc3d07878fd U drh -Z 883417057169f45a687263a717525500 +Z 7574b78d098e12a356337eb2bfd798e6 diff --git a/manifest.uuid b/manifest.uuid index 8b4b3fb8bf..baf170c9f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20ddfb4780b87953718f3a8e67b777dcff0e3b5e \ No newline at end of file +d0b347b412376d22e9f0770ac083dafb5e480dd0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d77fce4c8e..7166b93b90 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2754,11 +2754,12 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; btreeParseCellPtr(pPage, pCell, &info); - if( info.iOverflow ){ - if( iFrom==get4byte(&pCell[info.iOverflow]) ){ - put4byte(&pCell[info.iOverflow], iTo); - break; - } + if( info.iOverflow + && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage + && iFrom==get4byte(&pCell[info.iOverflow]) + ){ + put4byte(&pCell[info.iOverflow], iTo); + break; } }else{ if( get4byte(pCell)==iFrom ){ @@ -5190,6 +5191,9 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } + if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ + return SQLITE_CORRUPT; /* Cell extends past end of page */ + } ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; From 4515a45b8c8dad991941c164603dc310b6c1c48a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 17:46:50 +0000 Subject: [PATCH 14/36] Backslash escaping is not working right in tostr.awk on the latest ubuntu. The easiest fix is to simply not use any backslashes in the spaceanal.tcl script. FossilOrigin-Name: df550066657cb41bad34ac8e722b8148ab7eabfb --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/spaceanal.tcl | 25 +++++++++++-------------- tool/tostr.awk | 1 - 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 4c460d652a..fa2504aba7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\schecks\sto\smake\ssure\scells\sin\scorrupt\sdatabase\sfiles\s\ndo\snot\soverflow\sa\spage\swhen\sdoing\sautovacuum.\nProblem\sdetected\sby\svalgrind. -D 2011-08-31T13:27:19.588 +C Backslash\sescaping\sis\snot\sworking\sright\sin\stostr.awk\son\sthe\slatest\subuntu.\nThe\seasiest\sfix\sis\sto\ssimply\snot\suse\sany\sbackslashes\sin\sthe\sspaceanal.tcl\nscript. +D 2011-08-31T17:46:50.850 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -950,7 +950,7 @@ F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 F tool/showwal.c f09e5a80a293919290ec85a6a37c85a5ddcf37d9 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b -F tool/spaceanal.tcl b91879d52bf77a1ff5382493284f429d32a63490 +F tool/spaceanal.tcl 1ee4df4e190675ba67b8c60cf304496d0021cfb4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -958,10 +958,10 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 -F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 +F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 20ddfb4780b87953718f3a8e67b777dcff0e3b5e -R 513927bc09bdb01972234dc3d07878fd +P d0b347b412376d22e9f0770ac083dafb5e480dd0 +R b71c3db157ba0c4a396659741c76245f U drh -Z 7574b78d098e12a356337eb2bfd798e6 +Z 6dc79f498e7b45dccac5e2d58d584bd0 diff --git a/manifest.uuid b/manifest.uuid index baf170c9f2..16eff18a9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0b347b412376d22e9f0770ac083dafb5e480dd0 \ No newline at end of file +df550066657cb41bad34ac8e722b8148ab7eabfb \ No newline at end of file diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index bf6244e8c9..13ab1af5f7 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -41,8 +41,7 @@ set pageSize [db one {PRAGMA page_size}] # queries the in-memory db to produce the space-analysis report. # sqlite3 mem :memory: -set tabledef\ -{CREATE TABLE space_used( +set tabledef {CREATE TABLE space_used( name clob, -- Name of a table or index in the database file tblname clob, -- Name of associated table is_index boolean, -- TRUE if it is an index, false for a table @@ -293,16 +292,16 @@ proc subreport {title where} { statline {Overflow pages used} $ovfl_pages statline {Total pages used} $total_pages if {$int_unused>0} { - set int_unused_percent \ - [percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] + set int_unused_percent [ + percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] statline "Unused bytes on index pages" $int_unused $int_unused_percent } - statline "Unused bytes on primary pages" $leaf_unused \ - [percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] - statline "Unused bytes on overflow pages" $ovfl_unused \ - [percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] - statline "Unused bytes on all pages" $total_unused \ - [percent $total_unused $storage {of all space}] + statline "Unused bytes on primary pages" $leaf_unused [ + percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] + statline "Unused bytes on overflow pages" $ovfl_unused [ + percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] + statline "Unused bytes on all pages" $total_unused [ + percent $total_unused $storage {of all space}] return 1 } @@ -452,11 +451,9 @@ Page size in bytes Number of pages in the whole file } -puts \ -" The number of $pageSize-byte pages that go into forming the complete +puts " The number of $pageSize-byte pages that go into forming the complete database" -puts \ -{ +puts { Pages that store data The number of pages that store data, either as primary B*Tree pages or diff --git a/tool/tostr.awk b/tool/tostr.awk index 83c6cc1a50..b4f48d3db5 100644 --- a/tool/tostr.awk +++ b/tool/tostr.awk @@ -3,7 +3,6 @@ # Convert input text into a C string # { - gsub(/\\/,"\\\\"); gsub(/\"/,"\\\""); print "\"" $0 "\\n\""; } From 7730c4457ec47a04184be4756cb1a4751bcc268b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 18:04:54 +0000 Subject: [PATCH 15/36] Only create the sqlite3OSTrace global variable if compiling with SQLITE_TEST. FossilOrigin-Name: 9e6a4c1473a4cb061a88f6a437a2828368b3ddc8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_common.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fa2504aba7..0eeabd1cea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Backslash\sescaping\sis\snot\sworking\sright\sin\stostr.awk\son\sthe\slatest\subuntu.\nThe\seasiest\sfix\sis\sto\ssimply\snot\suse\sany\sbackslashes\sin\sthe\sspaceanal.tcl\nscript. -D 2011-08-31T17:46:50.850 +C Only\screate\sthe\ssqlite3OSTrace\sglobal\svariable\sif\scompiling\swith\s\nSQLITE_TEST. +D 2011-08-31T18:04:54.752 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 -F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 +F src/os_common.h b15945976976c80e9a29c2d331a9c6fb42d3efa5 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c ae82cf32c497d9a3a0f147de1b7219b636db4f53 F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P d0b347b412376d22e9f0770ac083dafb5e480dd0 -R b71c3db157ba0c4a396659741c76245f +P df550066657cb41bad34ac8e722b8148ab7eabfb +R ba2041aae6bc957791be99b48d3a5d63 U drh -Z 6dc79f498e7b45dccac5e2d58d584bd0 +Z 1de22a554319629ea944db035b2e3bd4 diff --git a/manifest.uuid b/manifest.uuid index 16eff18a9f..cad37e447c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -df550066657cb41bad34ac8e722b8148ab7eabfb \ No newline at end of file +9e6a4c1473a4cb061a88f6a437a2828368b3ddc8 \ No newline at end of file diff --git a/src/os_common.h b/src/os_common.h index aa3e18a8c5..f95e6181a0 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -29,7 +29,7 @@ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#ifdef SQLITE_DEBUG +#ifdef SQLITE_TEST # ifndef SQLITE_DEBUG_OS_TRACE # define SQLITE_DEBUG_OS_TRACE 0 # endif From fd53231c8587ac7eb19e0b8b5d92af1c865e23ab Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 18:35:34 +0000 Subject: [PATCH 16/36] Always include the unixShm.id field, even when not debugging. FossilOrigin-Name: 07803476206b8cde60ccc320b9ccdc0d48a41d65 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0eeabd1cea..b378643159 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\screate\sthe\ssqlite3OSTrace\sglobal\svariable\sif\scompiling\swith\s\nSQLITE_TEST. -D 2011-08-31T18:04:54.752 +C Always\sinclude\sthe\sunixShm.id\sfield,\seven\swhen\snot\sdebugging. +D 2011-08-31T18:35:34.234 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h b15945976976c80e9a29c2d331a9c6fb42d3efa5 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c ae82cf32c497d9a3a0f147de1b7219b636db4f53 +F src/os_unix.c 9e6e26b96b10e62a5dc678ea95bbb16d05fcca6e F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P df550066657cb41bad34ac8e722b8148ab7eabfb -R ba2041aae6bc957791be99b48d3a5d63 +P 9e6a4c1473a4cb061a88f6a437a2828368b3ddc8 +R 834dac199e1c9ebe831f91eccddde409 U drh -Z 1de22a554319629ea944db035b2e3bd4 +Z 761dac1f8bf1d111d08b2e615a88c98b diff --git a/manifest.uuid b/manifest.uuid index cad37e447c..5bdd31701d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e6a4c1473a4cb061a88f6a437a2828368b3ddc8 \ No newline at end of file +07803476206b8cde60ccc320b9ccdc0d48a41d65 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f273d04cb6..1c5a5bad56 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3640,11 +3640,9 @@ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG - u8 id; /* Id of this connection within its unixShmNode */ -#endif }; /* From 1b1e8a8be681b17388a98b5ba3af361e12415e58 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 19:40:58 +0000 Subject: [PATCH 17/36] The server1.test script should only run if mutexes are functional. Fix to the #ifdef change of check-in [9e6a4c1473]. FossilOrigin-Name: 6489848590db7088d965b7fa7feabc8c66e2e2b1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_common.h | 2 +- test/server1.test | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b378643159..bb36b63d67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sinclude\sthe\sunixShm.id\sfield,\seven\swhen\snot\sdebugging. -D 2011-08-31T18:35:34.234 +C The\sserver1.test\sscript\sshould\sonly\srun\sif\smutexes\sare\sfunctional.\nFix\sto\sthe\s#ifdef\schange\sof\scheck-in\s[9e6a4c1473]. +D 2011-08-31T19:40:58.259 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 -F src/os_common.h b15945976976c80e9a29c2d331a9c6fb42d3efa5 +F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 9e6e26b96b10e62a5dc678ea95bbb16d05fcca6e F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 @@ -661,7 +661,7 @@ F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 0d072c5846071b569766e6cd7f923f646a8b2bfa F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 -F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c +F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 34945a516532b11182c3eb26e31247eee3c9ae48 F test/shared2.test 8f71d4eb4d5261280de92284df74172545c852cc F test/shared3.test ebf77f023f4bdaa8f74f65822b559e86ce5c6257 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 9e6a4c1473a4cb061a88f6a437a2828368b3ddc8 -R 834dac199e1c9ebe831f91eccddde409 +P 07803476206b8cde60ccc320b9ccdc0d48a41d65 +R eb9ef004318c49388ea9aaadc3e8b5e6 U drh -Z 761dac1f8bf1d111d08b2e615a88c98b +Z 078a9843a586847568ff69812d8e24f3 diff --git a/manifest.uuid b/manifest.uuid index 5bdd31701d..2948eda20d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07803476206b8cde60ccc320b9ccdc0d48a41d65 \ No newline at end of file +6489848590db7088d965b7fa7feabc8c66e2e2b1 \ No newline at end of file diff --git a/src/os_common.h b/src/os_common.h index f95e6181a0..f6c3e7ff89 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -29,7 +29,7 @@ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#ifdef SQLITE_TEST +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) # ifndef SQLITE_DEBUG_OS_TRACE # define SQLITE_DEBUG_OS_TRACE 0 # endif diff --git a/test/server1.test b/test/server1.test index 134a9f5773..90673ef000 100644 --- a/test/server1.test +++ b/test/server1.test @@ -29,7 +29,7 @@ if {[llength [info command client_step]]==0 || [sqlite3 -has-codec]} { # The sample server implementation does not work right when memory # management is enabled. # -ifcapable memorymanage { +ifcapable (memorymanage||mutex_noop) { finish_test return } From 5a201fb41df348845f3db85a2d00e59e5d7a16c0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 20:47:50 +0000 Subject: [PATCH 18/36] Add the SQLITE_MAX_SCHEMA_RETRY compile-time option to the set of options understood by "PRAGMA compile_options;" and by the "sqlite_compileoption_used()" function. FossilOrigin-Name: 1b124af40a8fa4d3094f24a9213096b66411b8f9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/ctime.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index bb36b63d67..de84cf9560 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sserver1.test\sscript\sshould\sonly\srun\sif\smutexes\sare\sfunctional.\nFix\sto\sthe\s#ifdef\schange\sof\scheck-in\s[9e6a4c1473]. -D 2011-08-31T19:40:58.259 +C Add\sthe\sSQLITE_MAX_SCHEMA_RETRY\scompile-time\soption\sto\sthe\sset\sof\soptions\nunderstood\sby\s"PRAGMA\scompile_options;"\sand\sby\sthe\s\n"sqlite_compileoption_used()"\sfunction. +D 2011-08-31T20:47:50.717 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c caf51429be3e0d4114056a8273b0fff812ff8ae9 +F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 07803476206b8cde60ccc320b9ccdc0d48a41d65 -R eb9ef004318c49388ea9aaadc3e8b5e6 +P 6489848590db7088d965b7fa7feabc8c66e2e2b1 +R b679703af471462fe3522547f0d4dc68 U drh -Z 078a9843a586847568ff69812d8e24f3 +Z 6eb5e5b5812def8adf95eaa9a9b59bfd diff --git a/manifest.uuid b/manifest.uuid index 2948eda20d..136b8186f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6489848590db7088d965b7fa7feabc8c66e2e2b1 \ No newline at end of file +1b124af40a8fa4d3094f24a9213096b66411b8f9 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 77174d0dae..cbf8ed55fa 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -144,6 +144,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif #ifdef SQLITE_MEMDEBUG "MEMDEBUG", #endif From 20f8e13b4474622d9dced4bb8c2422656f114f22 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 21:01:55 +0000 Subject: [PATCH 19/36] Formerly, we enabled fdatasync() on linux only. But now we learn that fdatasync() is not supported on Android. So we disable fdatasync() on Linux too. It can be reenabled at compile-time for those who really need it. FossilOrigin-Name: 70b5b309568ac55565558d5456aca1e431cfd26b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index de84cf9560..cff029dee2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_MAX_SCHEMA_RETRY\scompile-time\soption\sto\sthe\sset\sof\soptions\nunderstood\sby\s"PRAGMA\scompile_options;"\sand\sby\sthe\s\n"sqlite_compileoption_used()"\sfunction. -D 2011-08-31T20:47:50.717 +C Formerly,\swe\senabled\sfdatasync()\son\slinux\sonly.\s\sBut\snow\swe\slearn\sthat\nfdatasync()\sis\snot\ssupported\son\sAndroid.\s\sSo\swe\sdisable\sfdatasync()\son\nLinux\stoo.\s\sIt\scan\sbe\sreenabled\sat\scompile-time\sfor\sthose\swho\sreally\sneed\sit. +D 2011-08-31T21:01:55.686 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 9e6e26b96b10e62a5dc678ea95bbb16d05fcca6e +F src/os_unix.c 10e0c4dcdbec8d4189890fdf3e71b32efae194e3 F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 6489848590db7088d965b7fa7feabc8c66e2e2b1 -R b679703af471462fe3522547f0d4dc68 +P 1b124af40a8fa4d3094f24a9213096b66411b8f9 +R e3c435a6ba2306a2d838c8b23577f672 U drh -Z 6eb5e5b5812def8adf95eaa9a9b59bfd +Z 9e7ae74ddd5128c7f3fb2205f914d095 diff --git a/manifest.uuid b/manifest.uuid index 136b8186f2..5cdc6053a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b124af40a8fa4d3094f24a9213096b66411b8f9 \ No newline at end of file +70b5b309568ac55565558d5456aca1e431cfd26b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1c5a5bad56..d85a7949b5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3146,11 +3146,11 @@ int sqlite3_fullsync_count = 0; /* ** We do not trust systems to provide a working fdatasync(). Some do. -** Others do no. To be safe, we will stick with the (slower) fsync(). -** If you know that your system does support fdatasync() correctly, +** Others do no. To be safe, we will stick with the (slightly slower) +** fsync(). If you know that your system does support fdatasync() correctly, ** then simply compile with -Dfdatasync=fdatasync */ -#if !defined(fdatasync) && !defined(__linux__) +#if !defined(fdatasync) # define fdatasync fsync #endif From 96862072b843e2431f7723b972618982e7d395d5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Aug 2011 23:57:22 +0000 Subject: [PATCH 20/36] Avoid using uninitialized variables after failures in the merge sort code. FossilOrigin-Name: 2869ed28299b1c9f355ecc24635830f7f1249126 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 43 ++++++++++++++++++++++--------------------- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index cff029dee2..cf95f75fde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Formerly,\swe\senabled\sfdatasync()\son\slinux\sonly.\s\sBut\snow\swe\slearn\sthat\nfdatasync()\sis\snot\ssupported\son\sAndroid.\s\sSo\swe\sdisable\sfdatasync()\son\nLinux\stoo.\s\sIt\scan\sbe\sreenabled\sat\scompile-time\sfor\sthose\swho\sreally\sneed\sit. -D 2011-08-31T21:01:55.686 +C Avoid\susing\suninitialized\svariables\safter\sfailures\sin\sthe\smerge\ssort\scode. +D 2011-08-31T23:57:22.695 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c de1e4cab060a45df9ebee68dd63543d14559f0e7 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e +F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 1b124af40a8fa4d3094f24a9213096b66411b8f9 -R e3c435a6ba2306a2d838c8b23577f672 +P 70b5b309568ac55565558d5456aca1e431cfd26b +R 09e04eb9163c46e529a61f2438cff2a1 U drh -Z 9e7ae74ddd5128c7f3fb2205f914d095 +Z bdb2832352809bf647849d42b7aa6060 diff --git a/manifest.uuid b/manifest.uuid index 5cdc6053a0..67e9d42efe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70b5b309568ac55565558d5456aca1e431cfd26b \ No newline at end of file +2869ed28299b1c9f355ecc24635830f7f1249126 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index be99d397d3..c3214b3afe 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -142,8 +142,8 @@ static int vdbeSorterIterNext( ){ int rc; /* Return Code */ int nRead; /* Number of bytes read */ - int nRec; /* Size of record in bytes */ - int iOff; /* Size of serialized size varint in bytes */ + int nRec = 0; /* Size of record in bytes */ + int iOff = 0; /* Size of serialized size varint in bytes */ nRead = pIter->iEof - pIter->iReadOff; if( nRead>5 ) nRead = 5; @@ -154,25 +154,26 @@ static int vdbeSorterIterNext( } rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff); - iOff = getVarint32(pIter->aAlloc, nRec); - - if( rc==SQLITE_OK && (iOff+nRec)>nRead ){ - int nRead2; /* Number of extra bytes to read */ - if( (iOff+nRec)>pIter->nAlloc ){ - int nNew = pIter->nAlloc*2; - while( (iOff+nRec)>nNew ) nNew = nNew*2; - pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); - if( !pIter->aAlloc ) return SQLITE_NOMEM; - pIter->nAlloc = nNew; + if( rc==SQLITE_OK ){ + iOff = getVarint32(pIter->aAlloc, nRec); + if( (iOff+nRec)>nRead ){ + int nRead2; /* Number of extra bytes to read */ + if( (iOff+nRec)>pIter->nAlloc ){ + int nNew = pIter->nAlloc*2; + while( (iOff+nRec)>nNew ) nNew = nNew*2; + pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); + if( !pIter->aAlloc ) return SQLITE_NOMEM; + pIter->nAlloc = nNew; + } + + nRead2 = iOff + nRec - nRead; + rc = sqlite3OsRead( + pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead + ); } - - nRead2 = iOff + nRec - nRead; - rc = sqlite3OsRead( - pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead - ); } - assert( nRec>0 || rc!=SQLITE_OK ); + assert( rc!=SQLITE_OK || nRec>0 ); pIter->iReadOff += iOff+nRec; pIter->nKey = nRec; pIter->aKey = &pIter->aAlloc[iOff]; @@ -543,12 +544,12 @@ static int vdbeSorterInitMerge( i64 nByte = 0; /* Total bytes in all opened PMAs */ /* Initialize the iterators. */ - for(i=0; rc==SQLITE_OK && iaIter[i]; rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); pSorter->iReadOff = pIter->iEof; - assert( pSorter->iReadOff<=pSorter->iWriteOff || rc!=SQLITE_OK ); - if( pSorter->iReadOff>=pSorter->iWriteOff ) break; + assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); + if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; } /* Initialize the aTree[] array. */ From c6aff30ca470f67c72c3a133ad3616989657fcec Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Sep 2011 15:32:47 +0000 Subject: [PATCH 21/36] Experimental code-generator changes to utilize new opcodes for sorting. FossilOrigin-Name: bab2e560f6cb989c83a96aad60f666960ede7abe --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/select.c | 35 ++++++++++++++++++++++++++++++----- src/sqliteInt.h | 1 + src/vdbe.c | 12 +++++++++++- src/vdbeInt.h | 1 + src/vdbeaux.c | 2 +- 7 files changed, 58 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index cf95f75fde..b233b88396 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\suninitialized\svariables\safter\sfailures\sin\sthe\smerge\ssort\scode. -D 2011-08-31T23:57:22.695 +C Experimental\scode-generator\schanges\sto\sutilize\snew\sopcodes\sfor\ssorting. +D 2011-09-01T15:32:47.873 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 14552e9ff4b27ec027a43fafb62ea5d049cd2809 +F src/select.c 20bef6860c5f69c6a90666b4968d4c5cb88056c0 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 86a4fdb3ba9ab31d98b266797606f30fefe5b8a9 +F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,11 +238,11 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 9165b35da939f4a66c7e68b0c6d3f017ca982cb1 +F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h 70767f6504aac4f0057ec2a55738470a890789ac +F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c de1e4cab060a45df9ebee68dd63543d14559f0e7 +F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167 @@ -961,7 +961,10 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 70b5b309568ac55565558d5456aca1e431cfd26b -R 09e04eb9163c46e529a61f2438cff2a1 +P 2869ed28299b1c9f355ecc24635830f7f1249126 +R e08eee2093eee8a9fb40d6646a4c9c76 +T *branch * merge-sort +T *sym-merge-sort * +T -sym-trunk * U drh -Z bdb2832352809bf647849d42b7aa6060 +Z c5e8ef1774d371c2becb6d0057b76727 diff --git a/manifest.uuid b/manifest.uuid index 67e9d42efe..c35621c659 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2869ed28299b1c9f355ecc24635830f7f1249126 \ No newline at end of file +bab2e560f6cb989c83a96aad60f666960ede7abe \ No newline at end of file diff --git a/src/select.c b/src/select.c index 2e44522c6d..8825326594 100644 --- a/src/select.c +++ b/src/select.c @@ -419,12 +419,18 @@ static void pushOntoSorter( int nExpr = pOrderBy->nExpr; int regBase = sqlite3GetTempRange(pParse, nExpr+2); int regRecord = sqlite3GetTempReg(pParse); + int op; sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); + if( pSelect->selFlags & SF_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); if( pSelect->iLimit ){ @@ -893,9 +899,20 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); + if( p->selFlags & SF_UseSorter ){ + int regSortOut = sqlite3GetTempReg(pParse); + int ptab2 = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -948,7 +965,11 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + if( p->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + } sqlite3VdbeResolveLabel(v, addrBreak); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); @@ -3914,6 +3935,10 @@ int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = (double)LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; + p->selFlags |= SF_UseSorter; + } /* Open a virtual index to use for the distinct set. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5934c74316..18beb32e0f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2082,6 +2082,7 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x0040 /* Sort using a sorter */ /* diff --git a/src/vdbe.c b/src/vdbe.c index e7c7ed7fe2..f4ae4c44a6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3170,6 +3170,7 @@ case OP_OpenWrite: { */ case OP_OpenSorter: case OP_OpenAutoindex: +case OP_SorterOpen: case OP_OpenEphemeral: { VdbeCursor *pCx; static const int vfsFlags = @@ -3214,6 +3215,7 @@ case OP_OpenEphemeral: { } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); pCx->isIndex = !pCx->isTable; + pCx->isSorter = pOp->opcode==OP_SorterOpen; #ifndef SQLITE_OMIT_MERGE_SORT if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){ rc = sqlite3VdbeSorterInit(db, pCx); @@ -4090,6 +4092,7 @@ case OP_ResetCount: { ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ +case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; @@ -4103,11 +4106,12 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC->isTable || pOp->opcode==OP_RowKey ); + assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isIndex || pOp->opcode==OP_RowData ); assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); if( isSorter(pC) ){ assert( pOp->opcode==OP_RowKey ); @@ -4271,6 +4275,7 @@ case OP_Last: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ +case OP_SorterSort: /* jump */ case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4295,6 +4300,7 @@ case OP_Rewind: { /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(db, pC, &res); @@ -4347,6 +4353,7 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ +case OP_SorterNext: /* jump */ case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4359,6 +4366,7 @@ case OP_Next: { /* jump */ if( pC==0 ){ break; /* See ticket #2273 */ } + assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ assert( pOp->opcode==OP_Next ); rc = sqlite3VdbeSorterNext(db, pC, &res); @@ -4395,6 +4403,7 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ +case OP_SorterInsert: case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; @@ -4404,6 +4413,7 @@ case OP_IdxInsert: { /* in2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); pCrsr = pC->pCursor; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 846d807075..70d80c9d38 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -59,6 +59,7 @@ struct VdbeCursor { Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ + Bool isSorter; /* True if a new-style sorter */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 053d89f3b5..f622e8d09a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -433,7 +433,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; #endif - }else if( opcode==OP_Next ){ + }else if( opcode==OP_Next || opcode==OP_SorterNext ){ pOp->p4.xAdvance = sqlite3BtreeNext; pOp->p4type = P4_ADVANCE; }else if( opcode==OP_Prev ){ From 1c9d835d495f2c23642dce5956ffb3dfcaa80d22 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Sep 2011 16:01:27 +0000 Subject: [PATCH 22/36] Use OP_SorterOpen instead of OP_OpenEphemeral to implement GROUP BY. FossilOrigin-Name: ebf819aaa555bd79fddfc0a6f9827a2539095d6c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/select.c | 21 +++++++++++++++------ test/distinct.test | 2 +- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index b233b88396..4ed8e1a4b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\scode-generator\schanges\sto\sutilize\snew\sopcodes\sfor\ssorting. -D 2011-09-01T15:32:47.873 +C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY. +D 2011-09-01T16:01:27.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 20bef6860c5f69c6a90666b4968d4c5cb88056c0 +F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -369,7 +369,7 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 8c4d951fc40aba84421060e07b16099d2f4c2fdf +F test/distinct.test df5b11ad606439129c88720a86787bc9ca181f31 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -961,10 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 2869ed28299b1c9f355ecc24635830f7f1249126 -R e08eee2093eee8a9fb40d6646a4c9c76 -T *branch * merge-sort -T *sym-merge-sort * -T -sym-trunk * +P bab2e560f6cb989c83a96aad60f666960ede7abe +R 47ae22012e17b717f429ee6a9d305e96 U drh -Z c5e8ef1774d371c2becb6d0057b76727 +Z 4edda214a8d1e652ef01c1f98ad8fd39 diff --git a/manifest.uuid b/manifest.uuid index c35621c659..e5664d914b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bab2e560f6cb989c83a96aad60f666960ede7abe \ No newline at end of file +ebf819aaa555bd79fddfc0a6f9827a2539095d6c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 8825326594..d8332eaaea 100644 --- a/src/select.c +++ b/src/select.c @@ -4033,6 +4033,8 @@ int sqlite3Select( int iAbortFlag; /* Mem address which causes query abort if positive */ int groupBySort; /* Rows come from source in GROUP BY order */ int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -4094,12 +4096,12 @@ int sqlite3Select( /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out - ** that we do not need it after all, the OpenEphemeral instruction + ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); @@ -4180,11 +4182,14 @@ int sqlite3Select( } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); + sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); @@ -4197,9 +4202,13 @@ int sqlite3Select( */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + } for(j=0; jnExpr; j++){ if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); @@ -4238,7 +4247,7 @@ int sqlite3Select( /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); }else{ sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); diff --git a/test/distinct.test b/test/distinct.test index 87456de876..e0a913604d 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -45,7 +45,7 @@ proc do_temptables_test {tn sql temptables} { uplevel [list do_test $tn [subst -novar { set ret "" db eval "EXPLAIN [set sql]" { - if {$opcode == "OpenEphemeral"} { + if {$opcode == "OpenEphemeral" || $opcode == "SorterOpen"} { if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" } if {$p5 == "10"} { lappend ret hash From 5134d1357e477c856109cecddb2e59be320eef0a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 10:31:11 +0000 Subject: [PATCH 23/36] Instead of a temporary b-tree, use a linked-list and merge-sort to sort records in main memory in vdbesort.c. FossilOrigin-Name: 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 --- manifest | 32 ++-- manifest.uuid | 2 +- src/btree.c | 11 -- src/btree.h | 1 - src/build.c | 26 ++- src/expr.c | 2 +- src/select.c | 2 +- src/sqliteInt.h | 1 + src/vdbe.c | 90 ++++++---- src/vdbeInt.h | 6 +- src/vdbesort.c | 438 +++++++++++++++++++++++++++++++++-------------- test/index4.test | 14 ++ 12 files changed, 423 insertions(+), 202 deletions(-) diff --git a/manifest b/manifest index 4ed8e1a4b9..9513f2ff2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sOP_SorterOpen\sinstead\sof\sOP_OpenEphemeral\sto\simplement\sGROUP\sBY. -D 2011-09-01T16:01:27.777 +C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c. +D 2011-09-02T10:31:11.173 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,16 +124,16 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4a2856b3bde9959986a7b9327841b3ff94023784 -F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 +F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0 +F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd +F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 -F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 +F src/expr.c cbcd8c2f1588a9862291a081699854c5e1cb28ab F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 9f00ea98f6b360d477b5a78b5b59a1fbde82431c F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 037ee5501fe0e743fa98936902c200ed9ed69156 +F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h f6debf9a9eb8463ab2ef8be4b2b740ea9af5afba +F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 9260e5138855399bea2611a29da336688bfa1b79 +F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h 51a902e12c7d571e3b516e5407e30f996494aafe +F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c f3d043a1bab7409d4a23cd7a35287c3ac440a167 +F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -511,7 +511,7 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637 F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 -F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f +F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P bab2e560f6cb989c83a96aad60f666960ede7abe -R 47ae22012e17b717f429ee6a9d305e96 -U drh -Z 4edda214a8d1e652ef01c1f98ad8fd39 +P ebf819aaa555bd79fddfc0a6f9827a2539095d6c +R c9d892df81c49366e067933871c3b776 +U dan +Z b3581bd03fb5a181801f57c568eb418b diff --git a/manifest.uuid b/manifest.uuid index e5664d914b..f82002ddab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebf819aaa555bd79fddfc0a6f9827a2539095d6c \ No newline at end of file +7769fb988d9be0f2d8129aaac19620ac88f9b4a6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7166b93b90..b335579bd0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1734,22 +1734,11 @@ int sqlite3BtreeOpen( /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); - /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */ -#ifdef SQLITE_OMIT_MERGE_SORT - assert( (flags & BTREE_SORTER)==0 ); -#endif - - /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */ - assert( (flags & BTREE_SORTER)==0 || - (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL)) - ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) ); - if( db->flags & SQLITE_NoReadlock ){ flags |= BTREE_NO_READLOCK; } if( isMemdb ){ flags |= BTREE_MEMORY; - flags &= ~BTREE_SORTER; } if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; diff --git a/src/btree.h b/src/btree.h index ce19826ad8..9e3a73b3b6 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,7 +61,6 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ -#define BTREE_SORTER 32 /* Used as accumulator in external merge sort */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/build.c b/src/build.c index 29fbf92713..0de1020b9b 100644 --- a/src/build.c +++ b/src/build.c @@ -2326,6 +2326,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ @@ -2372,25 +2373,34 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ if( bUseSorter ){ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - sqlite3VdbeChangeP5(v, BTREE_SORTER); } /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + addr2 = addr1 + 1; regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_Sort, iSorter, 0); - sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord); - } - - if( pIndex->onError!=OE_None ){ + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + }else if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); @@ -2411,7 +2421,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1); + sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/expr.c b/src/expr.c index ab4547db9c..ab218078db 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2287,7 +2287,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pCol->iMem; break; }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); break; } diff --git a/src/select.c b/src/select.c index d8332eaaea..7d9d26602e 100644 --- a/src/select.c +++ b/src/select.c @@ -4186,7 +4186,7 @@ int sqlite3Select( sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sortPTab = pParse->nTab++; + sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 18beb32e0f..694f4694eb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1550,6 +1550,7 @@ struct AggInfo { u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ int sortingIdx; /* Cursor number of the sorting index */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ ExprList *pGroupBy; /* The group by clause */ int nSortingColumn; /* Number of columns in the sorting index */ struct AggInfo_col { /* For each column used in source tables */ diff --git a/src/vdbe.c b/src/vdbe.c index f4ae4c44a6..d9310e0563 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3162,15 +3162,7 @@ case OP_OpenWrite: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -/* Opcode: OpenSorter P1 P2 * P4 * -** -** This opcode works like OP_OpenEphemeral except that it opens -** a transient index that is specifically designed to sort large -** tables using an external merge-sort algorithm. -*/ -case OP_OpenSorter: case OP_OpenAutoindex: -case OP_SorterOpen: case OP_OpenEphemeral: { VdbeCursor *pCx; static const int vfsFlags = @@ -3181,7 +3173,6 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); - assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; @@ -3215,12 +3206,24 @@ case OP_OpenEphemeral: { } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); pCx->isIndex = !pCx->isTable; - pCx->isSorter = pOp->opcode==OP_SorterOpen; -#ifndef SQLITE_OMIT_MERGE_SORT - if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){ - rc = sqlite3VdbeSorterInit(db, pCx); - } -#endif + break; +} + +/* Opcode: OpenSorter P1 P2 * P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +*/ +case OP_SorterOpen: +case OP_OpenSorter: { + VdbeCursor *pCx; + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->pKeyInfo = pOp->p4.pKeyInfo; + pCx->pKeyInfo->enc = ENC(p->db); + pCx->isSorter = 1; + rc = sqlite3VdbeSorterInit(db, pCx); break; } @@ -4072,6 +4075,40 @@ case OP_ResetCount: { break; } +/* Opcode: SorterCompare P1 P2 P3 +** +** P1 is a sorter cursor. This instruction compares the record blob in +** register P3 with the entry that the sorter cursor currently points to. +** If, excluding the rowid fields at the end, the two records are a match, +** fall through to the next instruction. Otherwise, jump to instruction P2. +*/ +case OP_SorterCompare: { + VdbeCursor *pC; + int res; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + pIn3 = &aMem[pOp->p3]; + rc = sqlite3VdbeSorterCompare(pC, pIn3, &res); + if( res ){ + pc = pOp->p2-1; + } + break; +}; + +/* Opcode: SorterData P1 P2 * * * +** +** Write into register P2 the current sorter data for sorter cursor P1. +*/ +case OP_SorterData: { + VdbeCursor *pC; + pOut = &aMem[pOp->p2]; + pC = p->apCsr[pOp->p1]; + assert( pC->isSorter ); + rc = sqlite3VdbeSorterRowkey(pC, pOut); + break; +} + /* Opcode: RowData P1 P2 * * * ** ** Write into register P2 the complete row data for cursor P1. @@ -4092,7 +4129,6 @@ case OP_ResetCount: { ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. */ -case OP_SorterData: case OP_RowKey: case OP_RowData: { VdbeCursor *pC; @@ -4106,6 +4142,7 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; + assert( pC->isSorter==0 ); assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isIndex || pOp->opcode==OP_RowData ); assert( pC!=0 ); @@ -4113,12 +4150,6 @@ case OP_RowData: { assert( pC->pseudoTableReg==0 ); assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); - if( isSorter(pC) ){ - assert( pOp->opcode==OP_RowKey ); - rc = sqlite3VdbeSorterRowkey(pC, pOut); - break; - } - assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4368,7 +4399,7 @@ case OP_Next: { /* jump */ } assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ - assert( pOp->opcode==OP_Next ); + assert( pOp->opcode==OP_SorterNext ); rc = sqlite3VdbeSorterNext(db, pC, &res); }else{ res = 1; @@ -4421,16 +4452,17 @@ case OP_IdxInsert: { /* in2 */ assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3VdbeSorterWrite(db, pC, nKey); - if( rc==SQLITE_OK ){ + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); + ); assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; } - pC->cacheStatus = CACHE_STALE; } } break; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 70d80c9d38..68ad8dcbf7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -403,13 +403,15 @@ void sqlite3VdbeMemStoreType(Mem *pMem); # define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK # define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK # define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK +# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK #else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); -int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, int); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *); -int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, Mem *); +int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbesort.c b/src/vdbesort.c index c3214b3afe..3ab5d7930a 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -21,6 +21,7 @@ #ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterRecord SorterRecord; /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: @@ -92,8 +93,7 @@ typedef struct VdbeSorterIter VdbeSorterIter; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { - int nWorking; /* Start a new b-tree after this many pages */ - int nBtree; /* Current size of b-tree contents as PMA */ + int nInMemory; /* Current size of b-tree contents as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ @@ -101,6 +101,9 @@ struct VdbeSorter { i64 iReadOff; /* Current read offset within file pTemp1 */ sqlite3_file *pTemp1; /* PMA file 1 */ int nPMA; /* Number of PMAs stored in pTemp1 */ + SorterRecord *pRecord; /* Head of in-memory record list */ + int nLimit1; /* Minimum PMA size, in bytes */ + int nLimit2; /* Maximum PMA size, in bytes */ }; /* @@ -117,6 +120,17 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ }; +/* +** A structure to store a single record. All in-memory records are connected +** together into a linked list headed at VdbeSorter.pRecord using the +** SorterRecord.pNext pointer. +*/ +struct SorterRecord { + void *pVal; + int nVal; + SorterRecord *pNext; +}; + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -275,6 +289,50 @@ static int vdbeSorterIterInit( return rc; } + +/* +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** size nKey2 bytes). Argument pKeyInfo supplies the collation functions +** used by the comparison. If an error occurs, return an SQLite error code. +** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive +** value, depending on whether key1 is smaller, equal to or larger than key2. +** +** If the bOmitRowid argument is non-zero, assume both keys end in a rowid +** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid +** is true and key1 contains even a single NULL value, it is considered to +** be less than key2. Even if key2 also contains NULL values. +*/ +static int vdbeSorterCompare( + KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + int bOmitRowid, /* Ignore rowid field at end of keys */ + void *pKey1, int nKey1, /* Left side of comparison */ + void *pKey2, int nKey2, /* Right side of comparison */ + int *pRes /* OUT: Result of comparison */ +){ + char aSpace[150]; + UnpackedRecord *r2; + int i; + + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace)); + if( r2==0 ) return SQLITE_NOMEM; + if( bOmitRowid ){ + for(i=0; inField-1; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + sqlite3VdbeDeleteUnpackedRecord(r2); + return SQLITE_OK; + } + } + r2->flags |= UNPACKED_PREFIX_MATCH; + r2->nField--; + assert( r2->nField>0 ); + } + + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); + sqlite3VdbeDeleteUnpackedRecord(r2); + return SQLITE_OK; +} + /* ** This function is called to compare two iterator keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] @@ -306,20 +364,16 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else if( p2->pFile==0 ){ iRes = i1; }else{ - char aSpace[150]; - UnpackedRecord *r1; - - r1 = sqlite3VdbeRecordUnpack( - pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace) + int res; + int rc = vdbeSorterCompare( + pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); - if( r1==0 ) return SQLITE_NOMEM; - - if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){ + if( rc!=SQLITE_OK ) return rc; + if( res<=0 ){ iRes = i1; }else{ iRes = i2; } - sqlite3VdbeDeleteUnpackedRecord(r1); } pSorter->aTree[iOut] = iRes; @@ -330,9 +384,32 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ ** Initialize the temporary index cursor just opened as a sorter cursor. */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ - assert( pCsr->pKeyInfo && pCsr->pBt ); + int pgsz; /* Page size of main database */ + + assert( pCsr->pKeyInfo && pCsr->pBt==0 ); pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM); + if( pCsr->pSorter==0 ){ + return SQLITE_NOMEM; + } + + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pCsr->pSorter->nLimit1 = 10 * pgsz; + pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz; + + return SQLITE_OK; +} + +/* +** Free the list of sorted records starting at pRecord. +*/ +static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ + SorterRecord *p; + SorterRecord *pNext; + for(p=pRecord; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p->pVal); + sqlite3DbFree(db, p); + } } /* @@ -351,6 +428,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter->pTemp1 ){ sqlite3OsCloseFree(pSorter->pTemp1); } + vdbeSorterRecordFree(db, pSorter->pRecord); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -370,10 +448,128 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ); } +/* +** Attemp to merge the two sorted lists p1 and p2 into a single list. If no +** error occurs set *ppOut to the head of the new list and return SQLITE_OK. +*/ +static int vdbeSorterMerge( + sqlite3 *db, /* Database handle */ + KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + SorterRecord *p1, /* First list to merge */ + SorterRecord *p2, /* Second list to merge */ + SorterRecord **ppOut /* OUT: Head of merged list */ +){ + int rc = SQLITE_OK; + SorterRecord *pFinal = 0; + SorterRecord **pp = &pFinal; + + while( p1 || p2 ){ + if( p1==0 ){ + *pp = p2; + p2 = 0; + }else if( p2==0 ){ + *pp = p1; + p1 = 0; + }else{ + int res; + rc = vdbeSorterCompare( + pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + ); + if( rc!=SQLITE_OK ){ + vdbeSorterRecordFree(db, p1); + vdbeSorterRecordFree(db, p2); + vdbeSorterRecordFree(db, pFinal); + pFinal = 0; + break; + } + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + } + *pp = 0; + } + } + + *ppOut = pFinal; + return rc; +} /* -** Write the current contents of the b-tree to a PMA. Return SQLITE_OK -** if successful, or an SQLite error code otherwise. +** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK +** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error +** occurs. +*/ +static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ + int rc = SQLITE_OK; + int i; + SorterRecord **aSlot; + SorterRecord *p; + VdbeSorter *pSorter = pCsr->pSorter; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); + if( !aSlot ){ + return SQLITE_NOMEM; + } + + p = pSorter->pRecord; + while( p ){ + SorterRecord *pNext = p->pNext; + p->pNext = 0; + for(i=0; rc==SQLITE_OK && aSlot[i]; i++){ + rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + aSlot[i] = 0; + } + if( rc!=SQLITE_OK ){ + vdbeSorterRecordFree(db, pNext); + break; + } + aSlot[i] = p; + p = pNext; + } + + p = 0; + for(i=0; i<64; i++){ + if( rc==SQLITE_OK ){ + rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + }else{ + vdbeSorterRecordFree(db, aSlot[i]); + } + } + pSorter->pRecord = p; + +#if 0 + { + SorterRecord *pTmp1 = 0; + SorterRecord *pTmp2; + for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ + if( pTmp1 ){ + int res; + rc = vdbeSorterCompare(pKeyInfo, + 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res + ); + assert( rc!=SQLITE_OK || res<0 ); + } + pTmp1 = pTmp2; + } + } +#endif + + if( rc!=SQLITE_OK ){ + } + sqlite3_free(aSlot); + return rc; +} + + +/* +** Write the current contents of the in-memory linked-list to a PMA. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -384,19 +580,19 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ int rc = SQLITE_OK; /* Return code */ VdbeSorter *pSorter = pCsr->pSorter; - int res = 0; - /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held - ** in memory and so an I/O error is not possible. */ - rc = sqlite3BtreeFirst(pCsr->pCursor, &res); - if( NEVER(rc!=SQLITE_OK) || res ) return rc; - assert( pSorter->nBtree>0 ); + if( pSorter->nInMemory==0 ){ + assert( pSorter->pRecord==0 ); + return rc; + } + + rc = vdbeSorterSort(db, pCsr); /* If the first temporary PMA file has not been opened, open it now. */ - if( pSorter->pTemp1==0 ){ + if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); assert( rc!=SQLITE_OK || pSorter->pTemp1 ); assert( pSorter->iWriteOff==0 ); @@ -404,129 +600,89 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ } if( rc==SQLITE_OK ){ - i64 iWriteOff = pSorter->iWriteOff; - void *aMalloc = 0; /* Array used to hold a single record */ - int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */ + i64 iOff = pSorter->iWriteOff; + SorterRecord *p; + SorterRecord *pNext = 0; pSorter->nPMA++; - for( - rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff); - rc==SQLITE_OK && res==0; - rc = sqlite3BtreeNext(pCsr->pCursor, &res) - ){ - i64 nKey; /* Size of this key in bytes */ + rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff); + for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){ + pNext = p->pNext; + rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff); - /* Write the size of the record in bytes to the output file */ - (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey); - rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff); - - /* Make sure the aMalloc[] buffer is large enough for the record */ - if( rc==SQLITE_OK && nKey>nMalloc ){ - aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey); - if( !aMalloc ){ - rc = SQLITE_NOMEM; - }else{ - nMalloc = nKey; - } - } - - /* Write the record itself to the output file */ if( rc==SQLITE_OK ){ - /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */ - rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); - if( ALWAYS(rc==SQLITE_OK) ){ - rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); - iWriteOff += nKey; - } + rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff); + iOff += p->nVal; } - if( rc!=SQLITE_OK ) break; + sqlite3DbFree(db, p->pVal); + sqlite3DbFree(db, p); } /* This assert verifies that unless an error has occurred, the size of ** the PMA on disk is the same as the expected size stored in - ** pSorter->nBtree. */ - assert( rc!=SQLITE_OK || pSorter->nBtree==( - iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree) + ** pSorter->nInMemory. */ + assert( rc!=SQLITE_OK || pSorter->nInMemory==( + iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory) )); - pSorter->iWriteOff = iWriteOff; - sqlite3DbFree(db, aMalloc); + pSorter->iWriteOff = iOff; + pSorter->pRecord = p; } - pSorter->nBtree = 0; return rc; } /* -** This function is called on a sorter cursor by the VDBE before each row -** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in -** bytes, about to be inserted. -** -** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr -** is large enough, its contents are written to a sorted PMA on disk and the -** tree emptied. This prevents the b-tree (which must be small enough to -** fit entirely in the cache in order to support efficient inserts) from -** growing too large. -** -** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +** Add a record to the sorter. */ -int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ - int rc = SQLITE_OK; /* Return code */ +int sqlite3VdbeSorterWrite( + sqlite3 *db, /* Database handle */ + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal /* Memory cell containing record */ +){ VdbeSorter *pSorter = pCsr->pSorter; - if( pSorter ){ - Pager *pPager = sqlite3BtreePager(pCsr->pBt); - int nPage; /* Current size of temporary file in pages */ + int rc; + SorterRecord *pNew; - /* Sorters never spill to disk */ - assert( sqlite3PagerFile(pPager)->pMethods==0 ); + assert( pSorter ); + pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - /* Determine how many pages the temporary b-tree has grown to */ - sqlite3PagerPagecount(pPager, &nPage); - - /* If pSorter->nWorking is still zero, but the temporary file has been - ** created in the file-system, then the most recent insert into the - ** current b-tree segment probably caused the cache to overflow (it is - ** also possible that sqlite3_release_memory() was called). So set the - ** size of the working set to a little less than the current size of the - ** file in pages. */ - if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){ - pSorter->nWorking = nPage-5; - if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING; - } + pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3VdbeMemMakeWriteable(pVal); + if( rc==SQLITE_OK ){ + pNew->pVal = pVal->z; + pNew->nVal = pVal->n; + pVal->zMalloc = 0; + sqlite3VdbeMemSetNull(pVal); + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + }else{ + sqlite3DbFree(db, pNew); + rc = SQLITE_NOMEM; } - - /* If the number of pages used by the current b-tree segment is greater - ** than the size of the working set (VdbeSorter.nWorking), start a new - ** segment b-tree. */ - if( pSorter->nWorking && nPage>=pSorter->nWorking ){ - BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */ - int iRoot; /* Root page of new tree */ - - /* Copy the current contents of the b-tree into a PMA in sorted order. - ** Close the currently open b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(p); - - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0); -#ifdef SQLITE_DEBUG - sqlite3PagerPagecount(pPager, &nPage); - assert( rc!=SQLITE_OK || nPage==1 ); -#endif - } - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY); - } - if( rc==SQLITE_OK ){ - assert( iRoot==2 ); - rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p); - } - } - - pSorter->nBtree += sqlite3VarintLen(nKey) + nKey; } + + /* See if the contents of the sorter should now be written out. They + ** are written out when either of the following are true: + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * cache-size), or + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. + */ + if( rc==SQLITE_OK && ( + (pSorter->nInMemory>pSorter->nLimit2) + || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull()) + )){ + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + } + return rc; } @@ -577,8 +733,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ assert( pSorter ); /* Write the current b-tree to a PMA. Close the b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(pCsr->pCursor); + rc = vdbeSorterListToPMA(db, pCsr); if( rc!=SQLITE_OK ) return rc; if( pSorter->nPMA==0 ){ *pbEof = 1; @@ -692,14 +847,7 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ VdbeSorterIter *pIter; pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - - /* Coverage testing note: As things are currently, this call will always - ** succeed. This is because the memory cell passed by the VDBE layer - ** happens to be the same one as was used to assemble the keys before they - ** were passed to the sorter - meaning it is always large enough for the - ** largest key. But this could change very easily, so we leave the call - ** to sqlite3VdbeMemGrow() in. */ - if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){ + if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){ return SQLITE_NOMEM; } pOut->n = pIter->nKey; @@ -709,4 +857,30 @@ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ return SQLITE_OK; } +/* +** Compare the key in memory cell pVal with the key that the sorter cursor +** passed as the first argument currently points to. For the purposes of +** the comparison, ignore the rowid field at the end of each record. +** +** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). +** Otherwise, set *pRes to a negative, zero or positive value if the +** key in pVal is smaller than, equal to or larger than the current sorter +** key. +*/ +int sqlite3VdbeSorterCompare( + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal, /* Value to compare to current sorter key */ + int *pRes /* OUT: Result of comparison */ +){ + int rc; + VdbeSorter *pSorter = pCsr->pSorter; + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, + pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes + ); + assert( rc!=SQLITE_OK || *pRes<=0 ); + return rc; +} + #endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ diff --git a/test/index4.test b/test/index4.test index 6400e34e4e..018ed744a1 100644 --- a/test/index4.test +++ b/test/index4.test @@ -108,5 +108,19 @@ do_execsql_test 1.8 { PRAGMA integrity_check } {ok} +do_execsql_test 2.1 { + BEGIN; + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(14); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(15); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(16); + COMMIT; +} +do_catchsql_test 2.2 { + CREATE UNIQUE INDEX i3 ON t2(x); +} {1 {indexed columns are not unique}} + finish_test From 9fed558df9dd95fe9442ba40927b8a86b12f998e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 11:45:31 +0000 Subject: [PATCH 24/36] If all data being sorted fits in memory, avoid writing any data out to temporary files in vdbesort.c. FossilOrigin-Name: 71075673c625f243969c3f34c73f28f378924007 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/vdbesort.c | 77 ++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 9513f2ff2e..7f37801bfe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Instead\sof\sa\stemporary\sb-tree,\suse\sa\slinked-list\sand\smerge-sort\sto\ssort\srecords\sin\smain\smemory\sin\svdbesort.c. -D 2011-09-02T10:31:11.173 +C If\sall\sdata\sbeing\ssorted\sfits\sin\smemory,\savoid\swriting\sany\sdata\sout\sto\stemporary\sfiles\sin\svdbesort.c. +D 2011-09-02T11:45:31.600 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 9c2e8ca23c2413a5162a4433cb72377588d896f8 +F src/vdbesort.c 1e9d4437c7520e09b18f37d2c520bfa01e638655 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P ebf819aaa555bd79fddfc0a6f9827a2539095d6c -R c9d892df81c49366e067933871c3b776 +P 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 +R 023e08b29250dcfb353aa799966d9eeb U dan -Z b3581bd03fb5a181801f57c568eb418b +Z 052620c5c09aadfa96f446950f8f8dff diff --git a/manifest.uuid b/manifest.uuid index f82002ddab..6963963883 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7769fb988d9be0f2d8129aaac19620ac88f9b4a6 \ No newline at end of file +71075673c625f243969c3f34c73f28f378924007 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 3ab5d7930a..83c1c5d132 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -732,13 +732,18 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ assert( pSorter ); + /* If no data has been written to disk, then do not do so now. Instead, + ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly + ** from the in-memory list. */ + if( pSorter->nPMA==0 ){ + *pbEof = !pSorter->pRecord; + assert( pSorter->aTree==0 ); + return vdbeSorterSort(db, pCsr); + } + /* Write the current b-tree to a PMA. Close the b-tree cursor. */ rc = vdbeSorterListToPMA(db, pCsr); if( rc!=SQLITE_OK ) return rc; - if( pSorter->nPMA==0 ){ - *pbEof = 1; - return SQLITE_OK; - } /* Allocate space for aIter[] and aTree[]. */ nIter = pSorter->nPMA; @@ -826,33 +831,64 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ */ int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; - int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ int rc; /* Return code */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( pSorter->aTree ){ + int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + }else{ + SorterRecord *pFree = pSorter->pRecord; + pSorter->pRecord = pFree->pNext; + pFree->pNext = 0; + vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->pRecord; + rc = SQLITE_OK; + } return rc; } +/* +** Return a pointer to a buffer owned by the sorter that contains the +** current key. +*/ +static void *vdbeSorterRowkey( + VdbeSorter *pSorter, /* Sorter object */ + int *pnKey /* OUT: Size of current key in bytes */ +){ + void *pKey; + if( pSorter->aTree ){ + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + *pnKey = pIter->nKey; + pKey = pIter->aKey; + }else{ + *pnKey = pSorter->pRecord->nVal; + pKey = pSorter->pRecord->pVal; + } + return pKey; +} + /* ** Copy the current sorter key into the memory cell pOut. */ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ VdbeSorter *pSorter = pCsr->pSorter; - VdbeSorterIter *pIter; + void *pKey; int nKey; /* Sorter key to copy into pOut */ - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - if( sqlite3VdbeMemGrow(pOut, pIter->nKey, 0) ){ + pKey = vdbeSorterRowkey(pSorter, &nKey); + if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ return SQLITE_NOMEM; } - pOut->n = pIter->nKey; + pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); - memcpy(pOut->z, pIter->aKey, pIter->nKey); + memcpy(pOut->z, pKey, nKey); return SQLITE_OK; } @@ -874,11 +910,10 @@ int sqlite3VdbeSorterCompare( ){ int rc; VdbeSorter *pSorter = pCsr->pSorter; - VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, - pVal->z, pVal->n, pIter->aKey, pIter->nKey, pRes - ); + void *pKey; int nKey; /* Sorter key to compare pVal with */ + + pKey = vdbeSorterRowkey(pSorter, &nKey); + rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, pVal->z, pVal->n, pKey, nKey, pRes); assert( rc!=SQLITE_OK || *pRes<=0 ); return rc; } From 9747a6063b60740aa18ce3c2c82a6a876004d322 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 2 Sep 2011 15:08:28 +0000 Subject: [PATCH 25/36] Remove unused local variable. FossilOrigin-Name: 61bda876af6df3170263d41d2933168305de58d2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cf95f75fde..20cd63560c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\suninitialized\svariables\safter\sfailures\sin\sthe\smerge\ssort\scode. -D 2011-08-31T23:57:22.695 +C Remove\sunused\slocal\svariable. +D 2011-09-02T15:08:28.662 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 10e0c4dcdbec8d4189890fdf3e71b32efae194e3 -F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 +F src/os_win.c 33b7b7b48939af5cef2305f5ded19d45c025e2c7 F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 70b5b309568ac55565558d5456aca1e431cfd26b -R 09e04eb9163c46e529a61f2438cff2a1 -U drh -Z bdb2832352809bf647849d42b7aa6060 +P 2869ed28299b1c9f355ecc24635830f7f1249126 +R d7407cbd16401c8c8a7c8d014c2baaa2 +U mistachkin +Z 3bc82a48805c63310ead923d005627b4 diff --git a/manifest.uuid b/manifest.uuid index 67e9d42efe..0f07cdea41 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2869ed28299b1c9f355ecc24635830f7f1249126 \ No newline at end of file +61bda876af6df3170263d41d2933168305de58d2 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2441398945..c16198bd5d 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1604,7 +1604,6 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_SIZE_HINT: { if( pFile->szChunk>0 ){ - winFile *pFile = (winFile*)id; sqlite3_int64 oldSz; int rc = winFileSize(id, &oldSz); if( rc==SQLITE_OK ){ From 7f72114547ec18b5458e332faca908f81916f27c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 15:41:33 +0000 Subject: [PATCH 26/36] Reduce the number of malloc() calls made when creating an index on more than 2 columns. FossilOrigin-Name: 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 41 ++++++++++++++++++++++++++++------------- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 7f37801bfe..3c9140975f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sall\sdata\sbeing\ssorted\sfits\sin\smemory,\savoid\swriting\sany\sdata\sout\sto\stemporary\sfiles\sin\svdbesort.c. -D 2011-09-02T11:45:31.600 +C Reduce\sthe\snumber\sof\smalloc()\scalls\smade\swhen\screating\san\sindex\son\smore\sthan\s2\scolumns. +D 2011-09-02T15:41:33.781 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 1e9d4437c7520e09b18f37d2c520bfa01e638655 +F src/vdbesort.c a49d44498cc19af9bdbdc8e9fa882dd26cc17066 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 7769fb988d9be0f2d8129aaac19620ac88f9b4a6 -R 023e08b29250dcfb353aa799966d9eeb +P 71075673c625f243969c3f34c73f28f378924007 +R 16433222f8a97f40a1aa6bc60c6c3db0 U dan -Z 052620c5c09aadfa96f446950f8f8dff +Z 8c008bdf93f87673bd214d386e7f6cf5 diff --git a/manifest.uuid b/manifest.uuid index 6963963883..c870d49dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71075673c625f243969c3f34c73f28f378924007 \ No newline at end of file +065b0c9858da0ebb41722f3c56bdaf62f28b2f2c \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 83c1c5d132..63c6f543b8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -104,6 +104,8 @@ struct VdbeSorter { SorterRecord *pRecord; /* Head of in-memory record list */ int nLimit1; /* Minimum PMA size, in bytes */ int nLimit2; /* Maximum PMA size, in bytes */ + char *aSpace; /* Space for UnpackRecord() */ + int nSpace; /* Size of aSpace in bytes */ }; /* @@ -303,18 +305,31 @@ static int vdbeSorterIterInit( ** be less than key2. Even if key2 also contains NULL values. */ static int vdbeSorterCompare( - KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ int bOmitRowid, /* Ignore rowid field at end of keys */ void *pKey1, int nKey1, /* Left side of comparison */ void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ ){ - char aSpace[150]; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + VdbeSorter *pSorter = pCsr->pSorter; + char *aSpace = pSorter->aSpace; + int nSpace = pSorter->nSpace; UnpackedRecord *r2; int i; - r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, sizeof(aSpace)); - if( r2==0 ) return SQLITE_NOMEM; + if( aSpace==0 ){ + nSpace = ROUND8(sizeof(UnpackedRecord))+(pKeyInfo->nField+1)*sizeof(Mem); + aSpace = (char *)sqlite3Malloc(nSpace); + if( aSpace==0 ) return SQLITE_NOMEM; + pSorter->aSpace = aSpace; + pSorter->nSpace = nSpace; + } + + /* This call cannot fail. As the memory is already allocated. */ + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); + assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + if( bOmitRowid ){ for(i=0; inField-1; i++){ if( r2->aMem[i].flags & MEM_Null ){ @@ -329,7 +344,7 @@ static int vdbeSorterCompare( } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); - sqlite3VdbeDeleteUnpackedRecord(r2); + /* sqlite3VdbeDeleteUnpackedRecord(r2); */ return SQLITE_OK; } @@ -366,7 +381,7 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else{ int res; int rc = vdbeSorterCompare( - pCsr->pKeyInfo, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); if( rc!=SQLITE_OK ) return rc; if( res<=0 ){ @@ -429,6 +444,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ sqlite3OsCloseFree(pSorter->pTemp1); } vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3_free(pSorter->aSpace); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -454,7 +470,7 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ */ static int vdbeSorterMerge( sqlite3 *db, /* Database handle */ - KeyInfo *pKeyInfo, /* Collation functions to use in comparison */ + VdbeCursor *pCsr, /* For pKeyInfo */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -473,7 +489,7 @@ static int vdbeSorterMerge( }else{ int res; rc = vdbeSorterCompare( - pKeyInfo, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + pCsr, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res ); if( rc!=SQLITE_OK ){ vdbeSorterRecordFree(db, p1); @@ -510,7 +526,6 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ SorterRecord **aSlot; SorterRecord *p; VdbeSorter *pSorter = pCsr->pSorter; - KeyInfo *pKeyInfo = pCsr->pKeyInfo; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ @@ -522,7 +537,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ SorterRecord *pNext = p->pNext; p->pNext = 0; for(i=0; rc==SQLITE_OK && aSlot[i]; i++){ - rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); aSlot[i] = 0; } if( rc!=SQLITE_OK ){ @@ -536,7 +551,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ if( rc==SQLITE_OK ){ - rc = vdbeSorterMerge(db, pKeyInfo, p, aSlot[i], &p); + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); }else{ vdbeSorterRecordFree(db, aSlot[i]); } @@ -550,7 +565,7 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ if( pTmp1 ){ int res; - rc = vdbeSorterCompare(pKeyInfo, + rc = vdbeSorterCompare(pCsr, 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res ); assert( rc!=SQLITE_OK || res<0 ); @@ -913,7 +928,7 @@ int sqlite3VdbeSorterCompare( void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - rc = vdbeSorterCompare(pCsr->pKeyInfo, 1, pVal->z, pVal->n, pKey, nKey, pRes); + rc = vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); assert( rc!=SQLITE_OK || *pRes<=0 ); return rc; } From 7733a4db4bce40774142c4fe86b13e6721703e12 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 Sep 2011 18:03:16 +0000 Subject: [PATCH 27/36] Combine two malloc calls in vdbesort.c. FossilOrigin-Name: cf48ad8353e28339d00f448bb729e10a7f2aad72 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 +------- src/vdbesort.c | 27 +++++++++------------------ 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 3c9140975f..b71aeffabe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\smalloc()\scalls\smade\swhen\screating\san\sindex\son\smore\sthan\s2\scolumns. -D 2011-09-02T15:41:33.781 +C Combine\stwo\smalloc\scalls\sin\svdbesort.c. +D 2011-09-02T18:03:16.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bc2099e7d3c22c52b2c54349b9c07c04f2a810d0 +F src/btree.c dc3f244447fd5aade08f66d475e2a6316bbef4e7 F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c a49d44498cc19af9bdbdc8e9fa882dd26cc17066 +F src/vdbesort.c b56437b8b3a0a17a2b110d37633dc9217c72fd60 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 71075673c625f243969c3f34c73f28f378924007 -R 16433222f8a97f40a1aa6bc60c6c3db0 +P 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c +R 06bb7ef7ee2f3a8f904497ea31ee4edb U dan -Z 8c008bdf93f87673bd214d386e7f6cf5 +Z ad89820409358feeee63c15f85c9298f diff --git a/manifest.uuid b/manifest.uuid index c870d49dd3..4ac5f7ba98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -065b0c9858da0ebb41722f3c56bdaf62f28b2f2c \ No newline at end of file +cf48ad8353e28339d00f448bb729e10a7f2aad72 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b335579bd0..f20afb6bf3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7288,13 +7288,7 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ BtShared *pBt = p->pBt; int rc; sqlite3BtreeEnter(p); - if( (pBt->openFlags&BTREE_SINGLE) ){ - pBt->nPage = 0; - sqlite3PagerTruncateImage(pBt->pPager, 1); - rc = newDatabase(pBt); - }else{ - rc = btreeDropTable(p, iTable, piMoved); - } + rc = btreeDropTable(p, iTable, piMoved); sqlite3BtreeLeave(p); return rc; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 63c6f543b8..74399edf5d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -93,7 +93,7 @@ typedef struct SorterRecord SorterRecord; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { - int nInMemory; /* Current size of b-tree contents as PMA */ + int nInMemory; /* Current size of pRecord list as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ @@ -422,7 +422,6 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *pNext; for(p=pRecord; p; p=pNext){ pNext = p->pNext; - sqlite3DbFree(db, p->pVal); sqlite3DbFree(db, p); } } @@ -630,7 +629,6 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ iOff += p->nVal; } - sqlite3DbFree(db, p->pVal); sqlite3DbFree(db, p); } @@ -657,28 +655,21 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; - int rc; - SorterRecord *pNew; + int rc = SQLITE_OK; /* Return Code */ + SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocZero(db, sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3VdbeMemMakeWriteable(pVal); - if( rc==SQLITE_OK ){ - pNew->pVal = pVal->z; - pNew->nVal = pVal->n; - pVal->zMalloc = 0; - sqlite3VdbeMemSetNull(pVal); - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - }else{ - sqlite3DbFree(db, pNew); - rc = SQLITE_NOMEM; - } + pNew->pVal = (void *)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; } /* See if the contents of the sorter should now be written out. They From 34163c68310d010907847a68cf0a9e46dba31aef Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 Sep 2011 21:42:33 +0000 Subject: [PATCH 28/36] Remove some dead code. Fix a faulty assert(). Improve some variable names. FossilOrigin-Name: a9a64592cf88580cb254fb0aac65a2f2085976ec --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/btree.c | 1 - src/pager.c | 27 --------------------------- src/pager.h | 3 --- src/vdbesort.c | 41 +++++++++++++---------------------------- 6 files changed, 24 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index b71aeffabe..1f98484757 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Combine\stwo\smalloc\scalls\sin\svdbesort.c. -D 2011-09-02T18:03:16.975 +C Remove\ssome\sdead\scode.\s\sFix\sa\sfaulty\sassert().\s\sImprove\ssome\svariable\snames. +D 2011-09-02T21:42:33.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c dc3f244447fd5aade08f66d475e2a6316bbef4e7 +F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c @@ -167,8 +167,8 @@ F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c 10e0c4dcdbec8d4189890fdf3e71b32efae194e3 F src/os_win.c 45de13c6c3501cfd6469b2b34149b823060e39f4 -F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d -F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 +F src/pager.c 5545863e4e246e1744cfb6993821c6e4b63ffb64 +F src/pager.h 6bea8d1949db33768de1c5b4133b267b40845f8b F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c b56437b8b3a0a17a2b110d37633dc9217c72fd60 +F src/vdbesort.c 2ae618bec4bec5faa075e82078008763666c655d F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 065b0c9858da0ebb41722f3c56bdaf62f28b2f2c -R 06bb7ef7ee2f3a8f904497ea31ee4edb -U dan -Z ad89820409358feeee63c15f85c9298f +P cf48ad8353e28339d00f448bb729e10a7f2aad72 +R d2091eb5dd8da1e99612b5b3cd229b4c +U drh +Z e655a0d761c2ed1b9d5806522d72983b diff --git a/manifest.uuid b/manifest.uuid index 4ac5f7ba98..2cb8cd7e4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf48ad8353e28339d00f448bb729e10a7f2aad72 \ No newline at end of file +a9a64592cf88580cb254fb0aac65a2f2085976ec \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f20afb6bf3..f0eb376dec 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7285,7 +7285,6 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ - BtShared *pBt = p->pBt; int rc; sqlite3BtreeEnter(p); rc = btreeDropTable(p, iTable, piMoved); diff --git a/src/pager.c b/src/pager.c index 373d06aeca..f40d2af74b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -621,7 +621,6 @@ struct Pager { u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ u8 hasSeenStress; /* pagerStress() called one or more times */ - u8 isSorter; /* True for a PAGER_SORTER */ /************************************************************************** ** The following block contains those class members that change during @@ -845,15 +844,6 @@ static int assert_pager_state(Pager *p){ assert( pagerUseWal(p)==0 ); } - /* A sorter is a temp file that never spills to disk and always has - ** the doNotSpill flag set - */ - if( p->isSorter ){ - assert( p->tempFile ); - assert( p->doNotSpill ); - assert( p->fd->pMethods==0 ); - } - /* If changeCountDone is set, a RESERVED lock or greater must be held ** on the file. */ @@ -4557,12 +4547,6 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ -#ifndef SQLITE_OMIT_MERGE_SORT - if( flags & PAGER_SORTER ){ - pPager->doNotSpill = 1; - pPager->isSorter = 1; - } -#endif *ppPager = pPager; return SQLITE_OK; @@ -6107,17 +6091,6 @@ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } -#ifndef SQLITE_OMIT_MERGE_SORT -/* -** Return true if the pager has seen a pagerStress callback. -*/ -int sqlite3PagerUnderStress(Pager *pPager){ - assert( pPager->isSorter ); - assert( pPager->doNotSpill ); - return pPager->hasSeenStress; -} -#endif - /* ** Check that there are at least nSavepoint savepoints open. If there are ** currently less than nSavepoints open, then open one or more savepoints diff --git a/src/pager.h b/src/pager.h index ccd7467d6a..2a02eff75c 100644 --- a/src/pager.h +++ b/src/pager.h @@ -156,9 +156,6 @@ const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -#ifndef SQLITE_OMIT_MERGE_SORT -int sqlite3PagerUnderStress(Pager*); -#endif /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/vdbesort.c b/src/vdbesort.c index 74399edf5d..c1ea8cadd3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -102,8 +102,8 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ int nPMA; /* Number of PMAs stored in pTemp1 */ SorterRecord *pRecord; /* Head of in-memory record list */ - int nLimit1; /* Minimum PMA size, in bytes */ - int nLimit2; /* Maximum PMA size, in bytes */ + int mnPmaSize; /* Minimum PMA size, in bytes */ + int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ char *aSpace; /* Space for UnpackRecord() */ int nSpace; /* Size of aSpace in bytes */ }; @@ -400,16 +400,20 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ + int mxCache; /* Cache size */ + VdbeSorter *pSorter; /* The new sorter */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - if( pCsr->pSorter==0 ){ + pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + if( pSorter==0 ){ return SQLITE_NOMEM; } pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pCsr->pSorter->nLimit1 = 10 * pgsz; - pCsr->pSorter->nLimit2 = db->aDb[0].pSchema->cache_size * pgsz; + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; return SQLITE_OK; } @@ -557,25 +561,6 @@ static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ } pSorter->pRecord = p; -#if 0 - { - SorterRecord *pTmp1 = 0; - SorterRecord *pTmp2; - for(pTmp2=pSorter->pRecord; pTmp2 && rc==SQLITE_OK; pTmp2=pTmp2->pNext){ - if( pTmp1 ){ - int res; - rc = vdbeSorterCompare(pCsr, - 0, pTmp1->pVal, pTmp1->nVal, pTmp2->pVal, pTmp2->nVal, &res - ); - assert( rc!=SQLITE_OK || res<0 ); - } - pTmp1 = pTmp2; - } - } -#endif - - if( rc!=SQLITE_OK ){ - } sqlite3_free(aSlot); return rc; } @@ -682,8 +667,8 @@ int sqlite3VdbeSorterWrite( ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ if( rc==SQLITE_OK && ( - (pSorter->nInMemory>pSorter->nLimit2) - || (pSorter->nInMemory>pSorter->nLimit1 && sqlite3HeapNearlyFull()) + (pSorter->nInMemory>pSorter->mxPmaSize) + || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ rc = vdbeSorterListToPMA(db, pCsr); pSorter->nInMemory = 0; @@ -920,7 +905,7 @@ int sqlite3VdbeSorterCompare( pKey = vdbeSorterRowkey(pSorter, &nKey); rc = vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); - assert( rc!=SQLITE_OK || *pRes<=0 ); + assert( rc!=SQLITE_OK || pVal->db->mallocFailed || (*pRes)<=0 ); return rc; } From ca892a725284f1ce7d9f4f5e3c4f75f4f01f055b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Sep 2011 00:17:51 +0000 Subject: [PATCH 29/36] The build works again with -DSQLITE_OMIT_MERGE_SORT. The merge-sorter now avoids spilling to disk (letting the in-memory linked list grow without bound) if PRAGMA temp_store=3. FossilOrigin-Name: 68e26c4487696d194ee85370380e4b0e56d206ee --- manifest | 20 ++++++++-------- manifest.uuid | 2 +- src/build.c | 61 +++++++++++++++++++++++-------------------------- src/sqliteInt.h | 8 ------- src/vdbe.c | 27 ++++++++++++++++++---- src/vdbeInt.h | 2 +- src/vdbesort.c | 14 +++++++----- 7 files changed, 70 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 1f98484757..6c9743bfa6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sdead\scode.\s\sFix\sa\sfaulty\sassert().\s\sImprove\ssome\svariable\snames. -D 2011-09-02T21:42:33.425 +C The\sbuild\sworks\sagain\swith\s-DSQLITE_OMIT_MERGE_SORT.\s\sThe\smerge-sorter\snow\navoids\sspilling\sto\sdisk\s(letting\sthe\sin-memory\slinked\slist\sgrow\swithout\nbound)\sif\sPRAGMA\stemp_store=3. +D 2011-09-03T00:17:51.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c dc367138cb3625e6d42b389e05d7267aece5753c +F src/build.c 851e81f26a75abbb98bd99a7c5f10e8670d867bb F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c e3132ec65240b2e2f3d50831021eac387f27584d @@ -183,7 +183,7 @@ F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 723cda73a33c91f5a0a145f4c0ced45d94921079 +F src/sqliteInt.h c7e37ee49b1a922ddcd18fa98dd750efa4d2db14 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c da9c7efc48dc79d7785f3f17a1c3df514bf18489 +F src/vdbe.c d4c8224cc931c6082557501d7f822fb12f273922 F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 -F src/vdbeInt.h a255da14be8c2794ce38e0d2142877bb29df9105 +F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 2ae618bec4bec5faa075e82078008763666c655d +F src/vdbesort.c aa7ad2ef91ad7d33eb614d3fb1c4cbd068b321ad F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P cf48ad8353e28339d00f448bb729e10a7f2aad72 -R d2091eb5dd8da1e99612b5b3cd229b4c +P a9a64592cf88580cb254fb0aac65a2f2085976ec +R 4edd7e2341e47b4ce23f768c53615b8c U drh -Z e655a0d761c2ed1b9d5806522d72983b +Z cbdc1caefa14410ba02a8248a6138d65 diff --git a/manifest.uuid b/manifest.uuid index 2cb8cd7e4f..2bc60c00a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9a64592cf88580cb254fb0aac65a2f2085976ec \ No newline at end of file +68e26c4487696d194ee85370380e4b0e56d206ee \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0de1020b9b..27130ef281 100644 --- a/src/build.c +++ b/src/build.c @@ -2335,15 +2335,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); - /* Set bUseSorter to use OP_OpenSorter, or clear it to insert directly - ** into the index. The sorter is used unless either OMIT_MERGE_SORT is - ** defined or the system is configured to store temp files in-memory. */ -#ifdef SQLITE_OMIT_MERGE_SORT - static const int bUseSorter = 0; -#else - const int bUseSorter = !sqlite3TempInMemory(pParse->db); -#endif - #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, db->aDb[iDb].zName ) ){ @@ -2369,11 +2360,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, 1); } +#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ - if( bUseSorter ){ - iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - } + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); +#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2383,24 +2374,27 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); - sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); - if( pIndex->onError!=OE_None ){ - int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); - addr2 = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); - sqlite3HaltConstraint( - pParse, OE_Abort, "indexed columns are not unique", P4_STATIC - ); - }else{ - addr2 = sqlite3VdbeCurrentAddr(v); - } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); - }else if( pIndex->onError!=OE_None ){ +#ifndef SQLITE_OMIT_MERGE_SORT + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); + } + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#else + if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); @@ -2418,10 +2412,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3HaltConstraint( pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#endif sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, bUseSorter ? OP_SorterNext : OP_Next, iSorter, addr2); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 694f4694eb..f27fea2b3a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -372,14 +372,6 @@ # define SQLITE_TEMP_STORE 1 #endif -/* -** If all temporary storage is in-memory, then omit the external merge-sort -** logic since it is superfluous. -*/ -#if SQLITE_TEMP_STORE==3 && !defined(SQLITE_OMIT_MERGE_SORT) -# define SQLITE_OMIT_MERGE_SORT -#endif - /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. diff --git a/src/vdbe.c b/src/vdbe.c index d9310e0563..acf113515f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3215,15 +3215,19 @@ case OP_OpenEphemeral: { ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. */ -case OP_SorterOpen: -case OP_OpenSorter: { +case OP_SorterOpen: { VdbeCursor *pCx; +#ifndef SQLITE_OMIT_MERGE_SORT pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); +#else + pOp->opcode = OP_OpenEphemeral; + pc--; +#endif break; } @@ -4102,10 +4106,15 @@ case OP_SorterCompare: { */ case OP_SorterData: { VdbeCursor *pC; +#ifndef SQLITE_OMIT_MERGE_SORT pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); +#else + pOp->opcode = OP_RowKey; + pc--; +#endif break; } @@ -4148,8 +4157,7 @@ case OP_RowData: { assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); - assert( pC->isSorter==(pOp->opcode==OP_SorterData) ); - + assert( !pC->isSorter ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4307,6 +4315,9 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Sort; +#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4385,6 +4396,9 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Next; +#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4434,7 +4448,10 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ -case OP_SorterInsert: +case OP_SorterInsert: /* in2 */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_IdxInsert; +#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 68ad8dcbf7..7fbd5693fb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -65,7 +65,7 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - VdbeSorter *pSorter; /* Sorter object for OP_OpenSorter cursors */ + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c1ea8cadd3..e8135f0f14 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -409,11 +409,13 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ return SQLITE_NOMEM; } - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; - mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = mxCache * pgsz; + if( !sqlite3TempInMemory(db) ){ + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; + } return SQLITE_OK; } @@ -666,7 +668,7 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && ( + if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ From 8b1ea14f8ca6883fdea4bd64b7f2b8a97c8401ac Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2011 14:36:13 +0000 Subject: [PATCH 30/36] Reduce the number of VdbeRecordUnpack() calls made in vdbesort.c. FossilOrigin-Name: 666c2c3cff51dac2ba5689b75705d99c3705673b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 24 +++++++++++++++++------- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 6c9743bfa6..dd25184f0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sbuild\sworks\sagain\swith\s-DSQLITE_OMIT_MERGE_SORT.\s\sThe\smerge-sorter\snow\navoids\sspilling\sto\sdisk\s(letting\sthe\sin-memory\slinked\slist\sgrow\swithout\nbound)\sif\sPRAGMA\stemp_store=3. -D 2011-09-03T00:17:51.023 +C Reduce\sthe\snumber\sof\sVdbeRecordUnpack()\scalls\smade\sin\svdbesort.c. +D 2011-09-03T14:36:13.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c aa7ad2ef91ad7d33eb614d3fb1c4cbd068b321ad +F src/vdbesort.c 5e38e7e1f5f6900b1b8bea49decd23c69221aed9 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P a9a64592cf88580cb254fb0aac65a2f2085976ec -R 4edd7e2341e47b4ce23f768c53615b8c -U drh -Z cbdc1caefa14410ba02a8248a6138d65 +P 68e26c4487696d194ee85370380e4b0e56d206ee +R f6e37dce79fea0b74c498631cfb45ff1 +U dan +Z cc6b7d2a23c219696518afb4690c5277 diff --git a/manifest.uuid b/manifest.uuid index 2bc60c00a6..af74b9885a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68e26c4487696d194ee85370380e4b0e56d206ee \ No newline at end of file +666c2c3cff51dac2ba5689b75705d99c3705673b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e8135f0f14..9e169c3e36 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -303,6 +303,9 @@ static int vdbeSorterIterInit( ** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid ** is true and key1 contains even a single NULL value, it is considered to ** be less than key2. Even if key2 also contains NULL values. +** +** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace +** has been allocated and contains an unpacked record that is used as key2. */ static int vdbeSorterCompare( VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ @@ -326,15 +329,20 @@ static int vdbeSorterCompare( pSorter->nSpace = nSpace; } - /* This call cannot fail. As the memory is already allocated. */ - r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); - assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + if( pKey2 ){ + /* This call cannot fail. As the memory is already allocated. */ + r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); + assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); + assert( r2==aSpace ); + }else{ + r2 = (UnpackedRecord *)aSpace; + assert( !bOmitRowid ); + } if( bOmitRowid ){ for(i=0; inField-1; i++){ if( r2->aMem[i].flags & MEM_Null ){ *pRes = -1; - sqlite3VdbeDeleteUnpackedRecord(r2); return SQLITE_OK; } } @@ -344,7 +352,6 @@ static int vdbeSorterCompare( } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); - /* sqlite3VdbeDeleteUnpackedRecord(r2); */ return SQLITE_OK; } @@ -483,6 +490,7 @@ static int vdbeSorterMerge( int rc = SQLITE_OK; SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; + int bKey2InSpace = 0; /* True if pCsr->aSpace contains key2 */ while( p1 || p2 ){ if( p1==0 ){ @@ -493,8 +501,8 @@ static int vdbeSorterMerge( p1 = 0; }else{ int res; - rc = vdbeSorterCompare( - pCsr, 0, p1->pVal, p1->nVal, p2->pVal, p2->nVal, &res + rc = vdbeSorterCompare(pCsr, 0, + p1->pVal, p1->nVal, (bKey2InSpace ? 0 : p2->pVal), p2->nVal, &res ); if( rc!=SQLITE_OK ){ vdbeSorterRecordFree(db, p1); @@ -507,10 +515,12 @@ static int vdbeSorterMerge( *pp = p1; pp = &p1->pNext; p1 = p1->pNext; + bKey2InSpace = 1; }else{ *pp = p2; pp = &p2->pNext; p2 = p2->pNext; + bKey2InSpace = 0; } *pp = 0; } From 04a962fe02e030f10f3bb1b984bb7a49da647a3b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 Sep 2011 16:42:38 +0000 Subject: [PATCH 31/36] Simplification and performance tweaks in vdbeSorterMerge(). FossilOrigin-Name: 99e34bdce4ccca15b79159b03b96787e7a7ff85b --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/vdbesort.c | 56 ++++++++++++++++++++++---------------------------- 3 files changed, 32 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index dd25184f0e..aa9396df26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\sVdbeRecordUnpack()\scalls\smade\sin\svdbesort.c. -D 2011-09-03T14:36:13.912 +C Simplification\sand\sperformance\stweaks\sin\svdbeSorterMerge(). +D 2011-09-03T16:42:38.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 5e38e7e1f5f6900b1b8bea49decd23c69221aed9 +F src/vdbesort.c e6d6f0c2aa003f7cbdea8c9be47a15a8e854fb97 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 68e26c4487696d194ee85370380e4b0e56d206ee -R f6e37dce79fea0b74c498631cfb45ff1 -U dan -Z cc6b7d2a23c219696518afb4690c5277 +P 666c2c3cff51dac2ba5689b75705d99c3705673b +R cfceda33d1a4c239172824f11326438a +U drh +Z 0da609a19e6bdcd2176629a62a8129db diff --git a/manifest.uuid b/manifest.uuid index af74b9885a..f0c89b9c6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -666c2c3cff51dac2ba5689b75705d99c3705673b \ No newline at end of file +99e34bdce4ccca15b79159b03b96787e7a7ff85b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 9e169c3e36..6681aad52d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -490,44 +490,36 @@ static int vdbeSorterMerge( int rc = SQLITE_OK; SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - int bKey2InSpace = 0; /* True if pCsr->aSpace contains key2 */ + void *pVal2 = p2 ? p2->pVal : 0; - while( p1 || p2 ){ - if( p1==0 ){ - *pp = p2; - p2 = 0; - }else if( p2==0 ){ - *pp = p1; - p1 = 0; - }else{ - int res; - rc = vdbeSorterCompare(pCsr, 0, - p1->pVal, p1->nVal, (bKey2InSpace ? 0 : p2->pVal), p2->nVal, &res - ); - if( rc!=SQLITE_OK ){ - vdbeSorterRecordFree(db, p1); - vdbeSorterRecordFree(db, p2); - vdbeSorterRecordFree(db, pFinal); - pFinal = 0; - break; - } - if( res<=0 ){ - *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; - bKey2InSpace = 1; - }else{ - *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; - bKey2InSpace = 0; - } + while( p1 && p2 ){ + int res; + rc = vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + if( rc!=SQLITE_OK ){ *pp = 0; + vdbeSorterRecordFree(db, p1); + vdbeSorterRecordFree(db, p2); + vdbeSorterRecordFree(db, pFinal); + *ppOut = 0; + return rc; + } + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + pVal2 = 0; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + if( p2==0 ) break; + pVal2 = p2->pVal; } } + *pp = p1 ? p1 : p2; *ppOut = pFinal; - return rc; + return SQLITE_OK; } /* From c2bb3282f18d90e77e461e5dd203a32a7d6bbf48 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 4 Sep 2011 01:11:46 +0000 Subject: [PATCH 32/36] Improve the testability of the merge-sort logic. FossilOrigin-Name: b5179baf87aa00ed5cecbdcaa65bee10e112a9e9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- src/vdbesort.c | 24 +++++++++++++++--------- 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 0291313011..fff4ed9132 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovements\sto\sthe\sexternal\smerge-sorter.\s\sKeep\scontent\son\san\nin-memory\slinked\slists\srather\sthan\san\sephemeral\stable\sprior\sto\sspilling\sto\ndisk.\s\sUse\sthe\sexternal\smerge-sorter\sto\simplement\sORDER\sBY\sand\sGROUP\sBY\nin\saddition\sto\sCREATE\sINDEX. -D 2011-09-03T17:07:26.674 +C Improve\sthe\stestability\sof\sthe\smerge-sort\slogic. +D 2011-09-04T01:11:46.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c 32d0f4e5513362706b8973e7f1b87cd0885dfbf5 +F src/select.c bf7b7ea6befb483619da5f597b0864668b828c3c F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c e6d6f0c2aa003f7cbdea8c9be47a15a8e854fb97 +F src/vdbesort.c 4ceadaa20c2a428ea2a5066f6d7d91d7a8118077 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 61bda876af6df3170263d41d2933168305de58d2 99e34bdce4ccca15b79159b03b96787e7a7ff85b -R da1f20303c5b87a4ef556300a531567d +P 4c43e8b2d2c1d8dcba3cd1c3f2ec4e19ab419430 +R 983c9c689f2d53c2e02840decc5ec993 U drh -Z 01e7a916345d0e3901e7a472c5d2f441 +Z ae9009821df61dee832eb776e8a5bada diff --git a/manifest.uuid b/manifest.uuid index 2d5de7cdee..1858f041fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c43e8b2d2c1d8dcba3cd1c3f2ec4e19ab419430 \ No newline at end of file +b5179baf87aa00ed5cecbdcaa65bee10e112a9e9 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7d9d26602e..a8ea08a781 100644 --- a/src/select.c +++ b/src/select.c @@ -900,7 +900,7 @@ static void generateSortTail( regRowid = sqlite3GetTempReg(pParse); } if( p->selFlags & SF_UseSorter ){ - int regSortOut = sqlite3GetTempReg(pParse); + int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); diff --git a/src/vdbesort.c b/src/vdbesort.c index 6681aad52d..6ce85811a8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -239,15 +239,10 @@ static int vdbeSorterReadVarint( ){ u8 aVarint[9]; /* Buffer large enough for a varint */ i64 iOff = *piOffset; /* Offset in file to read from */ - int nRead = 9; /* Number of bytes to read from file */ int rc; /* Return code */ assert( iEof>iOff ); - if( (iEof-iOff)flags & UNPACKED_NEED_FREE)==0 ); - assert( r2==aSpace ); + assert( r2==(UnpackedRecord*)aSpace ); }else{ r2 = (UnpackedRecord *)aSpace; assert( !bOmitRowid ); @@ -387,10 +382,15 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ iRes = i1; }else{ int res; - int rc = vdbeSorterCompare( + int rc; + assert( pCsr->pSorter->aSpace!=0 ); /* allocated in vdbeSorterMerge() */ + rc = vdbeSorterCompare( pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); - if( rc!=SQLITE_OK ) return rc; + /* The vdbeSorterCompare() call cannot fail since pCsr->pSorter->aSpace + ** has already been allocated. */ + assert( rc==SQLITE_OK ); + if( res<=0 ){ iRes = i1; }else{ @@ -606,6 +606,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ i64 iOff = pSorter->iWriteOff; SorterRecord *p; SorterRecord *pNext = 0; + static const char eightZeros[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; pSorter->nPMA++; rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff); @@ -629,6 +630,11 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ )); pSorter->iWriteOff = iOff; + if( rc==SQLITE_OK ){ + /* Terminate each file with 8 extra bytes so that from any offset + ** in the file we can always read 9 bytes without a SHORT_READ error */ + rc = sqlite3OsWrite(pSorter->pTemp1, eightZeros, 8, iOff); + } pSorter->pRecord = p; } From 19c88c1a99371d3ee2148cc2fcabb3bafbf4f8cb Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 4 Sep 2011 01:27:00 +0000 Subject: [PATCH 33/36] Fix a compiler warning about an unused parameter in the merge-sort code. FossilOrigin-Name: 6b657ae75035eb10b0ad640998d3c9eadfdffa6e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 5 +---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fff4ed9132..cdcd0fb2ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\stestability\sof\sthe\smerge-sort\slogic. -D 2011-09-04T01:11:46.977 +C Fix\sa\scompiler\swarning\sabout\san\sunused\sparameter\sin\sthe\smerge-sort\scode. +D 2011-09-04T01:27:00.510 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -245,7 +245,7 @@ F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 4ceadaa20c2a428ea2a5066f6d7d91d7a8118077 +F src/vdbesort.c 14d127565bab1032a20558e6c9ae5f2ec0499111 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 4c43e8b2d2c1d8dcba3cd1c3f2ec4e19ab419430 -R 983c9c689f2d53c2e02840decc5ec993 +P b5179baf87aa00ed5cecbdcaa65bee10e112a9e9 +R 19f8e6b3aaa4f6faa3ce68485711b8fa U drh -Z ae9009821df61dee832eb776e8a5bada +Z c68d8b01f68570c73b0670afe2bb6a90 diff --git a/manifest.uuid b/manifest.uuid index 1858f041fe..80fcf402aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5179baf87aa00ed5cecbdcaa65bee10e112a9e9 \ No newline at end of file +6b657ae75035eb10b0ad640998d3c9eadfdffa6e \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6ce85811a8..2acba8778d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -233,7 +233,6 @@ static int vdbeSorterWriteVarint( */ static int vdbeSorterReadVarint( sqlite3_file *pFile, /* File to read from */ - i64 iEof, /* Total number of bytes in file */ i64 *piOffset, /* IN/OUT: Read offset in pFile */ i64 *piVal /* OUT: Value read from file */ ){ @@ -241,7 +240,6 @@ static int vdbeSorterReadVarint( i64 iOff = *piOffset; /* Offset in file to read from */ int rc; /* Return code */ - assert( iEof>iOff ); rc = sqlite3OsRead(pFile, aVarint, 9, iOff); if( rc==SQLITE_OK ){ *piOffset += getVarint(aVarint, (u64 *)piVal); @@ -274,9 +272,8 @@ static int vdbeSorterIterInit( if( !pIter->aAlloc ){ rc = SQLITE_NOMEM; }else{ - i64 iEof = pSorter->iWriteOff; /* EOF of file pSorter->pTemp1 */ i64 nByte; /* Total size of PMA in bytes */ - rc = vdbeSorterReadVarint(pSorter->pTemp1, iEof, &pIter->iReadOff, &nByte); + rc = vdbeSorterReadVarint(pSorter->pTemp1, &pIter->iReadOff, &nByte); *pnByte += nByte; pIter->iEof = pIter->iReadOff + nByte; } From 03e9cfc2231c3ac9a7b458e1a2a48df3e177afb8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Sep 2011 14:20:27 +0000 Subject: [PATCH 34/36] Refactor the sqlite3VdbeRecordUnpack() interface to better accommodate the vdbesort.c module. FossilOrigin-Name: f115b2303509c678dbe83b2fa3d9c40d82882813 --- manifest | 22 +++++----- manifest.uuid | 2 +- src/btree.c | 9 +++-- src/vdbe.c | 13 +++--- src/vdbe.h | 4 +- src/vdbeaux.c | 106 ++++++++++++++++++++++--------------------------- src/vdbesort.c | 33 ++++++--------- 7 files changed, 87 insertions(+), 102 deletions(-) diff --git a/manifest b/manifest index cdcd0fb2ff..6146f80f7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scompiler\swarning\sabout\san\sunused\sparameter\sin\sthe\smerge-sort\scode. -D 2011-09-04T01:27:00.510 +C Refactor\sthe\ssqlite3VdbeRecordUnpack()\sinterface\sto\sbetter\saccommodate\sthe\svdbesort.c\smodule. +D 2011-09-05T14:20:27.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4d46fe30b8bc920f68b7d58a5f45316fa5d023ec +F src/btree.c 958f26c8f5ff001791ba1ba3825e69d1a383c2fd F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 851e81f26a75abbb98bd99a7c5f10e8670d867bb @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c d4c8224cc931c6082557501d7f822fb12f273922 -F src/vdbe.h c1eeedacab6bcf1e7c2cf8203ba9763a616f9a86 +F src/vdbe.c 7008edbf8f608d82c035dcb1c56367ad85e68aaa +F src/vdbe.h a10b360bf69474babc8aba8fcc64b824c5e97d38 F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c e58acbc5ea3823922a0cd8fa21f94f39af51ee88 +F src/vdbeaux.c 8546b9481d8c642f2403ae8ebf20823b54e00606 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 14d127565bab1032a20558e6c9ae5f2ec0499111 +F src/vdbesort.c 4f3265707c3277011fbb0c09d3d8e6461152f639 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P b5179baf87aa00ed5cecbdcaa65bee10e112a9e9 -R 19f8e6b3aaa4f6faa3ce68485711b8fa -U drh -Z c68d8b01f68570c73b0670afe2bb6a90 +P 6b657ae75035eb10b0ad640998d3c9eadfdffa6e +R 17c6c62caa602109512a646943ac3f7f +U dan +Z 1812248ebcb21a3973c4b6e0222ea90b diff --git a/manifest.uuid b/manifest.uuid index 80fcf402aa..ecb95ed250 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b657ae75035eb10b0ad640998d3c9eadfdffa6e \ No newline at end of file +f115b2303509c678dbe83b2fa3d9c40d82882813 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f0eb376dec..014267cc84 100644 --- a/src/btree.c +++ b/src/btree.c @@ -656,18 +656,21 @@ static int btreeMoveto( int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ char aSpace[150]; /* Temp space for pIdxKey - to avoid a malloc */ + char *pFree = 0; if( pKey ){ assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, - aSpace, sizeof(aSpace)); + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree + ); if( pIdxKey==0 ) return SQLITE_NOMEM; + sqlite3VdbeRecordUnpack(pCur->pKeyInfo, nKey, pKey, pIdxKey); }else{ pIdxKey = 0; } rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); if( pKey ){ - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + sqlite3DbFree(pCur->pKeyInfo->db, pFree); } return rc; } diff --git a/src/vdbe.c b/src/vdbe.c index acf113515f..906058c1f8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3531,6 +3531,7 @@ case OP_Found: { /* jump, in3 */ int alreadyExists; VdbeCursor *pC; int res; + char *pFree; UnpackedRecord *pIdxKey; UnpackedRecord r; char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; @@ -3558,18 +3559,18 @@ case OP_Found: { /* jump, in3 */ r.flags = UNPACKED_PREFIX_MATCH; pIdxKey = &r; }else{ + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree + ); + if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ - pIdxKey = sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, - aTempRec, sizeof(aTempRec)); - if( pIdxKey==0 ){ - goto no_mem; - } + sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); pIdxKey->flags |= UNPACKED_PREFIX_MATCH; } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); if( pOp->p4.i==0 ){ - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + sqlite3DbFree(db, pFree); } if( rc!=SQLITE_OK ){ break; diff --git a/src/vdbe.h b/src/vdbe.h index 9728548517..f4691c0e8f 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -212,9 +212,9 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif -UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); -void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); +void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f622e8d09a..06ed06dfcb 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2830,54 +2830,69 @@ u32 sqlite3VdbeSerialGet( return 0; } - /* -** Given the nKey-byte encoding of a record in pKey[], parse the -** record into a UnpackedRecord structure. Return a pointer to -** that structure. +** This routine is used to allocate sufficient space for an UnpackedRecord +** structure large enough to be used with sqlite3VdbeRecordUnpack() if +** the first argument is a pointer to KeyInfo structure pKeyInfo. ** -** The calling function might provide szSpace bytes of memory -** space at pSpace. This space can be used to hold the returned -** VDbeParsedRecord structure if it is large enough. If it is -** not big enough, space is obtained from sqlite3_malloc(). +** The space is either allocated using sqlite3DbMallocRaw() or from within +** the unaligned buffer passed via the second and third arguments (presumably +** stack space). If the former, then *ppFree is set to a pointer that should +** be eventually freed by the caller using sqlite3DbFree(). Or, if the +** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL +** before returning. ** -** The returned structure should be closed by a call to -** sqlite3VdbeDeleteUnpackedRecord(). -*/ -UnpackedRecord *sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ - int nKey, /* Size of the binary record */ - const void *pKey, /* The binary record */ - char *pSpace, /* Unaligned space available to hold the object */ - int szSpace /* Size of pSpace[] in bytes */ +** If an OOM error occurs, NULL is returned. +*/ +UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( + KeyInfo *pKeyInfo, /* Description of the record */ + char *pSpace, /* Unaligned space available */ + int szSpace, /* Size of pSpace[] in bytes */ + char **ppFree /* OUT: Caller should free this pointer */ ){ - const unsigned char *aKey = (const unsigned char *)pKey; - UnpackedRecord *p; /* The unpacked record that we will return */ - int nByte; /* Memory space needed to hold p, in bytes */ - int d; - u32 idx; - u16 u; /* Unsigned loop counter */ - u32 szHdr; - Mem *pMem; - int nOff; /* Increase pSpace by this much to 8-byte align it */ - - /* - ** We want to shift the pointer pSpace up such that it is 8-byte aligned. + UnpackedRecord *p; /* Unpacked record to return */ + int nOff; /* Increment pSpace by nOff to align it */ + int nByte; /* Number of bytes required for *p */ + + /* We want to shift the pointer pSpace up such that it is 8-byte aligned. ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift ** it by. If pSpace is already 8-byte aligned, nOff should be zero. */ nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; pSpace += nOff; szSpace -= nOff; + nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); if( nByte>szSpace ){ - p = sqlite3DbMallocRaw(pKeyInfo->db, nByte); - if( p==0 ) return 0; - p->flags = UNPACKED_NEED_FREE | UNPACKED_NEED_DESTROY; + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + *ppFree = (char *)p; }else{ p = (UnpackedRecord*)pSpace; - p->flags = UNPACKED_NEED_DESTROY; + *ppFree = 0; } + + return p; +} + +/* +** Given the nKey-byte encoding of a record in pKey[], populate the +** UnpackedRecord structure indicated by the fourth argument with the +** contents of the decoded record. +*/ +void sqlite3VdbeRecordUnpack( + KeyInfo *pKeyInfo, /* Information about the record format */ + int nKey, /* Size of the binary record */ + const void *pKey, /* The binary record */ + UnpackedRecord *p /* Populate this structure before returning. */ +){ + const unsigned char *aKey = (const unsigned char *)pKey; + int d; + u32 idx; /* Offset in aKey[] to read from */ + u16 u; /* Unsigned loop counter */ + u32 szHdr; + Mem *pMem; + + p->flags = 0; p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nField + 1; p->aMem = pMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; @@ -2899,31 +2914,6 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( } assert( u<=pKeyInfo->nField + 1 ); p->nField = u; - return (void*)p; -} - -/* -** This routine destroys a UnpackedRecord object. -*/ -void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ -#ifdef SQLITE_DEBUG - int i; - Mem *pMem; - - assert( p!=0 ); - assert( p->flags & UNPACKED_NEED_DESTROY ); - for(i=0, pMem=p->aMem; inField; i++, pMem++){ - /* The unpacked record is always constructed by the - ** sqlite3VdbeUnpackRecord() function above, which makes all - ** strings and blobs static. And none of the elements are - ** ever transformed, so there is never anything to delete. - */ - if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); - } -#endif - if( p->flags & UNPACKED_NEED_FREE ){ - sqlite3DbFree(p->pKeyInfo->db, p); - } } /* diff --git a/src/vdbesort.c b/src/vdbesort.c index 2acba8778d..79f17dcbc9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -104,8 +104,7 @@ struct VdbeSorter { SorterRecord *pRecord; /* Head of in-memory record list */ int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ - char *aSpace; /* Space for UnpackRecord() */ - int nSpace; /* Size of aSpace in bytes */ + UnpackedRecord *pUnpacked; /* Used to unpack keys */ }; /* @@ -308,27 +307,19 @@ static int vdbeSorterCompare( ){ KeyInfo *pKeyInfo = pCsr->pKeyInfo; VdbeSorter *pSorter = pCsr->pSorter; - char *aSpace = pSorter->aSpace; - int nSpace = pSorter->nSpace; - UnpackedRecord *r2; + UnpackedRecord *r2 = pSorter->pUnpacked; int i; - if( aSpace==0 ){ - nSpace = ROUND8(sizeof(UnpackedRecord))+(pKeyInfo->nField+1)*sizeof(Mem); - aSpace = (char *)sqlite3Malloc(nSpace); - if( aSpace==0 ) return SQLITE_NOMEM; - pSorter->aSpace = aSpace; - pSorter->nSpace = nSpace; + if( r2==0 ){ + char *pFree; + r2 = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &pFree); + if( r2==0 ) return SQLITE_NOMEM; + assert( pFree==(char *)r2 ); + pSorter->pUnpacked = r2; } if( pKey2 ){ - /* This call cannot fail. As the memory is already allocated. */ - r2 = sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, aSpace, nSpace); - assert( r2 && (r2->flags & UNPACKED_NEED_FREE)==0 ); - assert( r2==(UnpackedRecord*)aSpace ); - }else{ - r2 = (UnpackedRecord *)aSpace; - assert( !bOmitRowid ); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); } if( bOmitRowid ){ @@ -380,11 +371,11 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else{ int res; int rc; - assert( pCsr->pSorter->aSpace!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ rc = vdbeSorterCompare( pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); - /* The vdbeSorterCompare() call cannot fail since pCsr->pSorter->aSpace + /* The vdbeSorterCompare() call cannot fail since pCsr->pSorter->pUnpacked ** has already been allocated. */ assert( rc==SQLITE_OK ); @@ -453,7 +444,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ sqlite3OsCloseFree(pSorter->pTemp1); } vdbeSorterRecordFree(db, pSorter->pRecord); - sqlite3_free(pSorter->aSpace); + sqlite3DbFree(db, pSorter->pUnpacked); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } From 42acb3e597111de07a61f7c8215d0bd663b7693f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Sep 2011 20:16:38 +0000 Subject: [PATCH 35/36] Fix a minor performance regression in btreeMoveto(). FossilOrigin-Name: d0712dfb24867a807c9cddcc12bd0aebe1d3e085 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- src/vdbeaux.c | 18 ++++++++---------- src/vdbesort.c | 19 ++++++++----------- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 6146f80f7a..8a2d5d5008 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3VdbeRecordUnpack()\sinterface\sto\sbetter\saccommodate\sthe\svdbesort.c\smodule. -D 2011-09-05T14:20:27.320 +C Fix\sa\sminor\sperformance\sregression\sin\sbtreeMoveto(). +D 2011-09-05T20:16:38.385 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,7 +124,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 958f26c8f5ff001791ba1ba3825e69d1a383c2fd +F src/btree.c 4cb13cf019d1978b62494b970d20e9555211094b F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 F src/build.c 851e81f26a75abbb98bd99a7c5f10e8670d867bb @@ -242,10 +242,10 @@ F src/vdbe.c 7008edbf8f608d82c035dcb1c56367ad85e68aaa F src/vdbe.h a10b360bf69474babc8aba8fcc64b824c5e97d38 F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c 8546b9481d8c642f2403ae8ebf20823b54e00606 +F src/vdbeaux.c 079d37d4afac8359b2bbbf4af9876be04cc857b8 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 5e6effb96dd53d233361cbfaa3f0a43b9af689e9 -F src/vdbesort.c 4f3265707c3277011fbb0c09d3d8e6461152f639 +F src/vdbesort.c 667bc65254a9ebaf7bc466ebb7e58884933e351f F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P 6b657ae75035eb10b0ad640998d3c9eadfdffa6e -R 17c6c62caa602109512a646943ac3f7f +P f115b2303509c678dbe83b2fa3d9c40d82882813 +R 8196c804bc92b1e392918585a079f317 U dan -Z 1812248ebcb21a3973c4b6e0222ea90b +Z 34f3f7e2b2b4331a7d1652f52697a006 diff --git a/manifest.uuid b/manifest.uuid index ecb95ed250..ee13ef1682 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f115b2303509c678dbe83b2fa3d9c40d82882813 \ No newline at end of file +d0712dfb24867a807c9cddcc12bd0aebe1d3e085 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 014267cc84..130673cdcf 100644 --- a/src/btree.c +++ b/src/btree.c @@ -669,7 +669,7 @@ static int btreeMoveto( pIdxKey = 0; } rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); - if( pKey ){ + if( pFree ){ sqlite3DbFree(pCur->pKeyInfo->db, pFree); } return rc; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 06ed06dfcb..02fadabd87 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2859,18 +2859,19 @@ UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( ** it by. If pSpace is already 8-byte aligned, nOff should be zero. */ nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; - pSpace += nOff; - szSpace -= nOff; - nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); - if( nByte>szSpace ){ + if( nByte>szSpace+nOff ){ p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); *ppFree = (char *)p; + if( !p ) return 0; }else{ - p = (UnpackedRecord*)pSpace; + p = (UnpackedRecord*)&pSpace[nOff]; *ppFree = 0; } - + + p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + p->pKeyInfo = pKeyInfo; + p->nField = pKeyInfo->nField + 1; return p; } @@ -2890,12 +2891,9 @@ void sqlite3VdbeRecordUnpack( u32 idx; /* Offset in aKey[] to read from */ u16 u; /* Unsigned loop counter */ u32 szHdr; - Mem *pMem; + Mem *pMem = p->aMem; p->flags = 0; - p->pKeyInfo = pKeyInfo; - p->nField = pKeyInfo->nField + 1; - p->aMem = pMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); idx = getVarint32(aKey, szHdr); d = szHdr; diff --git a/src/vdbesort.c b/src/vdbesort.c index 79f17dcbc9..ead2f453e4 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -310,28 +310,20 @@ static int vdbeSorterCompare( UnpackedRecord *r2 = pSorter->pUnpacked; int i; - if( r2==0 ){ - char *pFree; - r2 = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &pFree); - if( r2==0 ) return SQLITE_NOMEM; - assert( pFree==(char *)r2 ); - pSorter->pUnpacked = r2; - } - if( pKey2 ){ sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); } if( bOmitRowid ){ - for(i=0; inField-1; i++){ + r2->nField = pKeyInfo->nField; + assert( r2->nField>0 ); + for(i=0; inField; i++){ if( r2->aMem[i].flags & MEM_Null ){ *pRes = -1; return SQLITE_OK; } } r2->flags |= UNPACKED_PREFIX_MATCH; - r2->nField--; - assert( r2->nField>0 ); } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); @@ -397,12 +389,17 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ + char *d; /* Dummy */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); if( pSorter==0 ){ return SQLITE_NOMEM; } + + pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); + if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; + assert( pSorter->pUnpacked==(UnpackedRecord *)d ); if( !sqlite3TempInMemory(db) ){ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); From 649591af239c5c296b767a59d8f48f079a2da76d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 11 Sep 2011 10:14:37 +0000 Subject: [PATCH 36/36] Cleanup pdb/ilk files generated by the MSVC makefile. FossilOrigin-Name: a9db247b752bcda0131b8f01c6f0182f3101d154 --- Makefile.msc | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index e139a745a1..a6c5d5a75b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -938,7 +938,7 @@ sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) clean: - del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib + del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* del /Q mkkeywordhash.exe keywordhash.h diff --git a/manifest b/manifest index 8a2d5d5008..f5503e1304 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\sminor\sperformance\sregression\sin\sbtreeMoveto(). -D 2011-09-05T20:16:38.385 +C Cleanup\spdb/ilk\sfiles\sgenerated\sby\sthe\sMSVC\smakefile. +D 2011-09-11T10:14:37.609 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc c39cc7a7a9fe8f6304f37b88f3bca9aae5b83f30 +F Makefile.msc 25da409ce0c7799e57f48a729a8e153b23027adc F Makefile.vxworks 1deb39c8bb047296c30161ffa10c1b5423e632f9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 @@ -961,7 +961,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2 -P f115b2303509c678dbe83b2fa3d9c40d82882813 -R 8196c804bc92b1e392918585a079f317 -U dan -Z 34f3f7e2b2b4331a7d1652f52697a006 +P d0712dfb24867a807c9cddcc12bd0aebe1d3e085 +R 2d6c3f1bb7ceef0c7739aec15017d5b3 +U mistachkin +Z a47fc06d3051baa0866b54de99b831cd diff --git a/manifest.uuid b/manifest.uuid index ee13ef1682..74ab610c51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0712dfb24867a807c9cddcc12bd0aebe1d3e085 \ No newline at end of file +a9db247b752bcda0131b8f01c6f0182f3101d154 \ No newline at end of file