From 0342b1f5424eea41a5c49e4aff13f495eaf0bac4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Sep 2005 03:07:44 +0000 Subject: [PATCH] Sorting is now done using a sorting index rather than loading the entire result set into memory and doing a merge sort. The old merge sort technique was a carry-over from SQLite version 1. The new method uses a bounded amount of memory and scales to much larger result sets. There are still errors: some 39 regression tests fail. (CVS 2653) FossilOrigin-Name: 09db0a24241f9248584250d1728117b8a3159626 --- manifest | 24 ++--- manifest.uuid | 2 +- src/expr.c | 7 +- src/select.c | 257 ++++++++++++++++++++++----------------------- src/sqliteInt.h | 22 +++- src/vdbe.c | 162 ++++------------------------ src/vdbeInt.h | 20 ---- src/vdbeaux.c | 16 --- test/conflict.test | 20 ++-- 9 files changed, 192 insertions(+), 338 deletions(-) diff --git a/manifest b/manifest index 79225b7f13..711d29e14b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C {quote:\sKeyInfo}\sgeneration\smoved\sto\sa\scommon\ssubroutine.\s(CVS\s2652) -D 2005-08-31T18:20:00 +C Sorting\sis\snow\sdone\susing\sa\ssorting\sindex\srather\sthan\sloading\sthe\sentire\nresult\sset\sinto\smemory\sand\sdoing\sa\smerge\ssort.\s\sThe\sold\smerge\ssort\stechnique\nwas\sa\scarry-over\sfrom\sSQLite\sversion\s1.\s\sThe\snew\smethod\suses\sa\sbounded\samount\nof\smemory\sand\sscales\sto\smuch\slarger\sresult\ssets.\s\sThere\sare\sstill\serrors:\nsome\s39\sregression\stests\sfail.\s(CVS\s2653) +D 2005-09-01T03:07:44 F Makefile.in 12784cdce5ffc8dfb707300c34e4f1eb3b8a14f1 F Makefile.linux-gcc 06be33b2a9ad4f005a5f42b22c4a19dab3cbb5c7 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -40,7 +40,7 @@ F src/complete.c 4de937dfdd4c79a501772ab2035b26082f337a79 F src/date.c 7444b0900a28da77e57e3337a636873cff0ae940 F src/delete.c be1fc25c9e109cd8cbab42a43ee696263da7c04b F src/experimental.c 50c1e3b34f752f4ac10c36f287db095c2b61766d -F src/expr.c aef4a3901a5bea6625b9613be9d9ddaccd575bc4 +F src/expr.c e9d0401bed1fa61cce356c2da4b53dae769c4fc3 F src/func.c 9da04a6241309a612cf610715944c6a2aaf0f297 F src/hash.c 2b1b13f7400e179631c83a1be0c664608c8f021f F src/hash.h 1b0c445e1c89ff2aaad9b4605ba61375af001e84 @@ -63,10 +63,10 @@ F src/pragma.c 69413fbdc0c6aaa493a776ea52c1b3e6cf35dfb2 F src/prepare.c 86f0d8e744b8d956eff6bc40e29049efee017610 F src/printf.c d2678b06cfa07be9b14c330a42310f62340e34ce F src/random.c 90adff4e73a3b249eb4f1fc2a6ff9cf78c7233a4 -F src/select.c cf566f995358f728288f0be481f12d20305117c0 +F src/select.c a0b10feee29d4e86731c6e381e0ff0ecf9d5eac8 F src/shell.c b21daba017b8feef2fdc65ecde57f70209494217 F src/sqlite.h.in d6561d51025d08de4f455607f3f9f9aa76e855d5 -F src/sqliteInt.h 207b63f9782d7faf1f19e694e8052e60841fb377 +F src/sqliteInt.h 845ff6f8019f80baafb1bdbb8ef80fcd04d9d0f9 F src/table.c 25b3ff2b39b7d87e8d4a5da0713d68dfc06cbee9 F src/tclsqlite.c ac94682f9e601dd373912c46414a5a842db2089a F src/test1.c 97425910c6adf87b5dc867ae7704f0430441663c @@ -80,11 +80,11 @@ F src/update.c a9d2c5f504212d62da1b094476f1389c0e02f83f F src/utf.c bda5eb85039ef16f2d17004c1e18c96e1ab0a80c F src/util.c 5650f6fe5ee30e0678985ad7b94da91e3f85752b F src/vacuum.c 829d9e1a6d7c094b80e0899686670932eafd768c -F src/vdbe.c 69f33e22c7d0a64b23fbb69e6da95a1bb6869032 +F src/vdbe.c 70e2dd078b0dfa15b70c0b9d31e7127da7408f15 F src/vdbe.h 3b29a9af6c7a64ed692bef1fc5f61338f40d2f67 -F src/vdbeInt.h e5f2855b0f0b120d870e0459816061b88b603774 +F src/vdbeInt.h c9dcaec8b411384da8b01c362cc856b6560b04f1 F src/vdbeapi.c f0d36ff0f06bb5315efac5645b62e99db2c175b8 -F src/vdbeaux.c 192e0dbeaaa0bfa652b0c2579c19894e5e5626fc +F src/vdbeaux.c 68d5d0881ded9867db1521fa2c0ae5ac8007a9d5 F src/vdbefifo.c 9efb94c8c3f4c979ebd0028219483f88e57584f5 F src/vdbemem.c 4732fd4d1a75dc38549493d7f9a81d02bf7c59b5 F src/where.c bbb973cbbd862b6b872faac39716a3fe13adfb44 @@ -129,7 +129,7 @@ F test/collate3.test 51362bdfb43a72bd2b087d90b2623b0695538e7a F test/collate4.test daf498e294dcd596b961d425c3f2dda117e4717e F test/collate5.test 5a49cd169e7565e4f92b42695667d6d5db25670d F test/collate6.test 6c9470d1606ee3e564675b229653e320c49ec638 -F test/conflict.test 3e7beba8c253095330c6853c00aaf6356e84cc68 +F test/conflict.test 774e10709f5e9a9a7352dc929b856929ec805e58 F test/corrupt.test 18c7a995b1af76a8c8600b996257f2c7b7bff083 F test/corrupt2.test 88342570828f2b8cbbd8369eff3891f5c0bdd5ba F test/crash.test f38b980a0508655d08c957a6dd27d66bca776504 @@ -306,7 +306,7 @@ F www/tclsqlite.tcl 3df553505b6efcad08f91e9b975deb2e6c9bb955 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl a99cf5f6d8bd4d5537584a2b342f0fb9fa601d8b F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513 -P b1dceef0508ffe20ab2ff8fa5e5b5a44f4f224aa -R 22b6fa7b5f0b031707b250894ea2c1df +P a25801df06e218e70570a6b9eae71603d590fe3a +R 7dae09495a78580995d7ecb2f7ce6601 U drh -Z 24532867563e79fa05da569ea322e240 +Z 29fbd68d397ac0605a1223b60872d370 diff --git a/manifest.uuid b/manifest.uuid index acc282824b..2524895de5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a25801df06e218e70570a6b9eae71603d590fe3a \ No newline at end of file +09db0a24241f9248584250d1728117b8a3159626 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3d3655d7ee..134de2446a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -12,7 +12,7 @@ ** This file contains routines used for analyzing expressions and ** for generating VDBE code that evaluates expressions in SQLite. ** -** $Id: expr.c,v 1.222 2005/08/30 00:54:02 drh Exp $ +** $Id: expr.c,v 1.223 2005/09/01 03:07:44 drh Exp $ */ #include "sqliteInt.h" #include @@ -546,9 +546,12 @@ Select *sqlite3SelectDup(Select *p){ pNew->pOffset = sqlite3ExprDup(p->pOffset); pNew->iLimit = -1; pNew->iOffset = -1; - pNew->ppOpenVirtual = 0; pNew->isResolved = p->isResolved; pNew->isAgg = p->isAgg; + pNew->pRightmost = 0; + pNew->addrOpenVirt[0] = -1; + pNew->addrOpenVirt[1] = -1; + pNew->addrOpenVirt[2] = -1; return pNew; } #else diff --git a/src/select.c b/src/select.c index 631f82d3ea..3da05e1087 100644 --- a/src/select.c +++ b/src/select.c @@ -12,7 +12,7 @@ ** This file contains C code routines that are called by the parser ** to handle SELECT statements in SQLite. ** -** $Id: select.c,v 1.257 2005/08/31 18:20:00 drh Exp $ +** $Id: select.c,v 1.258 2005/09/01 03:07:44 drh Exp $ */ #include "sqliteInt.h" @@ -60,6 +60,9 @@ Select *sqlite3SelectNew( pNew->pOffset = pOffset; pNew->iLimit = -1; pNew->iOffset = -1; + pNew->addrOpenVirt[0] = -1; + pNew->addrOpenVirt[1] = -1; + pNew->addrOpenVirt[2] = -1; } return pNew; } @@ -327,8 +330,9 @@ void sqlite3SelectDelete(Select *p){ */ static void pushOntoSorter(Parse *pParse, Vdbe *v, ExprList *pOrderBy){ sqlite3ExprCodeExprList(pParse, pOrderBy); - sqlite3VdbeAddOp(v, OP_MakeRecord, pOrderBy->nExpr, 0); - sqlite3VdbeAddOp(v, OP_SortInsert, 0, 0); + sqlite3VdbeAddOp(v, OP_Pull, pOrderBy->nExpr, 0); + sqlite3VdbeAddOp(v, OP_MakeRecord, pOrderBy->nExpr + 1, 0); + sqlite3VdbeAddOp(v, OP_IdxInsert, pOrderBy->iTab, 0); } /* @@ -555,6 +559,12 @@ static int selectInnerLoop( ** Given an expression list, generate a KeyInfo structure that records ** the collating sequence for each expression in that expression list. ** +** If the ExprList is an ORDER BY or GROUP BY clause then the resulting +** KeyInfo structure is appropriate for initializing a virtual index to +** implement that clause. If the ExprList is the result set of a SELECT +** then the KeyInfo structure is appropriate for initializing a virtual +** index to implement a DISTINCT test. +** ** Space to hold the KeyInfo structure is obtain from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. Add the KeyInfo structure to the P3 field of an opcode using @@ -601,17 +611,17 @@ static void generateSortTail( int eDest, /* Write the sorted results here */ int iParm /* Optional parameter associated with eDest */ ){ - int end1 = sqlite3VdbeMakeLabel(v); - int end2 = sqlite3VdbeMakeLabel(v); + int brk = sqlite3VdbeMakeLabel(v); + int cont = sqlite3VdbeMakeLabel(v); int addr; - KeyInfo *pInfo; + int iTab; + ExprList *pOrderBy = p->pOrderBy; if( eDest==SRT_Sorter ) return; - pInfo = keyInfoFromExprList(pParse, p->pOrderBy); - if( pInfo==0 ) return; - sqlite3VdbeOp3(v, OP_Sort, 0, 0, (char*)pInfo, P3_KEYINFO_HANDOFF); - addr = sqlite3VdbeAddOp(v, OP_SortNext, 0, end1); - codeLimiter(v, p, addr, end2, 1); + iTab = pOrderBy->iTab; + addr = 1 + sqlite3VdbeAddOp(v, OP_Sort, iTab, brk); + codeLimiter(v, p, cont, brk, 0); + sqlite3VdbeAddOp(v, OP_Column, iTab, pOrderBy->nExpr); switch( eDest ){ case SRT_Table: case SRT_TempTable: { @@ -634,7 +644,7 @@ static void generateSortTail( case SRT_Mem: { assert( nColumn==1 ); sqlite3VdbeAddOp(v, OP_MemStore, iParm, 1); - sqlite3VdbeAddOp(v, OP_Goto, 0, end1); + sqlite3VdbeAddOp(v, OP_Goto, 0, brk); break; } #endif @@ -659,11 +669,9 @@ static void generateSortTail( break; } } - sqlite3VdbeAddOp(v, OP_Goto, 0, addr); - sqlite3VdbeResolveLabel(v, end2); - sqlite3VdbeAddOp(v, OP_Pop, 1, 0); - sqlite3VdbeResolveLabel(v, end1); - sqlite3VdbeAddOp(v, OP_SortReset, 0, 0); + sqlite3VdbeResolveLabel(v, cont); + sqlite3VdbeAddOp(v, OP_Next, iTab, addr); + sqlite3VdbeResolveLabel(v, brk); } /* @@ -1327,48 +1335,20 @@ static void computeLimitRegisters(Parse *pParse, Select *p){ } /* -** Generate VDBE instructions that will open a transient table that -** will be used for an index or to store keyed results for a compound -** select. In other words, open a transient table that needs a -** KeyInfo structure. The number of columns in the KeyInfo is determined -** by the result set of the SELECT statement in the second argument. -** -** Specifically, this routine is called to open an index table for -** DISTINCT, UNION, INTERSECT and EXCEPT select statements (but not -** UNION ALL). -** -** The value returned is the address of the OP_OpenVirtual instruction. +** Allocate a virtual index to use for sorting. */ -static int openVirtualIndex(Parse *pParse, Select *p, int iTab){ - KeyInfo *pKeyInfo; - Vdbe *v = pParse->pVdbe; - int addr; - - if( prepSelectStmt(pParse, p) ){ - return 0; +static createSortingIndex(Parse *pParse, Select *p, ExprList *pOrderBy){ + if( pOrderBy ){ + int addr; + assert( pOrderBy->iTab==0 ); + pOrderBy->iTab = pParse->nTab++; + addr = sqlite3VdbeAddOp(pParse->pVdbe, OP_OpenVirtual, + pOrderBy->iTab, pOrderBy->nExpr+1); + assert( p->addrOpenVirt[2] == -1 ); + p->addrOpenVirt[2] = addr; } - pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - if( pKeyInfo==0 ) return 0; - addr = sqlite3VdbeOp3(v, OP_OpenVirtual, iTab, 0, - (char*)pKeyInfo, P3_KEYINFO_HANDOFF); - return addr; } -#ifndef SQLITE_OMIT_COMPOUND_SELECT -/* -** Add the address "addr" to the set of all OpenVirtual opcode addresses -** that are being accumulated in p->ppOpenVirtual. -*/ -static int multiSelectOpenVirtualAddr(Select *p, int addr){ - IdList *pList = *p->ppOpenVirtual = sqlite3IdListAppend(*p->ppOpenVirtual, 0); - if( pList==0 ){ - return SQLITE_NOMEM; - } - pList->a[pList->nId-1].idx = addr; - return SQLITE_OK; -} -#endif /* SQLITE_OMIT_COMPOUND_SELECT */ - #ifndef SQLITE_OMIT_COMPOUND_SELECT /* ** Return the appropriate collating sequence for the iCol-th column of @@ -1433,10 +1413,10 @@ static int multiSelect( int rc = SQLITE_OK; /* Success code from a subroutine */ Select *pPrior; /* Another SELECT immediately to our left */ Vdbe *v; /* Generate code to this VDBE */ - IdList *pOpenVirtual = 0;/* OP_OpenVirtual opcodes that need a KeyInfo */ - int aAddr[5]; /* Addresses of SetNumColumns operators */ - int nAddr = 0; /* Number used */ int nCol; /* Number of columns in the result set */ + ExprList *pOrderBy; /* The ORDER BY clause on p */ + int aSetP2[2]; /* Set P2 value of these op to number of columns */ + int nSetP2 = 0; /* Number of slots in aSetP2[] used */ /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. @@ -1446,6 +1426,8 @@ static int multiSelect( goto multi_select_end; } pPrior = p->pPrior; + assert( pPrior->pRightmost!=pPrior ); + assert( pPrior->pRightmost==p->pRightmost ); if( pPrior->pOrderBy ){ sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", selectOpName(p->op)); @@ -1467,32 +1449,21 @@ static int multiSelect( goto multi_select_end; } - /* If *p this is the right-most select statement, then initialize - ** p->ppOpenVirtual to point to pOpenVirtual. If *p is not the right most - ** statement then p->ppOpenVirtual will have already been initialized - ** by a prior call to this same procedure. Pass along the pOpenVirtual - ** pointer to pPrior, the next statement to our left. - */ - if( p->ppOpenVirtual==0 ){ - p->ppOpenVirtual = &pOpenVirtual; - } - pPrior->ppOpenVirtual = p->ppOpenVirtual; - /* Create the destination temporary table if necessary */ if( eDest==SRT_TempTable ){ assert( p->pEList ); - sqlite3VdbeAddOp(v, OP_OpenVirtual, iParm, 0); - assert( nAddr==0 ); - aAddr[nAddr++] = sqlite3VdbeAddOp(v, OP_SetNumColumns, iParm, 0); + assert( nSetP2pOrderBy; switch( p->op ){ case TK_ALL: { - if( p->pOrderBy==0 ){ + if( pOrderBy==0 ){ assert( !pPrior->pLimit ); pPrior->pLimit = p->pLimit; pPrior->pOffset = p->pOffset; @@ -1520,11 +1491,10 @@ static int multiSelect( int op = 0; /* One of the SRT_ operations to apply to self */ int priorOp; /* The SRT_ operation to apply to prior selects */ Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ - ExprList *pOrderBy; /* The ORDER BY clause for the right SELECT */ int addr; priorOp = p->op==TK_ALL ? SRT_Table : SRT_Union; - if( eDest==priorOp && p->pOrderBy==0 && !p->pLimit && !p->pOffset ){ + if( eDest==priorOp && pOrderBy==0 && !p->pLimit && !p->pOffset ){ /* We can reuse a temporary table generated by a SELECT to our ** right. */ @@ -1534,20 +1504,20 @@ static int multiSelect( ** intermediate results. */ unionTab = pParse->nTab++; - if( p->pOrderBy - && matchOrderbyToColumn(pParse, p, p->pOrderBy, unionTab, 1) ){ + if( pOrderBy && matchOrderbyToColumn(pParse, p, pOrderBy, unionTab,1) ){ rc = 1; goto multi_select_end; } addr = sqlite3VdbeAddOp(v, OP_OpenVirtual, unionTab, 0); - if( p->op!=TK_ALL ){ - rc = multiSelectOpenVirtualAddr(p, addr); - if( rc!=SQLITE_OK ){ - goto multi_select_end; - } + if( priorOp==SRT_Table ){ + assert( nSetP2addrOpenVirt[0] == -1 ); + p->addrOpenVirt[0] = addr; + p->pRightmost->usesVirt = 1; } - assert( nAddrpEList ); } @@ -1567,7 +1537,6 @@ static int multiSelect( case TK_ALL: op = SRT_Table; break; } p->pPrior = 0; - pOrderBy = p->pOrderBy; p->pOrderBy = 0; pLimit = p->pLimit; p->pLimit = 0; @@ -1601,7 +1570,7 @@ static int multiSelect( computeLimitRegisters(pParse, p); iStart = sqlite3VdbeCurrentAddr(v); rc = selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, - p->pOrderBy, -1, eDest, iParm, + pOrderBy, -1, eDest, iParm, iCont, iBreak, 0); if( rc ){ rc = 1; @@ -1626,18 +1595,16 @@ static int multiSelect( */ tab1 = pParse->nTab++; tab2 = pParse->nTab++; - if( p->pOrderBy && matchOrderbyToColumn(pParse,p,p->pOrderBy,tab1,1) ){ + if( pOrderBy && matchOrderbyToColumn(pParse,p,pOrderBy,tab1,1) ){ rc = 1; goto multi_select_end; } + createSortingIndex(pParse, p, pOrderBy); addr = sqlite3VdbeAddOp(v, OP_OpenVirtual, tab1, 0); - rc = multiSelectOpenVirtualAddr(p, addr); - if( rc!=SQLITE_OK ){ - goto multi_select_end; - } - assert( nAddraddrOpenVirt[0] == -1 ); + p->addrOpenVirt[0] = addr; + p->pRightmost->usesVirt = 1; assert( p->pEList ); /* Code the SELECTs to our left into temporary table "tab1". @@ -1650,12 +1617,8 @@ static int multiSelect( /* Code the current SELECT into temporary table "tab2" */ addr = sqlite3VdbeAddOp(v, OP_OpenVirtual, tab2, 0); - rc = multiSelectOpenVirtualAddr(p, addr); - if( rc!=SQLITE_OK ){ - goto multi_select_end; - } - assert( nAddraddrOpenVirt[1] == -1 ); + p->addrOpenVirt[1] = addr; p->pPrior = 0; pLimit = p->pLimit; p->pLimit = 0; @@ -1684,7 +1647,7 @@ static int multiSelect( iStart = sqlite3VdbeAddOp(v, OP_RowKey, tab1, 0); sqlite3VdbeAddOp(v, OP_NotFound, tab2, iCont); rc = selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, - p->pOrderBy, -1, eDest, iParm, + pOrderBy, -1, eDest, iParm, iCont, iBreak, 0); if( rc ){ rc = 1; @@ -1713,9 +1676,8 @@ static int multiSelect( /* Set the number of columns in temporary tables */ nCol = p->pEList->nExpr; - while( nAddr>0 ){ - nAddr--; - sqlite3VdbeChangeP2(v, aAddr[nAddr], nCol); + while( nSetP2 ){ + sqlite3VdbeChangeP2(v, aSetP2[--nSetP2], nCol); } /* Compute collating sequences used by either the ORDER BY clause or @@ -1728,12 +1690,15 @@ static int multiSelect( ** SELECT might also skip this part if it has no ORDER BY clause and ** no temp tables are required. */ - if( p->pOrderBy || (pOpenVirtual && pOpenVirtual->nId>0) ){ + if( pOrderBy || p->usesVirt ){ int i; /* Loop counter */ KeyInfo *pKeyInfo; /* Collating sequence for the result set */ + Select *pLoop; /* For looping through SELECT statements */ + CollSeq **apColl; + CollSeq **aCopy; - assert( p->ppOpenVirtual == &pOpenVirtual ); - pKeyInfo = sqliteMalloc(sizeof(*pKeyInfo)+nCol*sizeof(CollSeq*)); + assert( p->pRightmost==p ); + pKeyInfo = sqliteMalloc(sizeof(*pKeyInfo)+nCol*2*sizeof(CollSeq*)); if( !pKeyInfo ){ rc = SQLITE_NOMEM; goto multi_select_end; @@ -1742,46 +1707,57 @@ static int multiSelect( pKeyInfo->enc = pParse->db->enc; pKeyInfo->nField = nCol; - for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); - if( !pKeyInfo->aColl[i] ){ - pKeyInfo->aColl[i] = pParse->db->pDfltColl; + for(i=0, apColl=pKeyInfo->aColl; idb->pDfltColl; } } - for(i=0; pOpenVirtual && inId; i++){ - int p3type = (i==0?P3_KEYINFO_HANDOFF:P3_KEYINFO); - int addr = pOpenVirtual->a[i].idx; - sqlite3VdbeChangeP3(v, addr, (char *)pKeyInfo, p3type); + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + for(i=0; i<2; i++){ + int addr = pLoop->addrOpenVirt[i]; + if( addr<0 ){ + /* If [0] is unused then [1] is also unused. So we can + ** always safely abort as soon as the first unused slot is found */ + assert( pLoop->addrOpenVirt[1]<0 ); + break; + } + sqlite3VdbeChangeP2(v, addr, nCol); + sqlite3VdbeChangeP3(v, addr, (char*)pKeyInfo, P3_KEYINFO); + } } - if( p->pOrderBy ){ - struct ExprList_item *pOrderByTerm = p->pOrderBy->a; - for(i=0; ipOrderBy->nExpr; i++, pOrderByTerm++){ - Expr *pExpr = pOrderByTerm->pExpr; - char *zName = pOrderByTerm->zName; + if( pOrderBy ){ + struct ExprList_item *pOTerm = pOrderBy->a; + int nExpr = pOrderBy->nExpr; + int addr; + + aCopy = (CollSeq**)&pKeyInfo[1]; + memcpy(aCopy, pKeyInfo->aColl, nCol*sizeof(CollSeq*)); + apColl = pKeyInfo->aColl; + for(i=0; inExpr; i++, pOTerm++, apColl++){ + Expr *pExpr = pOTerm->pExpr; + char *zName = pOTerm->zName; assert( pExpr->op==TK_COLUMN && pExpr->iColumnpColl ); */ if( zName ){ - pExpr->pColl = sqlite3LocateCollSeq(pParse, zName, -1); + *apColl = sqlite3LocateCollSeq(pParse, zName, -1); }else{ - pExpr->pColl = pKeyInfo->aColl[pExpr->iColumn]; + *apColl = aCopy[pExpr->iColumn]; } } + assert( p->pRightmost==p ); + assert( p->addrOpenVirt[2]>=0 ); + addr = p->addrOpenVirt[2]; + sqlite3VdbeChangeP2(v, addr, p->pEList->nExpr+1); + sqlite3VdbeChangeP3(v, addr, (char*)pKeyInfo, P3_KEYINFO); generateSortTail(pParse, p, v, p->pEList->nExpr, eDest, iParm); } - if( !pOpenVirtual ){ - /* This happens for UNION ALL ... ORDER BY */ - sqliteFree(pKeyInfo); - } + sqliteFree(pKeyInfo); } multi_select_end: - if( pOpenVirtual ){ - sqlite3IdListDelete(pOpenVirtual); - } - p->ppOpenVirtual = 0; return rc; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ @@ -2517,6 +2493,12 @@ int sqlite3Select( /* If there is are a sequence of queries, do the earlier ones first. */ if( p->pPrior ){ + if( p->pRightmost==0 ){ + Select *pLoop; + for(pLoop=p; pLoop; pLoop=pLoop->pPrior){ + pLoop->pRightmost = p; + } + } return multiSelect(pParse, p, eDest, iParm, aff); } #endif @@ -2639,6 +2621,8 @@ int sqlite3Select( */ if( pOrderBy ){ struct ExprList_item *pTerm; + KeyInfo *pKeyInfo; + int addr; for(i=0, pTerm=pOrderBy->a; inExpr; i++, pTerm++){ if( pTerm->zName ){ pTerm->pExpr->pColl = sqlite3LocateCollSeq(pParse, pTerm->zName, -1); @@ -2647,6 +2631,11 @@ int sqlite3Select( if( pParse->nErr ){ goto select_end; } + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pOrderBy->iTab = pParse->nTab++; + addr = sqlite3VdbeOp3(v, OP_OpenVirtual, pOrderBy->iTab, pOrderBy->nExpr+1, + (char*)pKeyInfo, P3_KEYINFO_HANDOFF); + p->addrOpenVirt[2] = addr; } /* Set the limiter. @@ -2656,8 +2645,7 @@ int sqlite3Select( /* If the output is destined for a temporary table, open that table. */ if( eDest==SRT_TempTable ){ - sqlite3VdbeAddOp(v, OP_OpenVirtual, iParm, 0); - sqlite3VdbeAddOp(v, OP_SetNumColumns, iParm, pEList->nExpr); + sqlite3VdbeAddOp(v, OP_OpenVirtual, iParm, pEList->nExpr); } /* Do an analysis of aggregate expressions. @@ -2720,8 +2708,11 @@ int sqlite3Select( /* Open a virtual index to use for the distinct set. */ if( isDistinct ){ + KeyInfo *pKeyInfo; distinct = pParse->nTab++; - openVirtualIndex(pParse, p, distinct); + pKeyInfo = keyInfoFromExprList(pParse, p->pEList); + sqlite3VdbeOp3(v, OP_OpenVirtual, distinct, 0, + (char*)pKeyInfo, P3_KEYINFO_HANDOFF); }else{ distinct = -1; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b391eb3bf6..4dc7ed0530 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.406 2005/08/30 00:54:03 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.407 2005/09/01 03:07:44 drh Exp $ */ #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -886,13 +886,13 @@ struct Expr { struct ExprList { int nExpr; /* Number of expressions on the list */ int nAlloc; /* Number of entries allocated below */ + int iTab; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { Expr *pExpr; /* The list of expressions */ char *zName; /* Token associated with this expression */ u8 sortOrder; /* 1 for DESC or 0 for ASC */ u8 isAgg; /* True if this is an aggregate like count(*) */ u8 done; /* A flag to indicate when processing is finished */ - u8 orderByDup[2]; /* Corresponding term in OrderBy/GroupBy clause */ } *a; /* One entry for each expression */ }; @@ -1045,23 +1045,35 @@ struct NameContext { ** limit and nOffset to the value of the offset (or 0 if there is not ** offset). But later on, nLimit and nOffset become the memory locations ** in the VDBE that record the limit and offset counters. +** +** addrOpenVirt[] entries contain the address of OP_OpenVirtual opcodes. +** These addresses must be stored so that we can go back and fill in +** the P3_KEYINFO and P2 parameters later. Neither the KeyInfo nor +** the number of columns in P2 can be computed at the same time +** as the OP_OpenVirtual instruction is coded because not +** enough information about the compound query is known at that point. +** The KeyInfo for addrOpenVirt[0] and [1] contains collating sequences +** for the result set. The KeyInfo for addrOpenVirt[2] contains collating +** sequences for the ORDER BY clause. */ struct Select { ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u8 isDistinct; /* True if the DISTINCT keyword is present */ + u8 isResolved; /* True once sqlite3SelectResolve() has run. */ + u8 isAgg; /* True if this is an aggregate query */ + u8 usesVirt; /* True if uses an OpenVirtual opcode */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ Expr *pHaving; /* The HAVING clause */ ExprList *pOrderBy; /* The ORDER BY clause */ Select *pPrior; /* Prior select in a compound select statement */ + Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ - IdList **ppOpenVirtual;/* OP_OpenVirtual addresses used by multi-selects */ - u8 isResolved; /* True once sqlite3SelectResolve() has run. */ - u8 isAgg; /* True if this is an aggregate query */ + int addrOpenVirt[3]; /* OP_OpenVirtual opcodes related to this select */ }; /* diff --git a/src/vdbe.c b/src/vdbe.c index b6cdcdc9ad..57bda63e00 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -43,7 +43,7 @@ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. ** -** $Id: vdbe.c,v 1.478 2005/07/28 20:51:19 drh Exp $ +** $Id: vdbe.c,v 1.479 2005/09/01 03:07:44 drh Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -205,39 +205,6 @@ static void popStack(Mem **ppTos, int N){ *ppTos = pTos; } -/* -** The parameters are pointers to the head of two sorted lists -** of Sorter structures. Merge these two lists together and return -** a single sorted list. This routine forms the core of the merge-sort -** algorithm. -** -** In the case of a tie, left sorts in front of right. -*/ -static Sorter *Merge(Sorter *pLeft, Sorter *pRight, KeyInfo *pKeyInfo){ - Sorter sHead; - Sorter *pTail; - pTail = &sHead; - pTail->pNext = 0; - while( pLeft && pRight ){ - int c = sqlite3VdbeRecordCompare(pKeyInfo, pLeft->nKey, pLeft->zKey, - pRight->nKey, pRight->zKey); - if( c<=0 ){ - pTail->pNext = pLeft; - pLeft = pLeft->pNext; - }else{ - pTail->pNext = pRight; - pRight = pRight->pNext; - } - pTail = pTail->pNext; - } - if( pLeft ){ - pTail->pNext = pLeft; - }else if( pRight ){ - pTail->pNext = pRight; - } - return sHead.pNext; -} - /* ** Allocate cursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -635,7 +602,7 @@ case OP_Return: { /* no-push */ /* Opcode: Halt P1 P2 P3 ** -** Exit immediately. All open cursors, Lists, Sorts, etc are closed +** Exit immediately. All open cursors, Fifos, etc are closed ** automatically. ** ** P1 is the result code returned by sqlite3_exec(), sqlite3_reset(), @@ -2603,13 +2570,14 @@ case OP_OpenWrite: { /* no-push */ break; } -/* Opcode: OpenVirtual P1 * P3 +/* Opcode: OpenVirtual P1 P2 P3 ** -** Open a new cursor to a transient or virtual table. +** Open a new cursor P1 to a transient or virtual table. ** The cursor is always opened read/write even if ** the main database is read-only. The transient or virtual ** table is deleted automatically when the cursor is closed. ** +** P2 is the number of columns in the virtual table. ** The cursor points to a BTree table if P3==0 and to a BTree index ** if P3 is not 0. If P3 is not NULL, it points to a KeyInfo structure ** that defines the format of keys in the index. @@ -2650,6 +2618,7 @@ case OP_OpenVirtual: { /* no-push */ pCx->pIncrKey = &pCx->bogusIncrKey; } } + pCx->nField = pOp->p2; pCx->isIndex = !pCx->isTable; break; } @@ -3464,6 +3433,23 @@ case OP_Last: { /* no-push */ break; } + +/* Opcode: Sort P1 P2 * +** +** This opcode does exactly the same thing as OP_Rewind except that +** it increments an undocumented global variable used for testing. +** +** Sorting is accomplished by writing records into a sorting index, +** then rewinding that index and playing it back from beginning to +** end. We use the OP_Sort opcode instead of OP_Rewind to do the +** rewinding so that the global variable will be incremented and +** regression tests can determine whether or not the optimizer is +** correctly optimizing out sorts. +*/ +case OP_Sort: { /* no-push */ + sqlite3_sort_count++; + /* Fall through into OP_Rewind */ +} /* Opcode: Rewind P1 P2 * ** ** The next use of the Rowid or Column or Next instruction for P1 @@ -4091,108 +4077,6 @@ case OP_ContextPop: { /* no-push */ } #endif /* #ifndef SQLITE_OMIT_TRIGGER */ -/* Opcode: SortInsert * * * -** -** The TOS is the key and the NOS is the data. Pop both from the stack -** and put them on the sorter. The key and data should have been -** made using the MakeRecord opcode. -*/ -case OP_SortInsert: { /* no-push */ - Mem *pNos = &pTos[-1]; - Sorter *pSorter; - assert( pNos>=p->aStack ); - if( Dynamicify(pTos, db->enc) ) goto no_mem; - pSorter = sqliteMallocRaw( sizeof(Sorter) ); - if( pSorter==0 ) goto no_mem; - pSorter->pNext = 0; - if( p->pSortTail ){ - p->pSortTail->pNext = pSorter; - }else{ - p->pSort = pSorter; - } - p->pSortTail = pSorter; - assert( pTos->flags & MEM_Dyn ); - pSorter->nKey = pTos->n; - pSorter->zKey = pTos->z; - pSorter->data.flags = MEM_Null; - rc = sqlite3VdbeMemMove(&pSorter->data, pNos); - pTos -= 2; - break; -} - -/* Opcode: Sort * * P3 -** -** Sort all elements on the sorter. The algorithm is a -** mergesort. The P3 argument is a pointer to a KeyInfo structure -** that describes the keys to be sorted. -*/ -case OP_Sort: { /* no-push */ - int i; - KeyInfo *pKeyInfo = (KeyInfo*)pOp->p3; - Sorter *pElem; - Sorter *apSorter[NSORT]; - sqlite3_sort_count++; - pKeyInfo->enc = p->db->enc; - for(i=0; ipSort ){ - pElem = p->pSort; - p->pSort = pElem->pNext; - pElem->pNext = 0; - for(i=0; i=NSORT-1 ){ - apSorter[NSORT-1] = Merge(apSorter[NSORT-1],pElem, pKeyInfo); - } - } - pElem = 0; - for(i=0; ipSort = pElem; - break; -} - -/* Opcode: SortNext * P2 * -** -** Push the data for the topmost element in the sorter onto the -** stack, then remove the element from the sorter. If the sorter -** is empty, push nothing on the stack and instead jump immediately -** to instruction P2. -*/ -case OP_SortNext: { - Sorter *pSorter = p->pSort; - CHECK_FOR_INTERRUPT; - if( pSorter!=0 ){ - p->pSort = pSorter->pNext; - pTos++; - pTos->flags = MEM_Null; - rc = sqlite3VdbeMemMove(pTos, &pSorter->data); - sqliteFree(pSorter->zKey); - sqliteFree(pSorter); - }else{ - pc = pOp->p2 - 1; - } - break; -} - -/* Opcode: SortReset * * * -** -** Remove any elements that remain on the sorter. -*/ -case OP_SortReset: { /* no-push */ - sqlite3VdbeSorterReset(p); - break; -} - /* Opcode: MemStore P1 P2 * ** ** Write the top of the stack into memory location P1. diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 34e8d230be..1936d092d1 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -125,23 +125,6 @@ struct Mem { }; typedef struct Mem Mem; -/* -** A sorter builds a list of elements to be sorted. Each element of -** the list is an instance of the following structure. -*/ -typedef struct Sorter Sorter; -struct Sorter { - int nKey; /* Number of bytes in the key */ - char *zKey; /* The key by which we will sort */ - Mem data; - Sorter *pNext; /* Next in the list */ -}; - -/* -** Number of buckets used for merge-sort. -*/ -#define NSORT 30 - /* One or more of the following flags are set to indicate the validOK ** representations of the value stored in the Mem struct. ** @@ -324,8 +307,6 @@ struct Vdbe { Mem *aColName; /* Column names to return */ int nCursor; /* Number of slots in apCsr[] */ Cursor **apCsr; /* One element of this array for each open cursor */ - Sorter *pSort; /* A linked list of objects to be sorted */ - Sorter *pSortTail; /* Last element on the pSort list */ int nVar; /* Number of entries in aVar[] */ Mem *aVar; /* Values for the OP_Variable opcode. */ char **azVar; /* Name of variables */ @@ -373,7 +354,6 @@ struct Vdbe { ** Function prototypes */ void sqlite3VdbeFreeCursor(Cursor*); -void sqlite3VdbeSorterReset(Vdbe*); int sqlite3VdbeAggReset(sqlite3*, Agg *, KeyInfo *); void sqliteVdbePopStack(Vdbe*,int); int sqlite3VdbeCursorMoveto(Cursor*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a14f55ea9a..fc7c35c9fb 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -763,21 +763,6 @@ void sqlite3VdbeMakeReady( #endif } - -/* -** Remove any elements that remain on the sorter for the VDBE given. -*/ -void sqlite3VdbeSorterReset(Vdbe *p){ - while( p->pSort ){ - Sorter *pSorter = p->pSort; - p->pSort = pSorter->pNext; - sqliteFree(pSorter->zKey); - sqlite3VdbeMemRelease(&pSorter->data); - sqliteFree(pSorter); - } - p->pSortTail = 0; -} - /* ** Free all resources allociated with AggElem pElem, an element of ** aggregate pAgg. @@ -965,7 +950,6 @@ static void Cleanup(Vdbe *p){ } sqliteFree(p->contextStack); } - sqlite3VdbeSorterReset(p); for(i=0; inAgg; i++){ sqlite3VdbeAggReset(0, &p->apAgg[i], 0); } diff --git a/test/conflict.test b/test/conflict.test index cc30c17f6a..3726ce0a55 100644 --- a/test/conflict.test +++ b/test/conflict.test @@ -13,7 +13,7 @@ # This file implements tests for the conflict resolution extension # to SQLite. # -# $Id: conflict.test,v 1.24 2005/06/07 02:12:30 drh Exp $ +# $Id: conflict.test,v 1.25 2005/09/01 03:07:45 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -281,18 +281,18 @@ do_test conflict-6.0 { # foreach {i conf1 cmd t0 t1 t2 t3} { 1 {} UPDATE 1 {6 7 8 9} 1 1 - 2 REPLACE UPDATE 0 {7 6 9} 1 0 - 3 IGNORE UPDATE 0 {6 7 3 9} 1 0 + 2 REPLACE UPDATE 0 {7 6 9} 1 1 + 3 IGNORE UPDATE 0 {6 7 3 9} 1 1 4 FAIL UPDATE 1 {6 7 3 4} 1 0 5 ABORT UPDATE 1 {1 2 3 4} 1 1 6 ROLLBACK UPDATE 1 {1 2 3 4} 0 0 - 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 - 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 0 - 9 FAIL {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 - 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 0 - 11 ROLLBACK {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 - 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 0 - 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 0 + 7 REPLACE {UPDATE OR IGNORE} 0 {6 7 3 9} 1 1 + 8 IGNORE {UPDATE OR REPLACE} 0 {7 6 9} 1 1 + 9 FAIL {UPDATE OR IGNORE} 0 {6 7 3 9} 1 1 + 10 ABORT {UPDATE OR REPLACE} 0 {7 6 9} 1 1 + 11 ROLLBACK {UPDATE OR IGNORE} 0 {6 7 3 9} 1 1 + 12 {} {UPDATE OR IGNORE} 0 {6 7 3 9} 1 1 + 13 {} {UPDATE OR REPLACE} 0 {7 6 9} 1 1 14 {} {UPDATE OR FAIL} 1 {6 7 3 4} 1 0 15 {} {UPDATE OR ABORT} 1 {1 2 3 4} 1 1 16 {} {UPDATE OR ROLLBACK} 1 {1 2 3 4} 0 0