Add code to handle recursive CTEs.

FossilOrigin-Name: a5c2a54a07d35166911abc792008c05dea897742
This commit is contained in:
dan 2014-01-14 20:14:09 +00:00
parent a9f5c13d0c
commit 8ce7184bc2
9 changed files with 262 additions and 33 deletions

@ -1,5 +1,5 @@
C Fix\ssome\smemory\sleaks\sand\scrashes\sthat\scould\sfollow\san\sOOM\scondition\sduring\sWITH\sclause\sparsing.
D 2014-01-13T16:36:40.215
C Add\scode\sto\shandle\srecursive\sCTEs.
D 2014-01-14T20:14:09.053
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -166,7 +166,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53
F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df
F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142
F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9
F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0
F src/build.c 5a2daa6649640711ca60114046903be932348e52
@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd
F src/date.c 593c744b2623971e45affd0bde347631bdfa4625
F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff
F src/expr.c de86bf987c532ec9d63fb1bf8a6eb6ec2cf5ba69
F src/expr.c fee4b54fdcf5a979e6f5012da3dfb084ac5d3aac
F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5
F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19
@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269
F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6
F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
F src/select.c 9dc9177bfb9278a603e3835e828ff60ec68fba6f
F src/select.c 0f7779b0c1c317dcac0f65f695d036030c16775f
F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344
F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9
F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
F src/sqliteInt.h a350bcf6d62204b6b4720ce1922c95a575a0b5ad
F src/sqliteInt.h fc7b2516260c4e14bd4342cd9ed47f3d0ca27cc5
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb
F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269
F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c
F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179
F src/vdbe.c b110887e415b5d2af58c2374c4dfdcf774c5d46c
F src/vdbe.c 9b918126afb9a7b7ad232db7453680c7ef5821a4
F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26
F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56
F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad
@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74
F src/where.c 18f07fd0fd116a5880773c689eb7cd8e60175107
F src/where.c 830b42f452cfbc4e17582f6c1d388e15b379b833
F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358
F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361
F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
F test/with1.test fb8409a35b1314be6e73a87597322f3369b59b2b
F test/with1.test 12a6661eabcc9ae299020f7b4197a75a9c084748
F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34
F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8
F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
P a26f399ba485e8127c276c5f103ec6c555e11734
R 7e06730d3714a8aceb489d831c2bc18d
P 8839850c44a8938883e493eacd752fa686e542df
R 1d5d9f6beb143e76aa04b4bf9953f5da
U dan
Z 780f0749f7ee296248db4bd00ffb23e3
Z 1616366ca894deb1473f9b1eb1563ebc

@ -1 +1 @@
8839850c44a8938883e493eacd752fa686e542df
a5c2a54a07d35166911abc792008c05dea897742

@ -7350,6 +7350,7 @@ static int clearDatabasePage(
int rc;
unsigned char *pCell;
int i;
int hdr;
assert( sqlite3_mutex_held(pBt->mutex) );
if( pgno>btreePagecount(pBt) ){
@ -7358,6 +7359,7 @@ static int clearDatabasePage(
rc = getAndInitPage(pBt, pgno, &pPage, 0);
if( rc ) return rc;
hdr = pPage->hdrOffset;
for(i=0; i<pPage->nCell; i++){
pCell = findCell(pPage, i);
if( !pPage->leaf ){
@ -7368,7 +7370,7 @@ static int clearDatabasePage(
if( rc ) goto cleardatabasepage_out;
}
if( !pPage->leaf ){
rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange);
rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
if( rc ) goto cleardatabasepage_out;
}else if( pnChange ){
assert( pPage->intKey );
@ -7377,7 +7379,7 @@ static int clearDatabasePage(
if( freePageFlag ){
freePage(pPage, &rc);
}else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){
zeroPage(pPage, pPage->aData[0] | PTF_LEAF);
zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF);
}
cleardatabasepage_out:

@ -1055,6 +1055,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
pNew->addrOpenEphm[1] = -1;
pNew->addrOpenEphm[2] = -1;
pNew->pWith = withDup(db, p->pWith);
pNew->pRecurse = p->pRecurse;
return pNew;
}
#else

@ -691,12 +691,26 @@ static void selectInnerLoop(
/* Store the result as data using a unique key.
*/
case SRT_DistTable:
case SRT_Table:
case SRT_EphemTab: {
int r1 = sqlite3GetTempReg(pParse);
testcase( eDest==SRT_Table );
testcase( eDest==SRT_EphemTab );
sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1);
#ifndef SQLITE_OMIT_CTE
if( eDest==SRT_DistTable ){
/* If the destination is DistTable, then cursor (iParm+1) is open
** on an ephemeral index. If the current row is already present
** in the index, do not write it to the output. If not, add the
** current row to the index and proceed with writing it to the
** output table as well. */
int addr = sqlite3VdbeCurrentAddr(v) + 4;
sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0);
sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1);
assert( pOrderBy==0 );
}
#endif
if( pOrderBy ){
pushOntoSorter(pParse, pOrderBy, p, r1);
}else{
@ -1729,6 +1743,7 @@ static int multiSelect(
** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
*/
assert( p && p->pPrior ); /* Calling function guarantees this much */
assert( p->pRecurse==0 || p->op==TK_ALL || p->op==TK_UNION );
db = pParse->db;
pPrior = p->pPrior;
assert( pPrior->pRightmost!=pPrior );
@ -1774,12 +1789,82 @@ static int multiSelect(
goto multi_select_end;
}
/* If this is a recursive query, check that there is no ORDER BY or
** LIMIT clause. Neither of these are supported. */
assert( p->pOffset==0 || p->pLimit );
if( p->pRecurse && (p->pOrderBy || p->pLimit) ){
sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed",
p->pOrderBy ? "ORDER BY" : "LIMIT"
);
goto multi_select_end;
}
/* Compound SELECTs that have an ORDER BY clause are handled separately.
*/
if( p->pOrderBy ){
return multiSelectOrderBy(pParse, p, pDest);
}
#ifndef SQLITE_OMIT_CTE
if( p->pRecurse ){
int nCol = p->pEList->nExpr;
int addrNext;
int addrSwap;
int iCont, iBreak;
int tmp1, tmp2; /* Cursors used to access temporary tables */
int tmp3 = 0; /* To ensure unique results if UNION */
int eDest = SRT_Table;
SelectDest tmp2dest;
iBreak = sqlite3VdbeMakeLabel(v);
iCont = sqlite3VdbeMakeLabel(v);
tmp1 = pParse->nTab++;
tmp2 = pParse->nTab++;
if( p->op==TK_UNION ){
eDest = SRT_DistTable;
tmp3 = pParse->nTab++;
}
sqlite3SelectDestInit(&tmp2dest, eDest, tmp2);
sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol);
sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol);
if( tmp3 ){
p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0);
p->selFlags |= SF_UsesEphemeral;
}
/* Store the results of the initial SELECT in tmp2. */
rc = sqlite3Select(pParse, pPrior, &tmp2dest);
if( rc ) goto multi_select_end;
/* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then teturn
** the contents of tmp1 to the caller. Or, if tmp1 is empty at this
** point, the recursive query has finished - jump to address iBreak. */
addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2);
sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak);
addrNext = sqlite3VdbeCurrentAddr(v);
selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr,
0, 0, &dest, iCont, iBreak);
sqlite3VdbeResolveLabel(v, iCont);
sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext);
/* Execute the recursive SELECT. Store the results in tmp2. While this
** SELECT is running, the contents of tmp1 are read by recursive
** references to the current CTE. */
p->pPrior = 0;
p->pRecurse->tnum = tmp1;
p->pRecurse->tabFlags |= TF_Recursive;
rc = sqlite3Select(pParse, p, &tmp2dest);
p->pRecurse->tabFlags &= ~TF_Recursive;
p->pPrior = pPrior;
if( rc ) goto multi_select_end;
sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap);
sqlite3VdbeResolveLabel(v, iBreak);
}else
#endif
/* Generate code for the left and right SELECT statements.
*/
switch( p->op ){
@ -3449,6 +3534,73 @@ static void ctePop(Parse *pParse, struct Cte *pCte){
}
}
static int withExpand(
Walker *pWalker,
struct SrcList_item *pFrom,
struct Cte *pCte
){
Table *pTab;
Parse *pParse = pWalker->pParse;
sqlite3 *db = pParse->db;
assert( pFrom->pSelect==0 );
assert( pFrom->pTab==0 );
if( pCte==pParse->pCte && (pTab = pCte->pTab) ){
/* This is the recursive part of a recursive CTE */
pFrom->pTab = pTab;
pTab->nRef++;
}else{
ExprList *pEList;
Select *pSel;
int bRecursive;
pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
if( pTab==0 ) return WRC_Abort;
pTab->nRef = 1;
pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab);
pTab->iPKey = -1;
pTab->nRowEst = 1048576;
pTab->tabFlags |= TF_Ephemeral;
pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
if( db->mallocFailed ) return SQLITE_NOMEM;
assert( pFrom->pSelect );
if( ctePush(pParse, pCte) ) return WRC_Abort;
pSel = pFrom->pSelect;
bRecursive = (pSel->op==TK_ALL || pSel->op==TK_UNION);
if( bRecursive ){
assert( pSel->pPrior );
sqlite3WalkSelect(pWalker, pSel->pPrior);
}else{
sqlite3WalkSelect(pWalker, pSel);
}
if( pCte->pCols ){
pEList = pCte->pCols;
}else{
Select *pLeft;
for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
pEList = pLeft->pEList;
}
selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
if( bRecursive ){
int nRef = pTab->nRef;
pCte->pTab = pTab;
sqlite3WalkSelect(pWalker, pSel);
pCte->pTab = 0;
if( pTab->nRef > nRef){
pSel->pRecurse = pTab;
assert( pTab->tnum==0 );
}
}
ctePop(pParse, pCte);
}
return SQLITE_OK;
}
/*
** This routine is a Walker callback for "expanding" a SELECT statement.
@ -3515,31 +3667,22 @@ static int selectExpander(Walker *pWalker, Select *p){
#ifndef SQLITE_OMIT_CTE
pCte = searchWith(pParse, pFrom);
if( pCte ){
pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
if( pFrom->pSelect==0 ) return WRC_Abort;
}
if( withExpand(pWalker, pFrom, pCte) ) return WRC_Abort;
}else
#endif
if( pFrom->zName==0 || pCte ){
if( pFrom->zName==0 ){
#ifndef SQLITE_OMIT_SUBQUERY
ExprList *pEList;
Select *pSel = pFrom->pSelect;
/* A sub-query in the FROM clause of a SELECT */
assert( pSel!=0 );
assert( pFrom->pTab==0 );
if( ctePush(pParse, pCte) ) return WRC_Abort;
sqlite3WalkSelect(pWalker, pSel);
ctePop(pParse, pCte);
pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
if( pTab==0 ) return WRC_Abort;
pTab->nRef = 1;
pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab);
while( pSel->pPrior ){ pSel = pSel->pPrior; }
if( pCte && pCte->pCols ){
pEList = pCte->pCols;
}else{
pEList = pSel->pEList;
}
selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol);
pTab->iPKey = -1;
pTab->nRowEst = 1048576;
pTab->tabFlags |= TF_Ephemeral;
@ -3831,9 +3974,10 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){
/* A sub-query in the FROM clause of a SELECT */
Select *pSel = pFrom->pSelect;
assert( pSel );
while( pSel->pPrior ) pSel = pSel->pPrior;
selectAddColumnTypeAndCollation(pParse, pTab, pSel);
if( pSel ){
while( pSel->pPrior ) pSel = pSel->pPrior;
selectAddColumnTypeAndCollation(pParse, pTab, pSel);
}
}
}
}

@ -1429,7 +1429,7 @@ struct Table {
};
/*
** Allowed values for Tabe.tabFlags.
** Allowed values for Table.tabFlags.
*/
#define TF_Readonly 0x01 /* Read-only system table */
#define TF_Ephemeral 0x02 /* An ephemeral table */
@ -1437,6 +1437,7 @@ struct Table {
#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */
#define TF_Virtual 0x10 /* Is a virtual table */
#define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */
#define TF_Recursive 0x40 /* Recursive reference within CTE */
/*
@ -2129,6 +2130,7 @@ struct NameContext {
*/
struct Select {
ExprList *pEList; /* The fields of the result */
Table *pRecurse; /* Non-NULL for the recursive part of recursive CTE */
u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */
u16 selFlags; /* Various SF_* values */
int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */
@ -2182,6 +2184,7 @@ struct Select {
#define SRT_Table 8 /* Store result as data with an automatic rowid */
#define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */
#define SRT_Coroutine 10 /* Generate a single row of result */
#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */
/*
** An instance of this object describes where to put of the results of
@ -2647,6 +2650,7 @@ struct With {
ExprList *pCols; /* List of explicit column names, or NULL */
Select *pSelect; /* The contents of the CTE */
struct Cte *pOuterCte;
Table *pTab;
} a[1];
};

@ -3369,6 +3369,53 @@ case OP_OpenEphemeral: {
break;
}
/* Opcode: OpenEphreader P1 P2 * * *
**
** P2 is a cursor opened by the OpenEphemeral opcode. This opcode opens
** a new read-only cursor named P1 that accesses the same epheremal table
** as P2.
*/
case OP_OpenEphreader: {
VdbeCursor *pEph;
VdbeCursor *pCx;
Pgno pgno;
pEph = p->apCsr[pOp->p2];
pCx = allocateCursor(p, pOp->p1, pEph->nField, -1, 1);
if( pCx==0 ) goto no_mem;
pCx->nullRow = 1;
pCx->pKeyInfo = pEph->pKeyInfo;
pCx->isTable = pEph->isTable;
pCx->isOrdered = pEph->isOrdered;
pgno = MASTER_ROOT + !pCx->isTable;
rc = sqlite3BtreeCursor(pEph->pBt, pgno, 0, pCx->pKeyInfo, pCx->pCursor);
break;
}
/* Opcode: SwapCursors P1 P2 * * *
**
** Parameters P1 and P2 are both cursors opened by the OpenEphemeral
** opcode. This opcode deletes the contents of epheremal table P1,
** then renames P2 to P1 and P1 to P2. In other words, following this
** opcode cursor P2 is open on an empty table and P1 is open on the
** table that was initially accessed by P2.
*/
case OP_SwapCursors: {
Mem tmp;
VdbeCursor *pTmp;
tmp = p->aMem[p->nMem - pOp->p1];
p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2];
p->aMem[p->nMem - pOp->p2] = tmp;
pTmp = p->apCsr[pOp->p1];
p->apCsr[pOp->p1] = p->apCsr[pOp->p2];
p->apCsr[pOp->p2] = pTmp;
rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0);
break;
}
/* Opcode: SorterOpen P1 * * P4 *
**
** This opcode works like OP_OpenEphemeral except that it opens

@ -5653,7 +5653,11 @@ WhereInfo *sqlite3WhereBegin(
iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
pLoop = pLevel->pWLoop;
if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){
/* Do nothing */
if( pTab->tabFlags & TF_Recursive ){
int iCur = pTabItem->iCursor;
sqlite3VdbeAddOp2(v, OP_OpenEphreader, iCur, pTab->tnum);
}
/* Otherwise do nothing */
}else
#ifndef SQLITE_OMIT_VIRTUALTABLE
if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){

@ -137,6 +137,33 @@ do_execsql_test 4.3 {
SELECT * FROM t1;
} {1 3 8 9}
#-------------------------------------------------------------------------
#
do_execsql_test 5.1 {
WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i)
SELECT x FROM i LIMIT 10;
} {1 2 3 4 5 6 7 8 9 10}
do_catchsql_test 5.2 {
WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1)
SELECT x FROM i LIMIT 10;
} {1 {ORDER BY in a recursive query is not allowed}}
do_catchsql_test 5.3 {
WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 )
SELECT x FROM i LIMIT 10;
} {1 {LIMIT in a recursive query is not allowed}}
do_execsql_test 5.4 {
WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i)
SELECT x FROM i LIMIT 20;
} {1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0}
do_execsql_test 5.5 {
WITH i(x) AS ( VALUES(1) UNION SELECT (x+1)%10 FROM i)
SELECT x FROM i LIMIT 20;
} {1 2 3 4 5 6 7 8 9 0}
finish_test