From 381bdacc8fd374db52bba13be966189abc787257 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 4 Feb 2021 17:29:04 +0000 Subject: [PATCH] Preliminary changes for a new implementation of RETURNING that captures all results in a buffer and plays them all back after the DML statement completes. This avoids problems with interleaved DML statements. This particular check-in is a non-functional work in progress. FossilOrigin-Name: 04b77d63216ce11b4e797946953bcde504fc005807c7a5ac757fbf47d78698dc --- manifest | 19 +++++---- manifest.uuid | 2 +- src/build.c | 37 ++++++++++++++---- src/sqliteInt.h | 11 ++++-- src/trigger.c | 100 ++++++++++++++++++++++++++++++++---------------- 5 files changed, 117 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index cb5c4da361..50a026f7a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER()\sto\sa\sbranch\sthat\scheck-in\s[5d54d9fd40638138]\sapparently\smade\nunreachable. -D 2021-02-04T13:52:34.346 +C Preliminary\schanges\sfor\sa\snew\simplementation\sof\sRETURNING\sthat\scaptures\sall\nresults\sin\sa\sbuffer\sand\splays\sthem\sall\sback\safter\sthe\sDML\sstatement\ncompletes.\s\sThis\savoids\sproblems\swith\sinterleaved\sDML\sstatements.\nThis\sparticular\scheck-in\sis\sa\snon-functional\swork\sin\sprogress. +D 2021-02-04T17:29:04.659 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -485,7 +485,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c 694020ad8a3af3d79b09f74c8f1421272a419cdea42a13401e3b0f7dea6e9c3e F src/btree.h 285f8377aa1353185a32bf455faafa9ff9a0d40d074d60509534d14990c7829e F src/btreeInt.h 7614cae30f95b6aed0c7cac7718276a55cfe2c77058cbfd8bef5b75329757331 -F src/build.c 4a70f3ce393dba6963e7d74f72fab0108da6c87e43f1d8afe3fdb31034e190f6 +F src/build.c 5285b8d1e4cf21dd9e12619fe0a6067bcb036ee7f7ad5cd31a39a05c5823e2a2 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 2a322b9a3d75771fb4d99e0702851f4f68dda982507a0f798eefb0712969a410 @@ -546,7 +546,7 @@ F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a087 F src/sqlite.h.in 8855a19f37ade8dad189a9e48233a2ebe1b46faf469c7eb0906a654e252dcc57 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e -F src/sqliteInt.h 0fda3b2c05b1559135aa2c4ecb8e75bd2085ba4433310bbb5427d97c2d81315d +F src/sqliteInt.h 596729ed0c2509a32cb17cecc7baf2a3499299064c03dbe79f8daca08d4daf7c F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1 @@ -607,7 +607,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c F src/tokenize.c c64c49d7c2ec4490c2fef1f24350167ba16b03b0c6cee58ad1a1d70a4325d4e9 F src/treeview.c 4b92992176fb2caefbe06ba5bd06e0e0ebcde3d5564758da672631f17aa51cda -F src/trigger.c 0a242d65dd9b9822d4e990653eb4ece3557dcda01374934aa3cc1f9718d8dee3 +F src/trigger.c 803db1f98d84a86b31a13e8a42715936bc760d0192bc4264d57a96f362c2a591 F src/update.c 0f5a61f0787199983530a33f6fffe4f52742f35fcdf6ccfad1078b1a8bc17723 F src/upsert.c df8f1727d62b5987c4fd302cd4d7c0c84ae57cd65683c5a34a740dfe24039235 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 @@ -1899,7 +1899,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0f34f241d37cc4f31dc657a6fc33815fa5e24298487121cb27c29118524b8ee7 -R cff545e4a39efc504a1ff4bec6444fd1 +P 5c8e6296aa9f69a092364524b716ba894f113f7f1e6024b9a2eaa01c239e65c1 +R 9ba8b89a255b185f318ef97355eff6b3 +T *branch * returning-manifested +T *sym-returning-manifested * +T -sym-trunk * U drh -Z e8f529c01cdfd85106db62be4d9db740 +Z 04ce06a70e76b82a776d153a576955fb diff --git a/manifest.uuid b/manifest.uuid index 0cefd72434..be203d8902 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c8e6296aa9f69a092364524b716ba894f113f7f1e6024b9a2eaa01c239e65c1 \ No newline at end of file +04b77d63216ce11b4e797946953bcde504fc005807c7a5ac757fbf47d78698dc \ No newline at end of file diff --git a/src/build.c b/src/build.c index d592827792..48607bfe81 100644 --- a/src/build.c +++ b/src/build.c @@ -155,6 +155,24 @@ void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ + if( pParse->bReturning ){ + Returning *pReturning = pParse->u1.pReturning; + int addrRewind; + int i; + int reg; + + sqlite3VdbeAddOp0(v, OP_Noop); VdbeComment((v, "TODO: Commit changes")); + addrRewind = + sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); + reg = pParse->nMem+1; + pParse->nMem += pReturning->nRetCol; + for(i=0; inRetCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i); + sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1); + sqlite3VdbeJumpHere(v, addrRewind); + } sqlite3VdbeAddOp0(v, OP_Halt); #if SQLITE_USER_AUTHENTICATION @@ -232,6 +250,11 @@ void sqlite3FinishCoding(Parse *pParse){ } } + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + /* Finally, jump back to the beginning of the executable code. */ sqlite3VdbeGoto(v, 1); } @@ -1213,7 +1236,8 @@ void sqlite3StartTable( }else #endif { - pParse->addrCrTab = + assert( !pParse->bReturning ); + pParse->u1.addrCrTab = sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } sqlite3OpenSchemaTable(pParse, iDb); @@ -1291,6 +1315,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ sqlite3ExprListDelete(db, pList); return; } + pParse->u1.pReturning = pRet; pRet->pParse = pParse; pRet->pReturnEL = pList; sqlite3ParserAddCleanup(pParse, @@ -1304,10 +1329,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){ pRet->retTrig.step_list = &pRet->retTStep; pRet->retTStep.op = TK_RETURNING; pRet->retTStep.pTrig = &pRet->retTrig; - pRet->retTStep.pSelect = &pRet->retSel; - pRet->retSel.op = TK_ALL; - pRet->retSel.pEList = pList; - pRet->retSel.pSrc = (SrcList*)&pRet->retSrcList; + pRet->retTStep.pExprList = pList; pHash = &(db->aDb[1].pSchema->trigHash); assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 ); if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig) @@ -2147,9 +2169,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY ** into BTREE_BLOBKEY. */ - if( pParse->addrCrTab ){ + assert( !pParse->bReturning ); + if( pParse->u1.addrCrTab ){ assert( v ); - sqlite3VdbeChangeP3(v, pParse->addrCrTab, BTREE_BLOBKEY); + sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3cadb7f53a..528c908ca6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3425,7 +3425,10 @@ struct Parse { Table *pTriggerTab; /* Table triggers are being coded for */ Parse *pParentParse; /* Parent parser if this parser is nested */ AggInfo *pAggList; /* List of all AggInfo objects */ - int addrCrTab; /* Address of OP_CreateBtree opcode on CREATE TABLE */ + union { + int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ + Returning *pReturning; /* The RETURNING clause */ + } u1; u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ @@ -3647,7 +3650,7 @@ struct TriggerStep { char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ - ExprList *pExprList; /* SET clause for UPDATE */ + ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ Upsert *pUpsert; /* Upsert clauses on an INSERT */ char *zSpan; /* Original SQL text of this command */ @@ -3663,8 +3666,8 @@ struct Returning { ExprList *pReturnEL; /* List of expressions to return */ Trigger retTrig; /* The transient trigger that implements RETURNING */ TriggerStep retTStep; /* The trigger step */ - Select retSel; /* The SELECT statement that implements RETURNING */ - u64 retSrcList; /* The empty FROM clause of the SELECT */ + int iRetCur; /* Transient table holding RETURNING results */ + int nRetCol; /* Number of in pReturnEL after expansion */ }; /* diff --git a/src/trigger.c b/src/trigger.c index ef0a90b24f..fc7e64e51e 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -70,6 +70,8 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ pTrig->pNext = pList; pList = pTrig; }else if( pTrig->op==TK_RETURNING ){ + assert( pParse->bReturning ); + assert( &(pParse->u1.pReturning->retTrig) == pTrig ); pTrig->table = pTab->zName; pTrig->pTabSchema = pTab->pSchema; pTrig->pNext = pList; @@ -857,6 +859,55 @@ static ExprList *sqlite3ExpandReturning( return pNew; } +/* +** Generate code for the RETURNING trigger. Unlike other triggers +** that invoke a subprogram in the bytecode, the code for RETURNING +** is generated in-line. +*/ +static void codeReturningTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* The trigger step that defines the RETURNING */ + Table *pTab, /* The table to code triggers from */ + int reg /* The first in an array of registers */ +){ + Vdbe *v = pParse->pVdbe; + ExprList *pNew; + Returning *pReturning; + + assert( v!=0 ); + assert( pParse->bReturning ); + pReturning = pParse->u1.pReturning; + assert( pTrigger == &(pReturning->retTrig) ); + sqlite3VdbeAddOp0(v, OP_Noop); + VdbeComment((v, "RETURNING trigger goes here")); + pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); + if( pNew ){ + pReturning->nRetCol = pNew->nExpr; + pReturning->iRetCur = pParse->nTab++; + sqlite3ExprListDelete(pParse->db, pNew); + } + +#if 0 + pSelect->pEList = + sqlite3ExpandReturning(pParse, pList, pParse->pTriggerTab); + sqlite3SelectDestInit(&sDest, SRT_Output, 0); + pNew = sqlite3SelectDup(db, pSelect, 0); + if( pNew ){ + sqlite3Select(pParse, pNew, &sDest); + if( pNew->selFlags & (SF_Aggregate|SF_HasAgg|SF_WinRewrite) ){ + sqlite3ErrorMsg(pParse, "aggregates not allowed in RETURNING"); + } + sqlite3SelectDelete(db, pNew); + } + sqlite3ExprListDelete(db, pSelect->pEList); + pStep->pSelect->pEList = pList; + break; + } +#endif +} + + + /* ** Generate VDBE code for the statements inside the body of a single ** trigger. @@ -928,7 +979,7 @@ static int codeTriggerProgram( sqlite3VdbeAddOp0(v, OP_ResetCount); break; } - case TK_SELECT: { + default: assert( pStep->op==TK_SELECT ); { SelectDest sDest; Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0); sqlite3SelectDestInit(&sDest, SRT_Discard, 0); @@ -936,26 +987,6 @@ static int codeTriggerProgram( sqlite3SelectDelete(db, pSelect); break; } - default: assert( pStep->op==TK_RETURNING ); { - Select *pSelect = pStep->pSelect; - ExprList *pList = pSelect->pEList; - SelectDest sDest; - Select *pNew; - pSelect->pEList = - sqlite3ExpandReturning(pParse, pList, pParse->pTriggerTab); - sqlite3SelectDestInit(&sDest, SRT_Output, 0); - pNew = sqlite3SelectDup(db, pSelect, 0); - if( pNew ){ - sqlite3Select(pParse, pNew, &sDest); - if( pNew->selFlags & (SF_Aggregate|SF_HasAgg|SF_WinRewrite) ){ - sqlite3ErrorMsg(pParse, "aggregates not allowed in RETURNING"); - } - sqlite3SelectDelete(db, pNew); - } - sqlite3ExprListDelete(db, pSelect->pEList); - pStep->pSelect->pEList = pList; - break; - } } } @@ -1208,7 +1239,7 @@ void sqlite3CodeRowTriggerDirect( ** ... ... ** reg+N OLD.* value of right-most column of pTab ** reg+N+1 NEW.rowid -** reg+N+2 OLD.* value of left-most column of pTab +** reg+N+2 NEW.* value of left-most column of pTab ** ... ... ** reg+N+N+1 NEW.* value of right-most column of pTab ** @@ -1261,12 +1292,12 @@ void sqlite3CodeRowTrigger( if( (p->op==op || (p->bReturning && p->op==TK_INSERT && op==TK_UPDATE)) && p->tr_tm==tr_tm && checkColumnOverlap(p->pColumns, pChanges) - && (sqlite3IsToplevel(pParse) || !p->bReturning) ){ - u8 origOp = p->op; - p->op = op; - sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); - p->op = origOp; + if( !p->bReturning ){ + sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + }else if( sqlite3IsToplevel(pParse) ){ + codeReturningTrigger(pParse, p, pTab, reg); + } } } } @@ -1311,13 +1342,18 @@ u32 sqlite3TriggerColmask( assert( isNew==1 || isNew==0 ); for(p=pTrigger; p; p=p->pNext){ - if( p->op==op && (tr_tm&p->tr_tm) + if( p->op==op + && (tr_tm&p->tr_tm) && checkColumnOverlap(p->pColumns,pChanges) ){ - TriggerPrg *pPrg; - pPrg = getRowTrigger(pParse, p, pTab, orconf); - if( pPrg ){ - mask |= pPrg->aColmask[isNew]; + if( p->bReturning ){ + mask = 0xffffffff; + }else{ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + if( pPrg ){ + mask |= pPrg->aColmask[isNew]; + } } } }