Preliminary changes for a new implementation of RETURNING that captures all

results in a buffer and plays them all back after the DML statement
completes.  This avoids problems with interleaved DML statements.
This particular check-in is a non-functional work in progress.

FossilOrigin-Name: 04b77d63216ce11b4e797946953bcde504fc005807c7a5ac757fbf47d78698dc

manifest

@@ -1,5 +1,5 @@
-C Add\sNEVER()\sto\sa\sbranch\sthat\scheck-in\s[5d54d9fd40638138]\sapparently\smade\nunreachable.
+C Preliminary\schanges\sfor\sa\snew\simplementation\sof\sRETURNING\sthat\scaptures\sall\nresults\sin\sa\sbuffer\sand\splays\sthem\sall\sback\safter\sthe\sDML\sstatement\ncompletes.\s\sThis\savoids\sproblems\swith\sinterleaved\sDML\sstatements.\nThis\sparticular\scheck-in\sis\sa\snon-functional\swork\sin\sprogress.
-D 2021-02-04T13:52:34.346
+D 2021-02-04T17:29:04.659
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -485,7 +485,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c 694020ad8a3af3d79b09f74c8f1421272a419cdea42a13401e3b0f7dea6e9c3e
 F src/btree.h 285f8377aa1353185a32bf455faafa9ff9a0d40d074d60509534d14990c7829e
 F src/btreeInt.h 7614cae30f95b6aed0c7cac7718276a55cfe2c77058cbfd8bef5b75329757331
-F src/build.c 4a70f3ce393dba6963e7d74f72fab0108da6c87e43f1d8afe3fdb31034e190f6
+F src/build.c 5285b8d1e4cf21dd9e12619fe0a6067bcb036ee7f7ad5cd31a39a05c5823e2a2
 F src/callback.c d0b853dd413255d2e337b34545e54d888ea02f20da5ad0e63585b389624c4a6c
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 2a322b9a3d75771fb4d99e0702851f4f68dda982507a0f798eefb0712969a410
@@ -546,7 +546,7 @@ F src/shell.c.in 9ebc74e4f05cfbd0f4a36060fdaeff1da4e9af4458358722bc08c5a1ab9a087
 F src/sqlite.h.in 8855a19f37ade8dad189a9e48233a2ebe1b46faf469c7eb0906a654e252dcc57
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e
-F src/sqliteInt.h 0fda3b2c05b1559135aa2c4ecb8e75bd2085ba4433310bbb5427d97c2d81315d
+F src/sqliteInt.h 596729ed0c2509a32cb17cecc7baf2a3499299064c03dbe79f8daca08d4daf7c
 F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
 F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -607,7 +607,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c c64c49d7c2ec4490c2fef1f24350167ba16b03b0c6cee58ad1a1d70a4325d4e9
 F src/treeview.c 4b92992176fb2caefbe06ba5bd06e0e0ebcde3d5564758da672631f17aa51cda
-F src/trigger.c 0a242d65dd9b9822d4e990653eb4ece3557dcda01374934aa3cc1f9718d8dee3
+F src/trigger.c 803db1f98d84a86b31a13e8a42715936bc760d0192bc4264d57a96f362c2a591
 F src/update.c 0f5a61f0787199983530a33f6fffe4f52742f35fcdf6ccfad1078b1a8bc17723
 F src/upsert.c df8f1727d62b5987c4fd302cd4d7c0c84ae57cd65683c5a34a740dfe24039235
 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
@@ -1899,7 +1899,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 0f34f241d37cc4f31dc657a6fc33815fa5e24298487121cb27c29118524b8ee7
+P 5c8e6296aa9f69a092364524b716ba894f113f7f1e6024b9a2eaa01c239e65c1
-R cff545e4a39efc504a1ff4bec6444fd1
+R 9ba8b89a255b185f318ef97355eff6b3
+T *branch * returning-manifested
+T *sym-returning-manifested *
+T -sym-trunk *
 U drh
-Z e8f529c01cdfd85106db62be4d9db740
+Z 04ce06a70e76b82a776d153a576955fb

manifest.uuid

@@ -1 +1 @@
-5c8e6296aa9f69a092364524b716ba894f113f7f1e6024b9a2eaa01c239e65c1
+04b77d63216ce11b4e797946953bcde504fc005807c7a5ac757fbf47d78698dc

src/build.c

@@ -155,6 +155,24 @@ void sqlite3FinishCoding(Parse *pParse){
   assert( !pParse->isMultiWrite 
        || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort));
   if( v ){
+    if( pParse->bReturning ){
+      Returning *pReturning = pParse->u1.pReturning;
+      int addrRewind;
+      int i;
+      int reg;
+
+      sqlite3VdbeAddOp0(v, OP_Noop);  VdbeComment((v, "TODO: Commit changes"));
+      addrRewind =
+         sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur);
+      reg = pParse->nMem+1;
+      pParse->nMem += pReturning->nRetCol;
+      for(i=0; i<pReturning->nRetCol; i++){
+        sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i);
+      }
+      sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i);
+      sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1);
+      sqlite3VdbeJumpHere(v, addrRewind);
+    }
     sqlite3VdbeAddOp0(v, OP_Halt);
 
 #if SQLITE_USER_AUTHENTICATION
@@ -232,6 +250,11 @@ void sqlite3FinishCoding(Parse *pParse){
         }
       }
 
+      if( pParse->bReturning ){
+        Returning *pRet = pParse->u1.pReturning;
+        sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol);
+      }
+
       /* Finally, jump back to the beginning of the executable code. */
       sqlite3VdbeGoto(v, 1);
     }
@@ -1213,7 +1236,8 @@ void sqlite3StartTable(
     }else
 #endif
     {
-      pParse->addrCrTab =
+      assert( !pParse->bReturning );
+      pParse->u1.addrCrTab =
          sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY);
     }
     sqlite3OpenSchemaTable(pParse, iDb);
@@ -1291,6 +1315,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
     sqlite3ExprListDelete(db, pList);
     return;
   }
+  pParse->u1.pReturning = pRet;
   pRet->pParse = pParse;
   pRet->pReturnEL = pList;
   sqlite3ParserAddCleanup(pParse,
@@ -1304,10 +1329,7 @@ void sqlite3AddReturning(Parse *pParse, ExprList *pList){
   pRet->retTrig.step_list = &pRet->retTStep;
   pRet->retTStep.op = TK_RETURNING;
   pRet->retTStep.pTrig = &pRet->retTrig;
-  pRet->retTStep.pSelect = &pRet->retSel;
+  pRet->retTStep.pExprList = pList;
-  pRet->retSel.op = TK_ALL;
-  pRet->retSel.pEList = pList;
-  pRet->retSel.pSrc = (SrcList*)&pRet->retSrcList;
   pHash = &(db->aDb[1].pSchema->trigHash);
   assert( sqlite3HashFind(pHash, RETURNING_TRIGGER_NAME)==0 );
   if( sqlite3HashInsert(pHash, RETURNING_TRIGGER_NAME, &pRet->retTrig)
@@ -2147,9 +2169,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){
   /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY
   ** into BTREE_BLOBKEY.
   */
-  if( pParse->addrCrTab ){
+  assert( !pParse->bReturning );
+  if( pParse->u1.addrCrTab ){
     assert( v );
-    sqlite3VdbeChangeP3(v, pParse->addrCrTab, BTREE_BLOBKEY);
+    sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY);
   }
 
   /* Locate the PRIMARY KEY index.  Or, if this table was originally

src/sqliteInt.h

@@ -3425,7 +3425,10 @@ struct Parse {
   Table *pTriggerTab;  /* Table triggers are being coded for */
   Parse *pParentParse; /* Parent parser if this parser is nested */
   AggInfo *pAggList;   /* List of all AggInfo objects */
-  int addrCrTab;       /* Address of OP_CreateBtree opcode on CREATE TABLE */
+  union {
+    int addrCrTab;         /* Address of OP_CreateBtree on CREATE TABLE */
+    Returning *pReturning; /* The RETURNING clause */
+  } u1;
   u32 nQueryLoop;      /* Est number of iterations of a query (10*log2(N)) */
   u32 oldmask;         /* Mask of old.* columns referenced */
   u32 newmask;         /* Mask of new.* columns referenced */
@@ -3647,7 +3650,7 @@ struct TriggerStep {
   char *zTarget;       /* Target table for DELETE, UPDATE, INSERT */
   SrcList *pFrom;      /* FROM clause for UPDATE statement (if any) */
   Expr *pWhere;        /* The WHERE clause for DELETE or UPDATE steps */
-  ExprList *pExprList; /* SET clause for UPDATE */
+  ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */
   IdList *pIdList;     /* Column names for INSERT */
   Upsert *pUpsert;     /* Upsert clauses on an INSERT */
   char *zSpan;         /* Original SQL text of this command */
@@ -3663,8 +3666,8 @@ struct Returning {
   ExprList *pReturnEL;  /* List of expressions to return */
   Trigger retTrig;      /* The transient trigger that implements RETURNING */
   TriggerStep retTStep; /* The trigger step */
-  Select retSel;        /* The SELECT statement that implements RETURNING */
+  int iRetCur;          /* Transient table holding RETURNING results */
-  u64 retSrcList;       /* The empty FROM clause of the SELECT */
+  int nRetCol;          /* Number of in pReturnEL after expansion */
 };
 
 /*

src/trigger.c

@@ -70,6 +70,8 @@ Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){
         pTrig->pNext = pList;
         pList = pTrig;
       }else if( pTrig->op==TK_RETURNING ){
+        assert( pParse->bReturning );
+        assert( &(pParse->u1.pReturning->retTrig) == pTrig );
         pTrig->table = pTab->zName;
         pTrig->pTabSchema = pTab->pSchema;
         pTrig->pNext = pList;
@@ -857,6 +859,55 @@ static ExprList *sqlite3ExpandReturning(
   return pNew;
 }
 
+/*
+** Generate code for the RETURNING trigger.  Unlike other triggers
+** that invoke a subprogram in the bytecode, the code for RETURNING
+** is generated in-line.
+*/
+static void codeReturningTrigger(
+  Parse *pParse,       /* Parse context */
+  Trigger *pTrigger,   /* The trigger step that defines the RETURNING */
+  Table *pTab,         /* The table to code triggers from */
+  int reg              /* The first in an array of registers */
+){
+  Vdbe *v = pParse->pVdbe;
+  ExprList *pNew;
+  Returning *pReturning;
+
+  assert( v!=0 );
+  assert( pParse->bReturning );
+  pReturning = pParse->u1.pReturning;
+  assert( pTrigger == &(pReturning->retTrig) );
+  sqlite3VdbeAddOp0(v, OP_Noop);
+  VdbeComment((v, "RETURNING trigger goes here"));
+  pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab);
+  if( pNew ){
+    pReturning->nRetCol = pNew->nExpr;
+    pReturning->iRetCur = pParse->nTab++;
+    sqlite3ExprListDelete(pParse->db, pNew);
+  }
+
+#if 0
+        pSelect->pEList =
+           sqlite3ExpandReturning(pParse, pList, pParse->pTriggerTab);
+        sqlite3SelectDestInit(&sDest, SRT_Output, 0);
+        pNew = sqlite3SelectDup(db, pSelect, 0);
+        if( pNew ){
+          sqlite3Select(pParse, pNew, &sDest);
+          if( pNew->selFlags & (SF_Aggregate|SF_HasAgg|SF_WinRewrite) ){
+            sqlite3ErrorMsg(pParse, "aggregates not allowed in RETURNING");
+          }
+          sqlite3SelectDelete(db, pNew);
+        }
+        sqlite3ExprListDelete(db, pSelect->pEList);
+        pStep->pSelect->pEList = pList;
+        break;
+      }
+#endif
+}
+
+
+
 /*
 ** Generate VDBE code for the statements inside the body of a single 
 ** trigger.
@@ -928,7 +979,7 @@ static int codeTriggerProgram(
         sqlite3VdbeAddOp0(v, OP_ResetCount);
         break;
       }
-      case TK_SELECT: {
+      default: assert( pStep->op==TK_SELECT ); {
         SelectDest sDest;
         Select *pSelect = sqlite3SelectDup(db, pStep->pSelect, 0);
         sqlite3SelectDestInit(&sDest, SRT_Discard, 0);
@@ -936,26 +987,6 @@ static int codeTriggerProgram(
         sqlite3SelectDelete(db, pSelect);
         break;
       }
-      default: assert( pStep->op==TK_RETURNING ); {
-        Select *pSelect = pStep->pSelect;
-        ExprList *pList = pSelect->pEList;
-        SelectDest sDest;
-        Select *pNew;
-        pSelect->pEList =
-           sqlite3ExpandReturning(pParse, pList, pParse->pTriggerTab);
-        sqlite3SelectDestInit(&sDest, SRT_Output, 0);
-        pNew = sqlite3SelectDup(db, pSelect, 0);
-        if( pNew ){
-          sqlite3Select(pParse, pNew, &sDest);
-          if( pNew->selFlags & (SF_Aggregate|SF_HasAgg|SF_WinRewrite) ){
-            sqlite3ErrorMsg(pParse, "aggregates not allowed in RETURNING");
-          }
-          sqlite3SelectDelete(db, pNew);
-        }
-        sqlite3ExprListDelete(db, pSelect->pEList);
-        pStep->pSelect->pEList = pList;
-        break;
-      }
     } 
   }
 
@@ -1208,7 +1239,7 @@ void sqlite3CodeRowTriggerDirect(
 **   ...            ...
 **   reg+N          OLD.* value of right-most column of pTab
 **   reg+N+1        NEW.rowid
-**   reg+N+2        OLD.* value of left-most column of pTab
+**   reg+N+2        NEW.* value of left-most column of pTab
 **   ...            ...
 **   reg+N+N+1      NEW.* value of right-most column of pTab
 **
@@ -1261,12 +1292,12 @@ void sqlite3CodeRowTrigger(
     if( (p->op==op || (p->bReturning && p->op==TK_INSERT && op==TK_UPDATE))
      && p->tr_tm==tr_tm 
      && checkColumnOverlap(p->pColumns, pChanges)
-     && (sqlite3IsToplevel(pParse) || !p->bReturning)
     ){
-      u8 origOp = p->op;
+      if( !p->bReturning ){
-      p->op = op;
+        sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump);
-      sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump);
+      }else if( sqlite3IsToplevel(pParse) ){
-      p->op = origOp;
+        codeReturningTrigger(pParse, p, pTab, reg);
+      }
     }
   }
 }
@@ -1311,13 +1342,18 @@ u32 sqlite3TriggerColmask(
 
   assert( isNew==1 || isNew==0 );
   for(p=pTrigger; p; p=p->pNext){
-    if( p->op==op && (tr_tm&p->tr_tm)
+    if( p->op==op
+     && (tr_tm&p->tr_tm)
      && checkColumnOverlap(p->pColumns,pChanges)
     ){
-      TriggerPrg *pPrg;
+      if( p->bReturning ){
-      pPrg = getRowTrigger(pParse, p, pTab, orconf);
+        mask = 0xffffffff;
-      if( pPrg ){
+      }else{
-        mask |= pPrg->aColmask[isNew];
+        TriggerPrg *pPrg;
+        pPrg = getRowTrigger(pParse, p, pTab, orconf);
+        if( pPrg ){
+          mask |= pPrg->aColmask[isNew];
+        }
       }
     }
   }