Improved reuse of subqueries associated with IN operators, especially when

the IN operator is duplicated due to predicate push-down.

FossilOrigin-Name: c9a3498113074bbcd9a8c8d30286fef6c6a49ad2c84b90ec0f5a148389d6245c
This commit is contained in:
drh 2024-07-05 09:56:11 +00:00
commit 635c6b8086
8 changed files with 177 additions and 39 deletions

View File

@ -1,5 +1,5 @@
C Add\scomment\susing\sthe\sname\s"predicate\spush-down\soptimization"\sto\swhat\swe\shave\nalso\scalled\s"WHERE-clause\spush\sdown".\s\sNo\schanges\sto\scode.
D 2024-07-04T11:15:32.292
C Improved\sreuse\sof\ssubqueries\sassociated\swith\sIN\soperators,\sespecially\swhen\nthe\sIN\soperator\sis\sduplicated\sdue\sto\spredicate\spush-down.
D 2024-07-05T09:56:11.351
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -705,7 +705,7 @@ F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e
F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
F src/delete.c cb766727c78e715f9fb7ec8a7d03658ed2a3016343ca687acfcec9083cdca500
F src/expr.c d6f614d0c90cf8d72bbd54dbae96040e7feb495a77991fdc1b1c67db68eeb326
F src/expr.c 4d76a31f37302d11c89dd5184a0e1a80a95bd3f8c97e72fa2a18b64cc8672603
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c 852f93c0ef995e0c2b8983059a2b97151c194cc8259e21f5bc2b7ac508348c2a
F src/func.c 1f61e32e7a357e615b5d2e774bee563761fce4f2fd97ecb0f72c33e62a2ada5f
@ -760,7 +760,7 @@ F src/shell.c.in b7d435c137eb323981adff814f172dbaabb9ba504fef17cb11d4681c1633ee1
F src/sqlite.h.in 6c884a87bbf8828562b49272025a1e66e3801a196a58b0bdec87edcd2c9c8fc1
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
F src/sqliteInt.h 039b5309b49399340f260ad3d1e6b6fa13e04a7b587b9daa1e694ce0f13a04a3
F src/sqliteInt.h 5f940c22e14a7434b2fbefa53407372be2eed47b43cb9f4f98c5054c565d63b2
F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728
F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@ -826,10 +826,10 @@ F src/utf.c f23165685a67b4caf8ec08fb274cb3f319103decfb2a980b7cfd55d18dfa855e
F src/util.c 5d1a0134cf4240648d1c6bb5cc8efaca0ea2b5d5c840985aec7e947271f04375
F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104
F src/vdbe.c f941fd8dbd1a9e073866a941e7fbb553f1de2d25b249611005d8c27d5eac8995
F src/vdbe.h c2d78d15112c3fc5ab87f5e8e0b75d2db1c624409de2e858c3d1aafb1650bb4f
F src/vdbe.h c2549a215898a390de6669cfa32adba56f0d7e17ba5a7f7b14506d6fd5f0c36a
F src/vdbeInt.h 949669dfd8a41550d27dcb905b494f2ccde9a2e6c1b0b04daa1227e2e74c2b2c
F src/vdbeapi.c 80235ac380e9467fec1cb0883354d841f2a771976e766995f7e0c77f845406df
F src/vdbeaux.c 6e37cb918506c28fe7657454fcbc2e01e66bfba4164f306c2f075fd5c5fef609
F src/vdbeaux.c 25d685cafe119ff890c94345e884ea558a6b5d823bfa52ba708eb8ff3c70aa71
F src/vdbeblob.c 13f9287b55b6356b4b1845410382d6bede203ceb29ef69388a4a3d007ffacbe5
F src/vdbemem.c 831a244831eaa45335f9ae276b50a7a82ee10d8c46c2c72492d4eb8c98d94d89
F src/vdbesort.c 237840ca1947511fa59bd4e18b9eeae93f2af2468c34d2427b059f896230a547
@ -1283,7 +1283,7 @@ F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
F test/in4.test bb767ec1cfd1730256f0a83219f0acda36bc251b63f8b8bb7d8c7cff17875a4f
F test/in5.test 4fd79c70dfa0681313e8cdca07f5ff0400bdc0e20f808a5c59eaef1e4b48082a
F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b
F test/in7.test 742b18c284cd9a9cd1347d3a8affeee44b8de11e875e91a1d40498c18ba16441
F test/in7.test 9256cdb30dc487f2078bb4bb30f43f2c1ff4d277a9c7c9a14bd1c9510c9c8cae
F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
F test/incrblob2.test a494c9e848560039a23974b9119cfc2cf3ad3bd15cc2694ee6367ae537ef8f1f
F test/incrblob3.test 67621a04b3084113bf38ce03797d70eca012d9d8f948193b8f655df577b0da6f
@ -1518,7 +1518,7 @@ F test/printf.test 685fec5a0c5af2490ab0632775a301554361d674211d690f5bee0a97b0533
F test/printf2.test 3f55c1871a5a65507416076f6eb97e738d5210aeda7595a74ee895f2224cce60
F test/progress.test ebab27f670bd0d4eb9d20d49cef96e68141d92fb
F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc
F test/pushdown.test 9e655df51bc6559608dcc7af89a36f727eff520b4860261ed15a5f95a5ebbcd8
F test/pushdown.test 84b525767442b3695d671f9df59dd91cf0ed8fb24cbbcdc55959f0dadeee8b39
F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca
F test/quick.test 1681febc928d686362d50057c642f77a02c62e57
F test/quickcheck.test a4b7e878cd97e46108291c409b0bf8214f29e18fddd68a42bc5c1375ad1fb80a
@ -2195,8 +2195,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P a204ffc06b468c2edf8f11ccf3de639edd4f8282e69a44ceeb68a4d3a43e77ea
R 7ddaa7e2074acfdf6d0130f12e8505c2
P be77fe701c7b7d3407800e1ed6a4c8be14035831790419cc99362bff41c65b60 99fd34b58a9b09af20f1f1a3c37137ae55159310c8af97806550eeee6adc4fc5
R c2554732fd5d00a4bc9161c1ecfc2ecd
T +closed 99fd34b58a9b09af20f1f1a3c37137ae55159310c8af97806550eeee6adc4fc5
U drh
Z 940dec88e4e92fbd5a212ce2f1e6eedc
Z 4e117d1a68420f4abee82559a7eb0fa4
# Remove this line to create a well-formed Fossil manifest.

View File

@ -1 +1 @@
be77fe701c7b7d3407800e1ed6a4c8be14035831790419cc99362bff41c65b60
c9a3498113074bbcd9a8c8d30286fef6c6a49ad2c84b90ec0f5a148389d6245c

View File

@ -3420,6 +3420,49 @@ void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){
}
}
#ifndef SQLITE_OMIT_SUBQUERY
/*
** Scan all previously generated bytecode looking for an OP_BeginSubrtn
** that is compatible with pExpr. If found, add the y.sub values
** to pExpr and return true. If not found, return false.
*/
static int findCompatibleInRhsSubrtn(
Parse *pParse, /* Parsing context */
Expr *pExpr, /* IN operator with RHS that we want to reuse */
SubrtnSig *pNewSig /* Signature for the IN operator */
){
VdbeOp *pOp, *pEnd;
SubrtnSig *pSig;
Vdbe *v;
if( pNewSig==0 ) return 0;
if( pParse->bHasSubrtn==0 ) return 0;
assert( pExpr->op==TK_IN );
assert( !ExprUseYSub(pExpr) );
assert( ExprUseXSelect(pExpr) );
assert( pExpr->x.pSelect!=0 );
assert( (pExpr->x.pSelect->selFlags & SF_All)==0 );
v = pParse->pVdbe;
assert( v!=0 );
pOp = sqlite3VdbeGetOp(v, 1);
pEnd = sqlite3VdbeGetLastOp(v);
for(; pOp<pEnd; pOp++){
if( pOp->p4type!=P4_SUBRTNSIG ) continue;
assert( pOp->opcode==OP_BeginSubrtn );
pSig = pOp->p4.pSubrtnSig;
assert( pSig!=0 );
if( pNewSig->selId!=pSig->selId ) continue;
if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue;
pExpr->y.sub.iAddr = pSig->iAddr;
pExpr->y.sub.regReturn = pSig->regReturn;
pExpr->iTable = pSig->iTable;
ExprSetProperty(pExpr, EP_Subrtn);
return 1;
}
return 0;
}
#endif /* SQLITE_OMIT_SUBQUERY */
#ifndef SQLITE_OMIT_SUBQUERY
/*
** Generate code that will construct an ephemeral table containing all terms
@ -3469,11 +3512,28 @@ void sqlite3CodeRhsOfIN(
** and reuse it many names.
*/
if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){
/* Reuse of the RHS is allowed */
/* If this routine has already been coded, but the previous code
** might not have been invoked yet, so invoke it now as a subroutine.
/* Reuse of the RHS is allowed
**
** Compute a signature for the RHS of the IN operator to facility
** finding and reusing prior instances of the same IN operator.
*/
if( ExprHasProperty(pExpr, EP_Subrtn) ){
SubrtnSig *pSig = 0;
assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 );
if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){
pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0]));
if( pSig ){
pSig->selId = pExpr->x.pSelect->selId;
pSig->zAff = exprINAffinity(pParse, pExpr);
}
}
/* Check to see if there is a prior materialization of the RHS of
** this IN operator. If there is, then make use of that prior
** materialization rather than recomputing it.
*/
if( ExprHasProperty(pExpr, EP_Subrtn)
|| findCompatibleInRhsSubrtn(pParse, pExpr, pSig)
){
addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
if( ExprUseXSelect(pExpr) ){
ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d",
@ -3485,6 +3545,10 @@ void sqlite3CodeRhsOfIN(
assert( iTab!=pExpr->iTable );
sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable);
sqlite3VdbeJumpHere(v, addrOnce);
if( pSig ){
sqlite3DbFree(pParse->db, pSig->zAff);
sqlite3DbFree(pParse->db, pSig);
}
return;
}
@ -3495,7 +3559,13 @@ void sqlite3CodeRhsOfIN(
pExpr->y.sub.regReturn = ++pParse->nMem;
pExpr->y.sub.iAddr =
sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1;
if( pSig ){
pSig->iAddr = pExpr->y.sub.iAddr;
pSig->regReturn = pExpr->y.sub.regReturn;
pSig->iTable = iTab;
sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG);
pParse->bHasSubrtn = 1;
}
addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
}

View File

@ -3834,6 +3834,7 @@ struct Parse {
u8 prepFlags; /* SQLITE_PREPARE_* flags */
u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */
u8 bHasWith; /* True if statement contains WITH */
u8 bHasSubrtn; /* True if any P4_SUBRTNSIG has been set */
#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST)
u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */
#endif

View File

@ -32,6 +32,19 @@ typedef struct Vdbe Vdbe;
*/
typedef struct sqlite3_value Mem;
typedef struct SubProgram SubProgram;
typedef struct SubrtnSig SubrtnSig;
/*
** A signature for a reusable subroutine that materializes the RHS of
** an IN operator.
*/
struct SubrtnSig {
int selId; /* SELECT-id for the SELECT statement on the RHS */
char *zAff; /* Affinity of the overall IN expression */
int iTable; /* Ephemeral table generated by the subroutine */
int iAddr; /* Subroutine entry address */
int regReturn; /* Register used to hold return address */
};
/*
** A single instruction of the virtual machine has an opcode
@ -60,6 +73,7 @@ struct VdbeOp {
u32 *ai; /* Used when p4type is P4_INTARRAY */
SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */
Table *pTab; /* Used when p4type is P4_TABLE */
SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */
#ifdef SQLITE_ENABLE_CURSOR_HINTS
Expr *pExpr; /* Used when p4type is P4_EXPR */
#endif
@ -127,6 +141,7 @@ typedef struct VdbeOpList VdbeOpList;
#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */
#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */
#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */
#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */
/* Error message codes for OP_Halt */
#define P5_ConstraintNotNull 1

View File

@ -1413,6 +1413,12 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){
if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4);
break;
}
case P4_SUBRTNSIG: {
SubrtnSig *pSig = (SubrtnSig*)p4;
sqlite3DbFree(db, pSig->zAff);
sqlite3DbFree(db, pSig);
break;
}
}
}
@ -1992,6 +1998,11 @@ char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){
zP4 = pOp->p4.pTab->zName;
break;
}
case P4_SUBRTNSIG: {
SubrtnSig *pSig = pOp->p4.pSubrtnSig;
sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff);
break;
}
default: {
zP4 = pOp->p4.z;
}

View File

@ -137,5 +137,61 @@ do_execsql_test 2.1 {
SELECT b FROM t1 WHERE a IN (1,2,3) ORDER BY b ASC NULLS LAST;
} {one three {}}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE TABLE x1(a);
INSERT INTO x1 VALUES(1), (2), (3);
CREATE TABLE x2(b);
INSERT INTO x2 VALUES(4), (5), (6);
CREATE TABLE t1(u);
INSERT INTO t1 VALUES(1), (2), (3), (4), (5), (6);
CREATE VIEW v1 AS SELECT u FROM t1 WHERE u IN (
SELECT a FROM x1
);
CREATE VIEW v2 AS SELECT u FROM t1 WHERE u IN (
SELECT b FROM x2
);
}
do_execsql_test 3.1 {
SELECT * FROM v1
} {
1 2 3
}
do_execsql_test 3.2 {
SELECT * FROM v2
} {
4 5 6
}
do_execsql_test 3.3 {
SELECT * FROM v2
UNION ALL
SELECT * FROM v1
} {
4 5 6
1 2 3
}
do_execsql_test 3.4 {
WITH w1 AS (
SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3
),
w2 AS (
SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6
)
SELECT * FROM v1 WHERE u IN w1
UNION ALL
SELECT * FROM v2 WHERE u IN w2
} {
1 2 3 4 5 6
}
finish_test

View File

@ -279,13 +279,9 @@ do_eqp_test 6.1 {
| | `--CREATE BLOOM FILTER
| `--UNION ALL
| |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
| `--LIST SUBQUERY xxxxxx
| |--SCAN k
| `--CREATE BLOOM FILTER
| `--REUSE LIST SUBQUERY xxxxxx
|--SEARCH t0
`--LIST SUBQUERY xxxxxx
|--SCAN k
`--CREATE BLOOM FILTER
`--REUSE LIST SUBQUERY xxxxxx
}
# ^^^^--- The key feature above is that the SEARCH for each subquery
# uses all three fields of the index w, x, and y. Prior to the push-down
@ -307,17 +303,9 @@ do_eqp_test 6.2 {
| | `--CREATE BLOOM FILTER
| `--UNION ALL
| |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
| `--LIST SUBQUERY xxxxxx
| |--CO-ROUTINE v1
| | `--SCAN 3 CONSTANT ROWS
| |--SCAN v1
| `--CREATE BLOOM FILTER
| `--REUSE LIST SUBQUERY xxxxxx
|--SEARCH t0
`--LIST SUBQUERY xxxxxx
|--CO-ROUTINE v1
| `--SCAN 3 CONSTANT ROWS
|--SCAN v1
`--CREATE BLOOM FILTER
`--REUSE LIST SUBQUERY xxxxxx
}
do_eqp_test 6.3 {
SELECT max(z) FROM t0 WHERE w=123 AND x IN k1 AND y BETWEEN 44 AND 55;
@ -332,13 +320,9 @@ do_eqp_test 6.3 {
| | `--CREATE BLOOM FILTER
| `--UNION ALL
| |--SEARCH t02 USING INDEX t02x (w=? AND x=? AND y>? AND y<?)
| `--LIST SUBQUERY xxxxxx
| |--SCAN k1
| `--CREATE BLOOM FILTER
| `--REUSE LIST SUBQUERY xxxxxx
|--SEARCH t0
`--LIST SUBQUERY xxxxxx
|--SCAN k1
`--CREATE BLOOM FILTER
`--REUSE LIST SUBQUERY xxxxxx
}
finish_test