Improvements to the OP_SeekScan optimization. Added the
SQLITE_TESTCTRL_TUNE test-control. FossilOrigin-Name: 9a2ab6092d644fc3effc4ab4aca33efb2b4dd4213b2154d518a3e87eec4d3983
This commit is contained in:
drh
commit
946b65793c
25
manifest
25
manifest
@ -1,5 +1,5 @@
|
||||
C Add\sthe\s(missing)\sdependency\sof\sregexp.c\sto\sthe\sshell.c\srule\sin\sthe\smakefiles.\nThis\sdependency\sshould\shave\sbeen\sadded\sby\s[5d4535bfb603d7c8]\sbut\swas\smissed.
|
||||
D 2021-06-04T16:17:38.971
|
||||
C Improvements\sto\sthe\sOP_SeekScan\soptimization.\s\sAdded\sthe\nSQLITE_TESTCTRL_TUNE\stest-control.
|
||||
D 2021-06-04T17:47:08.529
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -508,7 +508,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71
|
||||
F src/insert.c 2189e0e596010a0dc5405d9f14f78db1ee2fa71138c931f5b6ea96610b95bfc1
|
||||
F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
|
||||
F src/loadext.c 8c9c8cd2bd8eecdb06d9b6e89de7e9e65bae45cc8fc33609cc74023a5c296067
|
||||
F src/main.c 2b2044221a10e7c5f49a98f50563ce9a944511241ba5946d1754dcc2a7437a07
|
||||
F src/main.c 7a1b7017af16977b298ef5a0b824b1153f1c878f94885a098620c3c1b1078ce9
|
||||
F src/malloc.c cbc93cdd429c4594912017d92ab656e2579aca64dbd1c6888551275bed46f25b
|
||||
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
|
||||
F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de
|
||||
@ -545,11 +545,11 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
|
||||
F src/resolve.c 35630effd4d16d2373caa41bae40a3d71f853f3ad0cb4f572f2ed4b8c350c1e9
|
||||
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
|
||||
F src/select.c 2f4429e0b0ca031e360489c45dca5dfa06cd85480e12c0d1360b91e1c8e7f7a2
|
||||
F src/shell.c.in ab37c8869390a7624e594cb1459277319d16521d4644c42f4ee6a3866cf5f0cd
|
||||
F src/sqlite.h.in 5c950066775ca9efdaa49077c05d38d0bef6418f3bd07d2dce0210f1d2f3c326
|
||||
F src/shell.c.in a4bc0e2ba9be798e293790f354dcc0099c6370127eec18cf49cb161b9dae2fbc
|
||||
F src/sqlite.h.in 7f184776cfd85900905420ea3c62e44929ba348ddc015bc5933835ee0529188e
|
||||
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
|
||||
F src/sqlite3ext.h 61b38c073d5e1e96a3d45271b257aef27d0d13da2bea5347692ae579475cd95e
|
||||
F src/sqliteInt.h 6d0c49ed218f9f6ffabfdc8bed1a5de89f61e70c96a62156aeebeb20c1be9bf0
|
||||
F src/sqliteInt.h 61f717ac4242af975c6d8abc11a3ada6153ad5bfb7ce08ef8a443c01e13c8031
|
||||
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
|
||||
F src/status.c 4b8bc2a6905163a38b739854a35b826c737333fab5b1f8e03fa7eb9a4799c4c1
|
||||
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
|
||||
@ -616,7 +616,7 @@ F src/upsert.c df8f1727d62b5987c4fd302cd4d7c0c84ae57cd65683c5a34a740dfe24039235
|
||||
F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0
|
||||
F src/util.c 41c7a72da1df47864faa378a1c720b38adb288c6838cb6be5594511b6287a048
|
||||
F src/vacuum.c 492422c1463c076473bae1858799c7a0a5fe87a133d1223239447c422cd26286
|
||||
F src/vdbe.c c42799bcd6cbeaf33a1767dde3ba926370afec4f82ba7ca92327a099251f62ec
|
||||
F src/vdbe.c a51f4947380b0561ef70730e1144bf3837216cfea61a59a498149d47eeb09943
|
||||
F src/vdbe.h 25dabb25c7e157b84e59260cfb5b466c3ac103ede9f36f4db371332c47601abe
|
||||
F src/vdbeInt.h 58980223a32495ad059d10581b83e133abdc77248b1bab85c080cab8a13bd819
|
||||
F src/vdbeapi.c d9e99daf59fec928986838b3389a7337e82fec6b3b5de30206cb99fb4661b94e
|
||||
@ -631,7 +631,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
|
||||
F src/wal.c c8ec20a1ca161d5635a4f19c2a4efec2e006e19a8a61f272bf6bce1c80ab7436
|
||||
F src/wal.h c3aa7825bfa2fe0d85bef2db94655f99870a285778baa36307c0a16da32b226a
|
||||
F src/walker.c 7342becedf3f8a26f9817f08436bdf8b56ad69af83705f6b9320a0ad3092c2ac
|
||||
F src/where.c db781a6fb79cd68155a4c33638d4a2885e45ad57f8a8e99e95778e259c2aee8f
|
||||
F src/where.c dc062832c30aa6c5f17805c1235b2b621cd9db506412d5d3ef0799d26ae3396f
|
||||
F src/whereInt.h 9248161dd004f625ce5d3841ca9b99fed3fc8d61522cf76340fc5217dbe1375b
|
||||
F src/wherecode.c 9f1f65d11437b25cd0a1497a170514c785f19ce6ad9d3e6fc73719cb5a49012f
|
||||
F src/whereexpr.c 5a9c9f5d2dac4bcdcaae3035034b4667523f731df228e0bb1d4efc669efa9da5
|
||||
@ -1074,7 +1074,7 @@ F test/imposter1.test c3f1db2d3db2c24611a6596a3fc0ffc14f1466c8
|
||||
F test/in.test 688ed2011d922d83141a45af431601738674a4c0bdde34b6351f688b82a169b3
|
||||
F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
|
||||
F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
|
||||
F test/in4.test cd08375ba470b248c3dc4ab30fd5dbcb682deafe3862d8249ba0ad6a11dd25ea
|
||||
F test/in4.test 3ce7e79419bcd7942744bee2394f125b45513186819ffe771c78d04caa547d54
|
||||
F test/in5.test b32ce7f4a93f44c5dee94af16886d922cc16ebe33c8e1765c73d4049d0f4b40f
|
||||
F test/in6.test f5f40d6816a8bb7c784424b58a10ac38efb76ab29127a2c17399e0cbeeda0e4b
|
||||
F test/incrblob.test c9b96afc292aeff43d6687bcb09b0280aa599822
|
||||
@ -1918,7 +1918,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 83aca2d8704e51eeed5652b1506c6fc883c0397728cda456f229369788b8608a
|
||||
R f21bb0c9aca0f2f99e6cb97aa8cc41b9
|
||||
P c93be7e66039e4c27b86f49d29f5c653cc6c02f6cbc6ad02d8bb1d32d0240863 677e645e69e1f06487c26da6671fc03f0fb89a0f8e0d35712e6bdcf7279bdfc4
|
||||
R 7f570944acb13c224bf3daa6354a37ca
|
||||
T +closed 677e645e69e1f06487c26da6671fc03f0fb89a0f8e0d35712e6bdcf7279bdfc4
|
||||
U drh
|
||||
Z d8be7c9aa67f8410dd7011c1d63c866b
|
||||
Z 1487fd9719459ccfb4970df5beff8402
|
||||
|
||||
@ -1 +1 @@
|
||||
c93be7e66039e4c27b86f49d29f5c653cc6c02f6cbc6ad02d8bb1d32d0240863
|
||||
9a2ab6092d644fc3effc4ab4aca33efb2b4dd4213b2154d518a3e87eec4d3983
|
||||
30
src/main.c
30
src/main.c
@ -4283,6 +4283,36 @@ int sqlite3_test_control(int op, ...){
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
#ifdef SQLITE_DEBUG
|
||||
/* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue)
|
||||
**
|
||||
** If "id" is an integer between 1 and SQLITE_NTUNE then set the value
|
||||
** of the id-th tuning parameter to *piValue. If "id" is between -1
|
||||
** and -SQLITE_NTUNE, then write the current value of the (-id)-th
|
||||
** tuning parameter into *piValue.
|
||||
**
|
||||
** Tuning parameters are for use during transient development builds,
|
||||
** to help find the best values for constants in the query planner.
|
||||
** Access tuning parameters using the Tuning(ID) macro. Set the
|
||||
** parameters in the CLI using ".testctrl tune ID VALUE".
|
||||
**
|
||||
** Transient use only. Tuning parameters should not be used in
|
||||
** checked-in code.
|
||||
*/
|
||||
case SQLITE_TESTCTRL_TUNE: {
|
||||
int id = va_arg(ap, int);
|
||||
int *piValue = va_arg(ap, int*);
|
||||
if( id>0 && id<=SQLITE_NTUNE ){
|
||||
Tuning(id) = *piValue;
|
||||
}else if( id<0 && id>=-SQLITE_NTUNE ){
|
||||
*piValue = Tuning(-id);
|
||||
}else{
|
||||
rc = SQLITE_NOTFOUND;
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
va_end(ap);
|
||||
#endif /* SQLITE_UNTESTABLE */
|
||||
|
||||
@ -9974,6 +9974,7 @@ static int do_meta_command(char *zLine, ShellState *p){
|
||||
{ "prng_save", SQLITE_TESTCTRL_PRNG_SAVE, "" },
|
||||
{ "prng_seed", SQLITE_TESTCTRL_PRNG_SEED, "SEED ?db?" },
|
||||
{ "seek_count", SQLITE_TESTCTRL_SEEK_COUNT, "" },
|
||||
{ "tune", SQLITE_TESTCTRL_TUNE, "ID VALUE" },
|
||||
};
|
||||
int testctrl = -1;
|
||||
int iCtrl = -1;
|
||||
@ -10118,11 +10119,40 @@ static int do_meta_command(char *zLine, ShellState *p){
|
||||
}
|
||||
|
||||
#ifdef YYCOVERAGE
|
||||
case SQLITE_TESTCTRL_PARSER_COVERAGE:
|
||||
case SQLITE_TESTCTRL_PARSER_COVERAGE: {
|
||||
if( nArg==2 ){
|
||||
sqlite3_test_control(testctrl, p->out);
|
||||
isOk = 3;
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#ifdef SQLITE_DEBUG
|
||||
case SQLITE_TESTCTRL_TUNE: {
|
||||
if( nArg==4 ){
|
||||
int id = (int)integerValue(azArg[2]);
|
||||
int val = (int)integerValue(azArg[3]);
|
||||
sqlite3_test_control(testctrl, id, &val);
|
||||
isOk = 3;
|
||||
}else if( nArg==3 ){
|
||||
int id = (int)integerValue(azArg[2]);
|
||||
sqlite3_test_control(testctrl, -id, &rc2);
|
||||
isOk = 1;
|
||||
}else if( nArg==2 ){
|
||||
int id = 1;
|
||||
while(1){
|
||||
int val = 0;
|
||||
rc2 = sqlite3_test_control(testctrl, -id, &val);
|
||||
if( rc2!=SQLITE_OK ) break;
|
||||
if( id>1 ) utf8_printf(p->out, " ");
|
||||
utf8_printf(p->out, "%d: %d", id, val);
|
||||
id++;
|
||||
}
|
||||
if( id>1 ) utf8_printf(p->out, "\n");
|
||||
isOk = 3;
|
||||
}
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@ -7811,7 +7811,8 @@ int sqlite3_test_control(int op, ...);
|
||||
#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29
|
||||
#define SQLITE_TESTCTRL_SEEK_COUNT 30
|
||||
#define SQLITE_TESTCTRL_TRACEFLAGS 31
|
||||
#define SQLITE_TESTCTRL_LAST 31 /* Largest TESTCTRL */
|
||||
#define SQLITE_TESTCTRL_TUNE 32
|
||||
#define SQLITE_TESTCTRL_LAST 32 /* Largest TESTCTRL */
|
||||
|
||||
/*
|
||||
** CAPI3REF: SQL Keyword Checking
|
||||
|
||||
@ -1731,6 +1731,7 @@ struct sqlite3 {
|
||||
#define SQLITE_SkipScan 0x00004000 /* Skip-scans */
|
||||
#define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */
|
||||
#define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */
|
||||
#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */
|
||||
#define SQLITE_AllOpts 0xffffffff /* All optimizations */
|
||||
|
||||
/*
|
||||
@ -3740,6 +3741,22 @@ typedef struct {
|
||||
#define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */
|
||||
#define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */
|
||||
|
||||
/* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled
|
||||
** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning
|
||||
** parameters are for temporary use during development, to help find
|
||||
** optimial values for parameters in the query planner. The should not
|
||||
** be used on trunk check-ins. They are a temporary mechanism available
|
||||
** for transient development builds only.
|
||||
**
|
||||
** Tuning parameters are numbered starting with 1.
|
||||
*/
|
||||
#define SQLITE_NTUNE 6 /* Should be zero for all trunk check-ins */
|
||||
#ifdef SQLITE_DEBUG
|
||||
# define Tuning(X) (sqlite3Config.aTune[(X)-1])
|
||||
#else
|
||||
# define Tuning(X) 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
** Structure containing global configuration data for the SQLite library.
|
||||
**
|
||||
@ -3804,6 +3821,10 @@ struct Sqlite3Config {
|
||||
int iOnceResetThreshold; /* When to reset OP_Once counters */
|
||||
u32 szSorterRef; /* Min size in bytes to use sorter-refs */
|
||||
unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */
|
||||
/* vvvv--- must be last ---vvv */
|
||||
#ifdef SQLITE_DEBUG
|
||||
sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */
|
||||
#endif
|
||||
};
|
||||
|
||||
/*
|
||||
|
||||
@ -4471,8 +4471,7 @@ case OP_SeekScan: {
|
||||
/* pOp->p2 points to the first instruction past the OP_IdxGT that
|
||||
** follows the OP_SeekGE. */
|
||||
assert( pOp->p2>=(int)(pOp-aOp)+2 );
|
||||
assert( aOp[pOp->p2-1].opcode==OP_IdxGT || aOp[pOp->p2-1].opcode==OP_IdxGE );
|
||||
testcase( aOp[pOp->p2-1].opcode==OP_IdxGE );
|
||||
assert( aOp[pOp->p2-1].opcode==OP_IdxGT );
|
||||
assert( pOp[1].p1==aOp[pOp->p2-1].p1 );
|
||||
assert( pOp[1].p2==aOp[pOp->p2-1].p2 );
|
||||
assert( pOp[1].p3==aOp[pOp->p2-1].p3 );
|
||||
|
||||
23
src/where.c
23
src/where.c
@ -2597,7 +2597,7 @@ static int whereLoopAddBtreeIndex(
|
||||
nIn = sqlite3LogEst(pExpr->x.pList->nExpr);
|
||||
}
|
||||
if( pProbe->hasStat1 && rLogSize>=10 ){
|
||||
LogEst M, logK, safetyMargin;
|
||||
LogEst M, logK, x;
|
||||
/* Let:
|
||||
** N = the total number of rows in the table
|
||||
** K = the number of entries on the RHS of the IN operator
|
||||
@ -2620,16 +2620,25 @@ static int whereLoopAddBtreeIndex(
|
||||
*/
|
||||
M = pProbe->aiRowLogEst[saved_nEq];
|
||||
logK = estLog(nIn);
|
||||
safetyMargin = 10; /* TUNING: extra weight for indexed IN */
|
||||
if( M + logK + safetyMargin < nIn + rLogSize ){
|
||||
/* TUNING v----- 10 to bias toward indexed IN */
|
||||
x = M + logK + 10 - (nIn + rLogSize);
|
||||
if( x>=0 ){
|
||||
WHERETRACE(0x40,
|
||||
("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n",
|
||||
saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
|
||||
("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) "
|
||||
"prefers indexed lookup\n",
|
||||
saved_nEq, M, logK, nIn, rLogSize, x));
|
||||
}else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){
|
||||
WHERETRACE(0x40,
|
||||
("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
|
||||
" nInMul=%d) prefers skip-scan\n",
|
||||
saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
|
||||
pNew->wsFlags |= WHERE_IN_SEEKSCAN;
|
||||
}else{
|
||||
WHERETRACE(0x40,
|
||||
("IN operator preferred on column %d of \"%s\" (%d>=%d)\n",
|
||||
saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize));
|
||||
("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d"
|
||||
" nInMul=%d) prefers normal scan\n",
|
||||
saved_nEq, M, logK, nIn, rLogSize, x, nInMul));
|
||||
continue;
|
||||
}
|
||||
}
|
||||
pNew->wsFlags |= WHERE_COLUMN_IN;
|
||||
|
||||
@ -392,5 +392,54 @@ do_execsql_test 8.3 {
|
||||
SELECT t1.x FROM t2 CROSS JOIN t1 WHERE t2.z = t1.y AND t1.x IN (111, 222);
|
||||
} {222 111}
|
||||
|
||||
# 2021-06-02 forum post https://sqlite.org/forum/forumpost/b4fcb8a598
|
||||
# OP_SeekScan changes from check-in 4a43430fd23f8835 on 2020-09-30 causes
|
||||
# performance regression.
|
||||
#
|
||||
reset_db
|
||||
do_execsql_test 9.0 {
|
||||
CREATE TABLE node(node_id INTEGER PRIMARY KEY);
|
||||
CREATE TABLE edge(node_from INT, node_to INT);
|
||||
CREATE TABLE sub_nodes(node_id INTEGER PRIMARY KEY);
|
||||
CREATE INDEX edge_from_to ON edge(node_from,node_to);
|
||||
CREATE INDEX edge_to_from ON edge(node_to,node_from);
|
||||
ANALYZE;
|
||||
DELETE FROM sqlite_stat1;
|
||||
INSERT INTO sqlite_stat1 VALUES
|
||||
('sub_nodes',NULL,'1000000'),
|
||||
('edge','edge_to_from','20000000 2 2'),
|
||||
('edge','edge_from_to','20000000 2 2'),
|
||||
('node',NULL,'10000000');
|
||||
ANALYZE sqlite_schema;
|
||||
} {}
|
||||
do_eqp_test 9.1 {
|
||||
SELECT count(*) FROM edge
|
||||
WHERE node_from IN sub_nodes AND node_to IN sub_nodes;
|
||||
} {
|
||||
QUERY PLAN
|
||||
|--SEARCH edge USING COVERING INDEX edge_to_from (node_to=?)
|
||||
|--USING ROWID SEARCH ON TABLE sub_nodes FOR IN-OPERATOR
|
||||
`--USING ROWID SEARCH ON TABLE sub_nodes FOR IN-OPERATOR
|
||||
}
|
||||
# ^^^^^ the key to the above is that the index should only use a single
|
||||
# term (node_to=?), not two terms (node_to=? AND node_from=).
|
||||
|
||||
# dbsqlfuzz case
|
||||
#
|
||||
reset_db
|
||||
do_execsql_test 10.0 {
|
||||
CREATE TABLE t1(a,b,c,d,PRIMARY KEY(a,b,c)) WITHOUT ROWID;
|
||||
INSERT INTO t1(a,b,c,d) VALUES
|
||||
(0,-2,2,3),
|
||||
(0,2,3,4),
|
||||
(0,5,8,10),
|
||||
(1,7,11,13);
|
||||
ANALYZE sqlite_schema;
|
||||
INSERT INTO sqlite_stat1 VALUES('t1','t1','10 3 2 1');
|
||||
ANALYZE sqlite_schema;
|
||||
PRAGMA reverse_unordered_selects(1);
|
||||
SELECT d FROM t1 WHERE 0=a AND b IN (-17,-4,-3,1,5,25,7798);
|
||||
} {10}
|
||||
|
||||
|
||||
finish_test
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user