Optimize the restarting of an RBU vacuum.

FossilOrigin-Name: 82062351a62f4ccc438a2b47a22ee581bd030dec952fecd610f360a25002a5ea
This commit is contained in:
dan 2019-05-07 16:37:37 +00:00
commit f4f86f3fca
6 changed files with 394 additions and 27 deletions

View File

@ -89,16 +89,16 @@ proc step_rbu_legacy {target rbu} {
proc do_rbu_vacuum_test {tn step {statedb state.db}} {
forcedelete $statedb
if {$statedb=="" && $step==1} breakpoint
uplevel [list do_test $tn.1 [string map [list %state% $statedb] {
if {$step==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
uplevel [list do_test $tn.1 [string map [list %state% $statedb %step% $step] {
if {%step%==0} { sqlite3rbu_vacuum rbu test.db {%state%}}
while 1 {
if {$step==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
if {%step%==1} { sqlite3rbu_vacuum rbu test.db {%state%}}
set state [rbu state]
check_prestep_state test.db $state
set rc [rbu step]
check_poststep_state $rc test.db $state
if {$rc!="SQLITE_OK"} break
if {$step==1} { rbu close }
if {%step%==1} { rbu close }
}
rbu close
}] {SQLITE_DONE}]

View File

@ -80,6 +80,10 @@ foreach {tn without_rowid a b c d} {
set step 0
do_rbu_vacuum_test $tn.1.5 0
do_test $tn.1.6 {
execsql { PRAGMA integrity_check }
} {ok}
}]
}

107
ext/rbu/rbuvacuum4.test Normal file
View File

@ -0,0 +1,107 @@
# 2019 Jan 3
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# This file contains tests for the RBU module. More specifically, it
# contains tests to ensure that the sqlite3rbu_vacuum() API works as
# expected.
#
source [file join [file dirname [info script]] rbu_common.tcl]
set testprefix rbuvacuum4
set step 1
do_execsql_test 1.0 {
CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID;
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
INSERT INTO t1 VALUES(7, 8, 9);
}
do_rbu_vacuum_test 1.1 1
#-------------------------------------------------------------------------
reset_db
do_execsql_test 2.0 {
CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b, c)) WITHOUT ROWID;
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
INSERT INTO t1 VALUES(7, 8, 9);
}
do_rbu_vacuum_test 2.1 1
do_execsql_test 2.2 {
SELECT * FROM t1;
} {1 2 3 4 5 6 7 8 9}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 3.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 oN t1(b, c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
INSERT INTO t1 VALUES(7, 8, 9);
}
do_rbu_vacuum_test 3.1 1
do_execsql_test 3.2 {
SELECT * FROM t1;
} {1 2 3 4 5 6 7 8 9}
#-------------------------------------------------------------------------
reset_db
do_execsql_test 4.0 {
CREATE TABLE x1(a, b, c, d, PRIMARY KEY(c, b)) WITHOUT ROWID;
INSERT INTO x1 VALUES(1, 1, 1, 1);
INSERT INTO x1 VALUES(1, 1, 2, 1);
INSERT INTO x1 VALUES(1, 2, 2, 1);
INSERT INTO x1 VALUES(NULL, 2, 3, NULL);
INSERT INTO x1 VALUES(NULL, 2, 4, NULL);
INSERT INTO x1 VALUES(NULL, 2, 5, NULL);
CREATE INDEX x1ad ON x1(d, a);
CREATE INDEX x1null ON x1(d, a) WHERE d>15;
}
do_rbu_vacuum_test 4.1.1 1
do_execsql_test 4.2 {
SELECT count(*) fROM x1
} 6
do_rbu_vacuum_test 4.1.2 0
#-------------------------------------------------------------------------
reset_db
do_execsql_test 5.0 {
CREATE TABLE "a b c"(a, "b b" PRIMARY KEY, "c c");
CREATE INDEX abc1 ON "a b c"(a, "c c");
INSERT INTO "a b c" VALUES(NULL, 'a', NULL);
INSERT INTO "a b c" VALUES(NULL, 'b', NULL);
INSERT INTO "a b c" VALUES(NULL, 'c', NULL);
INSERT INTO "a b c" VALUES(1, 2, 3);
INSERT INTO "a b c" VALUES(3, 9, 1);
INSERT INTO "a b c" VALUES('aaa', 'bbb', 'ccc');
CREATE TABLE x(a);
INSERT INTO x VALUES('a'), ('b'), ('d');
CREATE UNIQUE INDEX y ON x(a);
}
do_rbu_vacuum_test 5.1 1
finish_test

View File

@ -930,7 +930,8 @@ static void rbuTargetNameFunc(
zIn = (const char*)sqlite3_value_text(argv[0]);
if( zIn ){
if( rbuIsVacuum(p) ){
if( argc==1 || 0==sqlite3_value_int(argv[1]) ){
assert( argc==2 );
if( 0==sqlite3_value_int(argv[1]) ){
sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC);
}
}else{
@ -1381,7 +1382,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){
}
pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc);
pIter->abTblPk[iOrder] = (iPk!=0);
assert( iPk>=0 );
pIter->abTblPk[iOrder] = (u8)iPk;
pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0);
iOrder++;
}
@ -1416,6 +1418,218 @@ static char *rbuObjIterGetCollist(
return zList;
}
/*
** Return a comma separated list of the quoted PRIMARY KEY column names,
** in order, for the current table. Before each column name, add the text
** zPre. After each column name, add the zPost text. Use zSeparator as
** the separator text (usually ", ").
*/
static char *rbuObjIterGetPkList(
sqlite3rbu *p, /* RBU object */
RbuObjIter *pIter, /* Object iterator for column names */
const char *zPre, /* Before each quoted column name */
const char *zSeparator, /* Separator to use between columns */
const char *zPost /* After each quoted column name */
){
int iPk = 1;
char *zRet = 0;
const char *zSep = "";
while( 1 ){
int i;
for(i=0; i<pIter->nTblCol; i++){
if( (int)pIter->abTblPk[i]==iPk ){
const char *zCol = pIter->azTblCol[i];
zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost);
zSep = zSeparator;
break;
}
}
if( i==pIter->nTblCol ) break;
iPk++;
}
return zRet;
}
/*
** This function is called as part of restarting an RBU vacuum within
** stage 1 of the process (while the *-oal file is being built) while
** updating a table (not an index). The table may be a rowid table or
** a WITHOUT ROWID table. It queries the target database to find the
** largest key that has already been written to the target table and
** constructs a WHERE clause that can be used to extract the remaining
** rows from the source table. For a rowid table, the WHERE clause
** is of the form:
**
** "WHERE _rowid_ > ?"
**
** and for WITHOUT ROWID tables:
**
** "WHERE (key1, key2) > (?, ?)"
**
** Instead of "?" placeholders, the actual WHERE clauses created by
** this function contain literal SQL values.
*/
static char *rbuVacuumTableStart(
sqlite3rbu *p, /* RBU handle */
RbuObjIter *pIter, /* RBU iterator object */
int bRowid, /* True for a rowid table */
const char *zWrite /* Target table name prefix */
){
sqlite3_stmt *pMax = 0;
char *zRet = 0;
if( bRowid ){
p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
sqlite3_mprintf(
"SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl
)
);
if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0);
zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax);
}
rbuFinalize(p, pMax);
}else{
char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC");
char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")");
char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", "");
if( p->rc==SQLITE_OK ){
p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg,
sqlite3_mprintf(
"SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1",
zSelect, zWrite, pIter->zTbl, zOrder
)
);
if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){
const char *zVal = (const char*)sqlite3_column_text(pMax, 0);
zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal);
}
rbuFinalize(p, pMax);
}
sqlite3_free(zOrder);
sqlite3_free(zSelect);
sqlite3_free(zList);
}
return zRet;
}
/*
** This function is called as part of restating an RBU vacuum when the
** current operation is writing content to an index. If possible, it
** queries the target index b-tree for the largest key already written to
** it, then composes and returns an expression that can be used in a WHERE
** clause to select the remaining required rows from the source table.
** It is only possible to return such an expression if:
**
** * The index contains no DESC columns, and
** * The last key written to the index before the operation was
** suspended does not contain any NULL values.
**
** The expression is of the form:
**
** (index-field1, index-field2, ...) > (?, ?, ...)
**
** except that the "?" placeholders are replaced with literal values.
**
** If the expression cannot be created, NULL is returned. In this case,
** the caller has to use an OFFSET clause to extract only the required
** rows from the sourct table, just as it does for an RBU update operation.
*/
char *rbuVacuumIndexStart(
sqlite3rbu *p, /* RBU handle */
RbuObjIter *pIter /* RBU iterator object */
){
char *zOrder = 0;
char *zLhs = 0;
char *zSelect = 0;
char *zVector = 0;
char *zRet = 0;
int bFailed = 0;
if( p->rc==SQLITE_OK ){
const char *zSep = "";
int iCol = 0;
sqlite3_stmt *pXInfo = 0;
p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg,
sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx)
);
while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){
int iCid = sqlite3_column_int(pXInfo, 1);
const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4);
const char *zCol;
if( sqlite3_column_int(pXInfo, 3) ){
bFailed = 1;
break;
}
if( iCid<0 ){
if( pIter->eType==RBU_PK_IPK ){
int i;
for(i=0; pIter->abTblPk[i]==0; i++);
assert( i<pIter->nTblCol );
zCol = pIter->azTblCol[i];
}else{
zCol = "_rowid_";
}
}else{
zCol = pIter->azTblCol[iCid];
}
zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q",
zLhs, zSep, zCol, zCollate
);
zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC",
zOrder, zSep, iCol, zCol, zCollate
);
zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")",
zSelect, zSep, iCol, zCol
);
zSep = ", ";
iCol++;
}
rbuFinalize(p, pXInfo);
}
if( bFailed ) goto index_start_out;
if( p->rc==SQLITE_OK ){
int iCol;
sqlite3_stmt *pSel = 0;
if( p->rc==SQLITE_OK ){
p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg,
sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1",
zSelect, pIter->zTbl, zOrder
)
);
}
if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){
const char *zSep = "";
for(iCol=0; iCol<pIter->nCol; iCol++){
const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol);
if( zQuoted[0]=='N' ){
bFailed = 1;
break;
}
zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted);
zSep = ", ";
}
if( !bFailed ){
zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector);
}
}
rbuFinalize(p, pSel);
}
index_start_out:
sqlite3_free(zOrder);
sqlite3_free(zSelect);
sqlite3_free(zVector);
sqlite3_free(zLhs);
return zRet;
}
/*
** This function is used to create a SELECT list (the list of SQL
** expressions that follows a SELECT keyword) for a SELECT statement
@ -2092,12 +2306,24 @@ static int rbuObjIterPrepareAll(
if( p->rc==SQLITE_OK ){
char *zSql;
if( rbuIsVacuum(p) ){
char *zStart = 0;
if( nOffset ){
zStart = rbuVacuumIndexStart(p, pIter);
if( zStart ){
sqlite3_free(zLimit);
zLimit = 0;
}
}
zSql = sqlite3_mprintf(
"SELECT %s, 0 AS rbu_control FROM '%q' %s ORDER BY %s%s",
"SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s",
zCollist,
pIter->zDataTbl,
zPart, zCollist, zLimit
zPart,
(zStart ? (zPart ? "AND" : "WHERE") : ""), zStart,
zCollist, zLimit
);
sqlite3_free(zStart);
}else
if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){
@ -2120,7 +2346,11 @@ static int rbuObjIterPrepareAll(
zCollist, zLimit
);
}
p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql);
if( p->rc==SQLITE_OK ){
p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql);
}else{
sqlite3_free(zSql);
}
}
sqlite3_free(zImposterCols);
@ -2220,18 +2450,42 @@ static int rbuObjIterPrepareAll(
/* Create the SELECT statement to read keys from data_xxx */
if( p->rc==SQLITE_OK ){
const char *zRbuRowid = "";
char *zStart = 0;
char *zOrder = 0;
if( bRbuRowid ){
zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid";
}
p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
sqlite3_mprintf(
"SELECT %s,%s rbu_control%s FROM '%q'%s",
zCollist,
(rbuIsVacuum(p) ? "0 AS " : ""),
zRbuRowid,
pIter->zDataTbl, zLimit
)
);
if( rbuIsVacuum(p) ){
if( nOffset ){
zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite);
if( zStart ){
sqlite3_free(zLimit);
zLimit = 0;
}
}
if( bRbuRowid ){
zOrder = rbuMPrintf(p, "_rowid_");
}else{
zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", "");
}
}
if( p->rc==SQLITE_OK ){
p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz,
sqlite3_mprintf(
"SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s",
zCollist,
(rbuIsVacuum(p) ? "0 AS " : ""),
zRbuRowid,
pIter->zDataTbl, (zStart ? zStart : ""),
(zOrder ? "ORDER BY" : ""), zOrder,
zLimit
)
);
}
sqlite3_free(zStart);
sqlite3_free(zOrder);
}
sqlite3_free(zWhere);

View File

@ -1,5 +1,5 @@
C Fix\sa\sproblem\swith\srenaming\san\sINTEGER\sPRIMARY\sKEY\scolumn\sof\sa\sWITHOUT\sROWID\stable\susing\sALTER\sTABLE.
D 2019-05-06T16:15:28.529
C Optimize\sthe\srestarting\sof\san\sRBU\svacuum.
D 2019-05-07T16:37:37.162
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -338,7 +338,7 @@ F ext/rbu/rbu9.test 0e4d985e25620d61920597e8ea69c871c9e8c1f5a0be2ae9fa70bb641d74
F ext/rbu/rbuA.test b34a90cb495682c25b5fc03a9d5e7a4fc99541c29256f25e2e2a4f6542b4f5b3
F ext/rbu/rbuB.test 52b07158824c6927b7e25554ace92a695cdebfc296ae3d308ac386984aded9bc
F ext/rbu/rbuC.test 80f1cc2fb74f44b1128fd0ed8eedab3a76fefeb72a947860e2869ef76fc8dc6b
F ext/rbu/rbu_common.tcl 4b3d033b3e3844292ae3a1aefc0e524e64b0db5a0e4310657919e4504ac3073f
F ext/rbu/rbu_common.tcl 60d904133ff843fe72cc0514e9dd2486707181e6e0fbab20979da28c48d21de9
F ext/rbu/rbucollate.test cac528a9a46318cba42e61258bb42660bbbf4fdb9a8c863de5a54ad0c658d197
F ext/rbu/rbucrash.test 000981a1fe8a6e4d9a684232f6a129e66a3ef595f5ed74655e2f9c68ffa613b4
F ext/rbu/rbucrash2.test efa143cc94228eb0266d3f1abfbee60a5838a84cef7cc3fcb8c145b74d96fd41
@ -350,7 +350,7 @@ F ext/rbu/rbufault3.test e0052ccba428ffdd2bb989d3ae84716f058ec5ab5f7196c64ba407b
F ext/rbu/rbufault4.test 03d2849c3df7d7bd14a622e789ff049e5080edd34a79cd432e01204db2a5930a
F ext/rbu/rbufts.test 0ae8d1da191c75bd776b86e24456db0fb6e97b7c944259fae5407ea55d23c31d
F ext/rbu/rbumulti.test 5fb139058f37ddc5a113c5b93238de915b769b7792de41b44c983bc7c18cf5b9
F ext/rbu/rbupartial.test 73baf12a5941fe6891a829106a6f2e0a973f89aa49bd8659b12f547beb29b482
F ext/rbu/rbupartial.test 1c8bd6d42615b94caf08f129f5817fa26975523f0f51bceda1dca90e8114c7c4
F ext/rbu/rbuprogress.test 04614ff8820bab9c1ec1b7dbec1edc4b45474421d4fe7abbd2a879a9c02884f9
F ext/rbu/rburesume.test dbdc4ca504e9c76375a69e5f0d91205db967dcc509a5166ca80231f8fda49eb1
F ext/rbu/rbusave.test f4190a1a86fccf84f723af5c93813365ae33feda35845ba107b59683d1cdd926
@ -359,7 +359,8 @@ F ext/rbu/rbutemplimit.test 7f408f49b90fa0a720d7599f3aec74a3c85e6cd78e56fdf726ce
F ext/rbu/rbuvacuum.test 55e101e90168c2b31df6c9638fe73dc7f7cc666b6142266d1563697d79f73534
F ext/rbu/rbuvacuum2.test b8e5b51dc8b2c0153373d024c0936be3f66f9234acbd6d0baab0869d56b14e6b
F ext/rbu/rbuvacuum3.test 8addd82e4b83b4c93fa47428eae4fd0dbf410f8512c186f38e348feb49ba03dc
F ext/rbu/sqlite3rbu.c f222350c33f063cbc754001cd4e9683164c6cb06be76ae43f15b396ec6fc1993
F ext/rbu/rbuvacuum4.test 08abd9e18d8aee5bb812de0d6a928cc8bbacae58ecb1f150637e80f4c93c8afa
F ext/rbu/sqlite3rbu.c ec93d28c5cef16ccef7ed76c2b6c410e8eb078c73705b6e7b683f88f722228f2
F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812
F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a
F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
@ -1823,7 +1824,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P c2e439bccc40825e211bfa9a88e6a251ff066ca7453d4e7cb5eab56ce7332635
R 7ba20988fc908649bb79cc600d524699
P 91f701d39852ef1ddb29d1527d1520867c5efea110333315c55d8b08084384e7 7b051698d8a763d9db190fd662931266d24a4982f34f100a28641934c222e37b
R 8e0011462278f668f3cc12d5a8c4e1e9
T +closed 7b051698d8a763d9db190fd662931266d24a4982f34f100a28641934c222e37b
U dan
Z c4cd27021c89d31b393f236a10b2a3a7
Z a1edb1bdf5023ec77a1208563fa197ca

View File

@ -1 +1 @@
91f701d39852ef1ddb29d1527d1520867c5efea110333315c55d8b08084384e7
82062351a62f4ccc438a2b47a22ee581bd030dec952fecd610f360a25002a5ea