mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add an API to indicate the percentage progress of an rbu update.

Browse Source 
FossilOrigin-Name: ffc58d2c2576a5b6e1c2c7112612c5760e711afd
This commit is contained in:
dan 2016-03-17 21:06:42 +00:00
parent 4b058dd4d1
commit e42195b044
6 changed files with 377 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

179
ext/rbu/rbuprogress.test Normal file
View File

@ -0,0 +1,179 @@
# 2016 March 18
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbuprogress
# Create a simple RBU database. That expects to write to a table:
#
# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_rbu1 {filename} {
forcedelete $filename
sqlite3 rbu1 $filename
rbu1 eval {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(1, 2, 3, 0);
INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 3);
}
rbu1 close
return $filename
}
do_execsql_test 1.0 {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
do_test 1.1 {
create_rbu1 rbu.db
sqlite3rbu rbu test.db rbu.db
rbu stage_progress
} {0 0}
do_test 1.2 { rbu step ; rbu stage_progress } {3333 0}
do_test 1.3 { rbu step ; rbu stage_progress } {6666 0}
do_test 1.4 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.5 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.6 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.7 { rbu step ; rbu stage_progress } {10000 5000}
do_test 1.8 { rbu step ; rbu stage_progress } {10000 10000}
do_test 1.9 { rbu step ; rbu stage_progress } {10000 10000}
do_test 1.10 {
rbu close
} {SQLITE_DONE}
#-------------------------------------------------------------------------
proc do_sp_test {tn target rbu reslist} {
uplevel [list do_test $tn [subst -nocommands {
sqlite3rbu rbu $target $rbu
set res [list]
while 1 {
set rc [rbu step]
if {[set rc] != "SQLITE_OK"} { error "error 1" }
lappend res [lindex [rbu stage_progress] 0]
if {[lindex [set res] end]==10000} break
}
if {[set res] != [list $reslist]} {
error "reslist is incorrect (expect=$reslist got=[set res])"
}
# One step to clean up the temporary tables used to update the only
# target table in the rbu database. And one more to move the *-oal
# file to *-wal.
rbu step
rbu step
# Do the checkpoint.
while {[rbu step]=="SQLITE_OK"} { }
rbu close
}] {SQLITE_DONE}]
}
proc create_db_file {filename sql} {
forcedelete $filename
sqlite3 tmpdb $filename
tmpdb eval $sql
tmpdb close
}
reset_db
do_test 2.1.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(5, 5, 5, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 2);
}
} {}
do_sp_test 2.1.1 test.db rbu.db {5000 10000}
reset_db
do_test 2.2.0 {
execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) }
create_rbu1 rbu.db
} {rbu.db}
do_sp_test 2.2.1 test.db rbu.db {3333 6666 10000}
reset_db
do_test 2.3.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 3);
}
} {}
do_sp_test 2.3.1 test.db rbu.db {1666 3333 6000 8000 10000}
reset_db
do_test 2.4.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(2, 4, 4, '.xx');
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
}
} {}
do_sp_test 2.4.1 test.db rbu.db {3333 6666 10000}
reset_db
do_test 2.5.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx');
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
}
} {}
do_sp_test 2.5.1 test.db rbu.db {10000}
finish_test

179
ext/rbu/sqlite3rbu.c
View File

@ -147,14 +147,15 @@
** RBU_STATE_OALSZ:
** Valid if STAGE==1. The size in bytes of the *-oal file.
*/
#define RBU_STATE_STAGE 1
#define RBU_STATE_TBL 2
#define RBU_STATE_IDX 3
#define RBU_STATE_ROW 4
#define RBU_STATE_PROGRESS 5
#define RBU_STATE_CKPT 6
#define RBU_STATE_COOKIE 7
#define RBU_STATE_OALSZ 8
#define RBU_STATE_STAGE 1
#define RBU_STATE_TBL 2
#define RBU_STATE_IDX 3
#define RBU_STATE_ROW 4
#define RBU_STATE_PROGRESS 5
#define RBU_STATE_CKPT 6
#define RBU_STATE_COOKIE 7
#define RBU_STATE_OALSZ 8
#define RBU_STATE_PHASEONESTEP 9
#define RBU_STAGE_OAL 1
#define RBU_STAGE_MOVE 2
@ -200,6 +201,7 @@ struct RbuState {
i64 nProgress;
u32 iCookie;
i64 iOalSz;
i64 nPhaseOneStep;
};
struct RbuUpdateStmt {
@ -244,6 +246,7 @@ struct RbuObjIter {
int iTnum; /* Root page of current object */
int iPkTnum; /* If eType==EXTERNAL, root of PK index */
int bUnique; /* Current index is unique */
int nIndex; /* Number of aux. indexes on table zTbl */
/* Statements created by rbuObjIterPrepareAll() */
int nCol; /* Number of columns in current object */
@ -314,6 +317,7 @@ struct sqlite3rbu {
const char *zVfsName; /* Name of automatically created rbu vfs */
rbu_file *pTargetFd; /* File handle open on target db */
i64 iOalSz;
i64 nPhaseOneStep;
/* The following state variables are used as part of the incremental
** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding
@ -1144,6 +1148,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
);
}
pIter->nIndex = 0;
while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){
const char *zIdx = (const char*)sqlite3_column_text(pList, 1);
sqlite3_stmt *pXInfo = 0;
@ -1157,6 +1162,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){
}
rbuFinalize(p, pXInfo);
bIndex = 1;
pIter->nIndex++;
}
rbuFinalize(p, pList);
@ -1823,6 +1829,14 @@ static void rbuTmpInsertFunc(
int rc = SQLITE_OK;
int i;
assert( sqlite3_value_int(apVal[0])!=0
|| p->objiter.eType==RBU_PK_EXTERNAL
|| p->objiter.eType==RBU_PK_NONE
);
if( sqlite3_value_int(apVal[0])!=0 ){
p->nPhaseOneStep += p->objiter.nIndex;
}
for(i=0; rc==SQLITE_OK && i<nVal; i++){
rc = sqlite3_bind_value(p->objiter.pTmpInsert, i+1, apVal[i]);
}
@ -2567,6 +2581,17 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){
assert( p->rc==SQLITE_OK );
assert( eType!=RBU_DELETE || pIter->zIdx==0 );
assert( eType==RBU_DELETE || eType==RBU_IDX_DELETE
|| eType==RBU_INSERT || eType==RBU_IDX_INSERT
);
/* If this is a delete, decrement nPhaseOneStep by nIndex. If the DELETE
** statement below does actually delete a row, nPhaseOneStep will be
** incremented by the same amount when SQL function rbu_tmp_insert()
** is invoked by the trigger. */
if( eType==RBU_DELETE ){
p->nPhaseOneStep -= p->objiter.nIndex;
}
if( eType==RBU_IDX_DELETE || eType==RBU_DELETE ){
pWriter = pIter->pDelete;
@ -2642,7 +2667,10 @@ static int rbuStep(sqlite3rbu *p){
rbuBadControlError(p);
}
else if( eType==RBU_REPLACE ){
if( pIter->zIdx==0 ) rbuStepOneOp(p, RBU_DELETE);
if( pIter->zIdx==0 ){
p->nPhaseOneStep += p->objiter.nIndex;
rbuStepOneOp(p, RBU_DELETE);
}
if( p->rc==SQLITE_OK ) rbuStepOneOp(p, RBU_INSERT);
}
else if( eType!=RBU_UPDATE ){
@ -2652,6 +2680,7 @@ static int rbuStep(sqlite3rbu *p){
sqlite3_value *pVal;
sqlite3_stmt *pUpdate = 0;
assert( eType==RBU_UPDATE );
p->nPhaseOneStep -= p->objiter.nIndex;
rbuGetUpdateStmt(p, pIter, zMask, &pUpdate);
if( pUpdate ){
int i;
@ -2729,6 +2758,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){
"(%d, %d), "
"(%d, %lld), "
"(%d, %lld), "
"(%d, %lld), "
"(%d, %lld) ",
p->zStateDb,
RBU_STATE_STAGE, eStage,
@ -2738,7 +2768,8 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){
RBU_STATE_PROGRESS, p->nProgress,
RBU_STATE_CKPT, p->iWalCksum,
RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie,
RBU_STATE_OALSZ, p->iOalSz
RBU_STATE_OALSZ, p->iOalSz,
RBU_STATE_PHASEONESTEP, p->nPhaseOneStep
)
);
assert( pInsert==0 || rc==SQLITE_OK );
@ -2925,6 +2956,10 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1);
break;
case RBU_STATE_PHASEONESTEP:
pRet->nPhaseOneStep = (u32)sqlite3_column_int64(pStmt, 1);
break;
default:
rc = SQLITE_CORRUPT;
break;
@ -3032,6 +3067,97 @@ static void rbuDeleteVfs(sqlite3rbu *p){
}
}
/*
**
*/
static void rbuIndexCntFunc(
sqlite3_context *pCtx,
int nVal,
sqlite3_value **apVal
){
sqlite3rbu *p = (sqlite3rbu*)sqlite3_user_data(pCtx);
sqlite3_stmt *pStmt = 0;
char *zErrmsg = 0;
int rc;
assert( nVal==1 );
rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg,
sqlite3_mprintf("PRAGMA index_list = %Q", sqlite3_value_text(apVal[0]))
);
if( rc!=SQLITE_OK ){
sqlite3_result_error(pCtx, zErrmsg, -1);
}else{
int nIndex = 0;
while( SQLITE_ROW==sqlite3_step(pStmt) ){
nIndex++;
}
rc = sqlite3_finalize(pStmt);
if( rc==SQLITE_OK ){
sqlite3_result_int(pCtx, nIndex);
}else{
sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1);
}
}
sqlite3_free(zErrmsg);
}
/*
** If the RBU database contains the rbu_count table, use it to initialize
** the sqlite3rbu.nPhaseOneStep variable. The schema of the rbu_count table
** is assumed to contain the same columns as:
**
** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
**
** There should be one row in the table for each data_xxx table in the
** database. The 'tbl' column should contain the name of a data_xxx table,
** and the cnt column the number of rows it contains.
**
** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt
** for all rows in the rbu_count table, where nIndex is the number of
** indexes on the corresponding target database table.
*/
static void rbuInitPhaseOneSteps(sqlite3rbu *p){
if( p->rc==SQLITE_OK ){
sqlite3_stmt *pStmt = 0;
int bExists = 0; /* True if rbu_count exists */
p->nPhaseOneStep = -1;
p->rc = sqlite3_create_function(p->dbRbu,
"rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0
);
/* Check for the rbu_count table. If it does not exist, or if an error
** occurs, nPhaseOneStep will be left set to -1. */
if( p->rc==SQLITE_OK ){
p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
"SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'"
);
}
if( p->rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
bExists = 1;
}
p->rc = sqlite3_finalize(pStmt);
}
if( p->rc==SQLITE_OK && bExists ){
p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg,
"SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))"
"FROM rbu_count"
);
if( p->rc==SQLITE_OK ){
if( SQLITE_ROW==sqlite3_step(pStmt) ){
p->nPhaseOneStep = sqlite3_column_int64(pStmt, 0);
}
p->rc = sqlite3_finalize(pStmt);
}
}
}
}
/*
** Open and return a new RBU handle.
*/
@ -3077,6 +3203,7 @@ sqlite3rbu *sqlite3rbu_open(
if( pState->eStage==0 ){
rbuDeleteOalFile(p);
rbuInitPhaseOneSteps(p);
p->eStage = RBU_STAGE_OAL;
}else{
p->eStage = pState->eStage;
@ -3243,6 +3370,38 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
return pRbu->nProgress;
}
void sqlite3rbu_stage_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
const int MAX_PROGRESS = 10000;
switch( p->eStage ){
case RBU_STAGE_OAL:
if( p->nPhaseOneStep>0 ){
*pnOne = (int)(MAX_PROGRESS * (i64)p->nProgress/(i64)p->nPhaseOneStep);
}else{
*pnOne = -1;
}
*pnTwo = 0;
break;
case RBU_STAGE_MOVE:
*pnOne = MAX_PROGRESS;
*pnTwo = 0;
break;
case RBU_STAGE_CKPT:
*pnOne = MAX_PROGRESS;
*pnTwo = (int)(MAX_PROGRESS * (i64)p->nStep / (i64)p->nFrame);
break;
case RBU_STAGE_DONE:
*pnOne = MAX_PROGRESS;
*pnTwo = MAX_PROGRESS;
break;
default:
assert( 0 );
}
}
int sqlite3rbu_savestate(sqlite3rbu *p){
int rc = p->rc;

2
ext/rbu/sqlite3rbu.h
View File

@ -400,6 +400,8 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);
*/
sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
void sqlite3rbu_stage_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
/*
** Create an RBU VFS named zName that accesses the underlying file-system
** via existing VFS zParent. Or, if the zParent parameter is passed NULL,

13
ext/rbu/test_rbu.c
View File

@ -66,6 +66,7 @@ static int test_sqlite3rbu_cmd(
{"create_rbu_delta", 2, ""}, /* 2 */
{"savestate", 2, ""}, /* 3 */
{"dbMain_eval", 3, "SQL"}, /* 4 */
{"stage_progress", 2, ""}, /* 5 */
{0,0,0}
};
int iCmd;
@ -136,6 +137,18 @@ static int test_sqlite3rbu_cmd(
break;
}
case 5: /* stage_progress */ {
int one, two;
Tcl_Obj *pObj;
sqlite3rbu_stage_progress(pRbu, &one, &two);
pObj = Tcl_NewObj();
Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one));
Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(two));
Tcl_SetObjResult(interp, pObj);
break;
}
default: /* seems unlikely */
assert( !"cannot happen" );
break;

23
manifest
View File

@ -1,5 +1,5 @@
C The\sprepared\sstatements\sfor\ssome\spragmas\scan\snow\sbe\sreused\swithout\sinvoking\nan\sautomatic\sreprepare.
D 2016-03-16T21:29:54.761
C Add\san\sAPI\sto\sindicate\sthe\spercentage\sprogress\sof\san\srbu\supdate.
D 2016-03-17T21:06:42.412
F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@ -242,10 +242,11 @@ F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d
F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
F ext/rbu/rbuprogress.test d63b70f838a20422dce6445bcaed10890e506d02
F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
F ext/rbu/sqlite3rbu.c 5956f8bee63b5ab2b04e65c1801ea0f5920dac92
F ext/rbu/sqlite3rbu.h 0bdeb3be211aaba7d85445fa36f4701a25a3dbde
F ext/rbu/test_rbu.c 4a4cdcef4ef9379fc2a21f008805c80b27bcf573
F ext/rbu/sqlite3rbu.c 9bcf35b2f1d8eaf1c82b47bead114b2289ea06c6
F ext/rbu/sqlite3rbu.h f8ee94f95fc80a35b7cb7ef3f2f5ea740b662477
F ext/rbu/test_rbu.c 5b6d31af188193d929234d1cd08ee967df092f66
F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920
F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@ -1456,8 +1457,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 10a3e2a01db9f80452a2a3369fd25b6fd9798274 db1ce7e13e656fcd2766f1b1f225cbfefe8f73ad
R 2716c51dfa2bfe8ed5e18127e51ae3c8
T +closed db1ce7e13e656fcd2766f1b1f225cbfefe8f73ad
U drh
Z e3b23194e47816995dbce6c752704ef0
P 97b0e88cc7c3d677217d0bfab4cb4a34a4abb238
R 9de86de419de6ede9128347e06885dfc
T *branch * rbu-percent-progress
T *sym-rbu-percent-progress *
T -sym-trunk *
U dan
Z f4146c58d33c341586c7f6e824f90dbe

2
manifest.uuid
View File

@ -1 +1 @@
97b0e88cc7c3d677217d0bfab4cb4a34a4abb238
ffc58d2c2576a5b6e1c2c7112612c5760e711afd