mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change the name of the new API on this branch to "sqlite3_bp_progress". Add tests and documentation for the same.

Browse Source 
FossilOrigin-Name: 1a1b69e87eb7d18f76f5b733e44da75136a686b6
This commit is contained in:
dan 2016-03-18 18:56:45 +00:00
parent fe485c0e56
commit 789780d8f6
6 changed files with 217 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
ext/rbu/rbuprogress.test
View File

@ -13,14 +13,20 @@
source [file join [file dirname [info script]] rbu_common.tcl]
set ::testprefix rbuprogress
proc create_db_file {filename sql} {
forcedelete $filename
sqlite3 tmpdb $filename
tmpdb eval $sql
tmpdb close
}
# Create a simple RBU database. That expects to write to a table:
#
# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
#
proc create_rbu1 {filename} {
forcedelete $filename
sqlite3 rbu1 $filename
rbu1 eval {
create_db_file $filename {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(1, 2, 3, 0);
INSERT INTO data_t1 VALUES(2, 'two', 'three', 0);
@ -29,7 +35,6 @@ proc create_rbu1 {filename} {
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 3);
}
rbu1 close
return $filename
}
@ -41,16 +46,16 @@ do_execsql_test 1.0 {
do_test 1.1 {
create_rbu1 rbu.db
sqlite3rbu rbu test.db rbu.db
rbu stage_progress
rbu bp_progress
} {0 0}
do_test 1.2 { rbu step ; rbu stage_progress } {3333 0}
do_test 1.3 { rbu step ; rbu stage_progress } {6666 0}
do_test 1.4 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.5 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.6 { rbu step ; rbu stage_progress } {10000 0}
do_test 1.7 { rbu step ; rbu stage_progress } {10000 5000}
do_test 1.8 { rbu step ; rbu stage_progress } {10000 10000}
do_test 1.9 { rbu step ; rbu stage_progress } {10000 10000}
do_test 1.2 { rbu step ; rbu bp_progress } {3333 0}
do_test 1.3 { rbu step ; rbu bp_progress } {6666 0}
do_test 1.4 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.5 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.6 { rbu step ; rbu bp_progress } {10000 0}
do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000}
do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000}
do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000}
do_test 1.10 {
rbu close
@ -66,7 +71,7 @@ proc do_sp_test {tn bReopen target rbu reslist} {
if {$bReopen} { sqlite3rbu rbu $target $rbu }
set rc [rbu step]
if {[set rc] != "SQLITE_OK"} { error "error 1" }
lappend res [lindex [rbu stage_progress] 0]
lappend res [lindex [rbu bp_progress] 0]
if {[lindex [set res] end]==10000} break
if {$bReopen} { rbu close }
}
@ -79,26 +84,29 @@ proc do_sp_test {tn bReopen target rbu reslist} {
# file to *-wal. After each of these steps, the progress remains
# at "10000 0".
#
rbu step
set res [rbu stage_progress]
if {[set res] != [list 10000 0]} {
error "2. reslist incorrect (expect=10000 0 got=[set res])"
if {[lindex [list $reslist] 0]!=-1} {
rbu step
set res [rbu bp_progress]
if {[set res] != [list 10000 0]} {
error "2. reslist incorrect (expect=10000 0 got=[set res])"
}
}
rbu step
set res [rbu stage_progress]
set res [rbu bp_progress]
if {[set res] != [list 10000 0]} {
error "3. reslist incorrect (expect=10000 0 got=[set res])"
}
# Do the checkpoint.
while {[rbu step]=="SQLITE_OK"} {
foreach {a b} [rbu stage_progress] {}
foreach {a b} [rbu bp_progress] {}
if {[set a]!=10000 || [set b]<=0 || [set b]>10000} {
error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])"
}
}
set res [rbu stage_progress]
set res [rbu bp_progress]
if {[set res] != [list 10000 10000]} {
error "5. reslist is incorrect (expect=10000 10000 got=[set res])"
}
@ -107,13 +115,6 @@ proc do_sp_test {tn bReopen target rbu reslist} {
}] {SQLITE_DONE}]
}
proc create_db_file {filename sql} {
forcedelete $filename
sqlite3 tmpdb $filename
tmpdb eval $sql
tmpdb close
}
foreach {bReopen} { 0 1 } {
reset_db
@ -197,7 +198,95 @@ foreach {bReopen} { 0 1 } {
}
} {}
do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000}
reset_db
do_test 2.$bReopen.6.0 {
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c);
CREATE INDEX i1 ON t1(b);
INSERT INTO t1 VALUES(1, 1, 1);
INSERT INTO t1 VALUES(2, 2, 2);
INSERT INTO t1 VALUES(3, 3, 3);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, c, rbu_control);
INSERT INTO data_t1 VALUES(4, 4, 4, 0);
INSERT INTO data_t1 VALUES(2, NULL, NULL, 1);
INSERT INTO data_t1 VALUES(5, NULL, NULL, 1);
}
} {}
do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000}
}
#-------------------------------------------------------------------------
# The following tests verify that the API works when resuming an update
# during the incremental checkpoint stage.
#
proc do_phase2_test {tn bReopen target rbu nStep} {
uplevel [list do_test $tn [subst -nocommands {
# Build the OAL/WAL file:
sqlite3rbu rbu $target $rbu
while {[lindex [rbu bp_progress] 0]<10000} {
set rc [rbu step]
if {"SQLITE_OK" != [set rc]} { rbu close }
}
# Clean up the temp tables and move the *-oal file to *-wal.
rbu step
rbu step
for {set i 0} {[set i] < $nStep} {incr i} {
if {$bReopen} {
rbu close
sqlite3rbu rbu $target $rbu
}
rbu step
set res [rbu bp_progress]
set expect [expr (1 + [set i]) * 10000 / $nStep]
if {[lindex [set res] 1] != [set expect]} {
error "Have [set res], expected 10000 [set expect]"
}
}
set rc [rbu step]
if {[set rc] != "SQLITE_DONE"} {
error "Have [set rc], expected SQLITE_DONE"
}
rbu close
}] {SQLITE_DONE}]
}
foreach bReopen {0 1} {
do_test 3.$bReopen.1.0 {
reset_db
execsql {
CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
CREATE TABLE t2(a INTEGER PRIMARY KEY, b);
CREATE TABLE t3(a INTEGER PRIMARY KEY, b);
CREATE TABLE t4(a INTEGER PRIMARY KEY, b);
}
create_db_file rbu.db {
CREATE TABLE data_t1(a, b, rbu_control);
CREATE TABLE data_t2(a, b, rbu_control);
CREATE TABLE data_t3(a, b, rbu_control);
CREATE TABLE data_t4(a, b, rbu_control);
INSERT INTO data_t1 VALUES(1, 2, 0);
INSERT INTO data_t2 VALUES(1, 2, 0);
INSERT INTO data_t3 VALUES(1, 2, 0);
INSERT INTO data_t4 VALUES(1, 2, 0);
CREATE TABLE rbu_count(tbl, cnt);
INSERT INTO rbu_count VALUES('data_t1', 1);
INSERT INTO rbu_count VALUES('data_t2', 1);
INSERT INTO rbu_count VALUES('data_t3', 1);
INSERT INTO rbu_count VALUES('data_t4', 1);
}
} {}
do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5
}
finish_test

49
ext/rbu/sqlite3rbu.c
View File

@ -300,6 +300,43 @@ struct RbuFrame {
/*
** RBU handle.
**
** nPhaseOneStep:
** If the RBU database contains an rbu_count table, this value is set to
** a running estimate of the number of b-tree operations required to
** finish populating the *-oal file. This allows the sqlite3_bp_progress()
** API to calculate the permyriadage progress of populating the *-oal file
** using the formula:
**
** permyriadage = (10000 * nProgress) / nPhaseOneStep
**
** nPhaseOneStep is initialized to the sum of:
**
** nRow * (nIndex + 1)
**
** for all source tables in the RBU database, where nRow is the number
** of rows in the source table and nIndex the number of indexes on the
** corresponding target database table.
**
** This estimate is accurate if the RBU update consists entirely of
** INSERT operations. However, it is inaccurate if:
**
** * the RBU update contains any UPDATE operations. If the PK specified
** for an UPDATE operation does not exist in the target table, then
** no b-tree operations are required on index b-trees. Or if the
** specified PK does exist, then (nIndex*2) such operations are
** required (one delete and one insert on each index b-tree).
**
** * the RBU update contains any DELETE operations for which the specified
** PK does not exist. In this case no operations are required on index
** b-trees.
**
** * the RBU update contains REPLACE operations. These are similar to
** UPDATE operations.
**
** nPhaseOneStep is updated to account for the conditions above during the
** first pass of each source table. The updated nPhaseOneStep value is
** stored in the rbu_state table if the RBU update is suspended.
*/
struct sqlite3rbu {
int eStage; /* Value of RBU_STATE_STAGE field */
@ -2957,7 +2994,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){
break;
case RBU_STATE_PHASEONESTEP:
pRet->nPhaseOneStep = (u32)sqlite3_column_int64(pStmt, 1);
pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1);
break;
default:
@ -3068,7 +3105,9 @@ static void rbuDeleteVfs(sqlite3rbu *p){
}
/*
**
** This user-defined SQL function is invoked with a single argument - the
** name of a table expected to appear in the target database. It returns
** the number of auxilliary indexes on the table.
*/
static void rbuIndexCntFunc(
sqlite3_context *pCtx,
@ -3371,7 +3410,11 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){
return pRbu->nProgress;
}
void sqlite3rbu_stage_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
/*
** Return permyriadage progress indications for the two main stages of
** an RBU update.
*/
void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){
const int MAX_PROGRESS = 10000;
switch( p->eStage ){
case RBU_STAGE_OAL:

42
ext/rbu/sqlite3rbu.h
View File

@ -400,7 +400,47 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg);
*/
sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu);
void sqlite3rbu_stage_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
/*
** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100)
** progress indications for the two stages of an RBU update. This API may
** be useful for driving GUI progress indicators and similar.
**
** An RBU update is divided into two stages:
**
** * Stage 1, in which changes are accumulated in an oal/wal file, and
** * Stage 2, in which the contents of the wal file are copied into the
** main database.
**
** The update is visible to non-RBU clients during stage 2. During stage 1
** non-RBU reader clients may see the original database.
**
** If this API is called during stage 2 of the update, output variable
** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo)
** to a value between 0 and 10000 to indicate the permyriadage progress of
** stage 2. A value of 5000 indicates that stage 2 is half finished,
** 9000 indicates that it is 90% finished, and so on.
**
** If this API is called during stage 1 of the update, output variable
** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The
** value to which (*pnOne) is set depends on whether or not the RBU
** database contains an "rbu_count" table. The rbu_count table, if it
** exists, must contain the same columns as the following:
**
** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID;
**
** There must be one row in the table for each source (data_xxx) table within
** the RBU database. The 'tbl' column should contain the name of the source
** table. The 'cnt' column should contain the number of rows within the
** source table.
**
** If the rbu_count table is present and populated correctly and this
** API is called during stage 1, the *pnOne output variable is set to the
** permyriadage progress of the same stage. If the rbu_count table does
** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count
** table exists but is not correctly populated, the value of the *pnOne
** output variable during stage 1 is undefined.
*/
void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo);
/*
** Create an RBU VFS named zName that accesses the underlying file-system

6
ext/rbu/test_rbu.c
View File

@ -66,7 +66,7 @@ static int test_sqlite3rbu_cmd(
{"create_rbu_delta", 2, ""}, /* 2 */
{"savestate", 2, ""}, /* 3 */
{"dbMain_eval", 3, "SQL"}, /* 4 */
{"stage_progress", 2, ""}, /* 5 */
{"bp_progress", 2, ""}, /* 5 */
{0,0,0}
};
int iCmd;
@ -137,10 +137,10 @@ static int test_sqlite3rbu_cmd(
break;
}
case 5: /* stage_progress */ {
case 5: /* bp_progress */ {
int one, two;
Tcl_Obj *pObj;
sqlite3rbu_stage_progress(pRbu, &one, &two);
sqlite3rbu_bp_progress(pRbu, &one, &two);
pObj = Tcl_NewObj();
Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one));

18
manifest
View File

@ -1,5 +1,5 @@
C Add\stests\sfor\sthe\schanges\son\sthis\sbranch.\sFix\sa\sproblem\swith\scalls\sto\sthe\snew\sprogress\sindicator\sAPI\smade\safter\san\srbu\supdate\shas\sbeen\sresumed.
D 2016-03-18T10:29:47.529
C Change\sthe\sname\sof\sthe\snew\sAPI\son\sthis\sbranch\sto\s"sqlite3_bp_progress".\sAdd\stests\sand\sdocumentation\sfor\sthe\ssame.
D 2016-03-18T18:56:45.343
F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66
@ -242,11 +242,11 @@ F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d
F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
F ext/rbu/rbuprogress.test c4a9b3262bc0cafbf19709b56fbda0c3a9e69ac2
F ext/rbu/rbuprogress.test 77fe3cd10d3c408bef364bc2e529310495ddd07e
F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
F ext/rbu/sqlite3rbu.c c89f1e59eb09257e126d2cdcb3c5588e16a44fee
F ext/rbu/sqlite3rbu.h f8ee94f95fc80a35b7cb7ef3f2f5ea740b662477
F ext/rbu/test_rbu.c 5b6d31af188193d929234d1cd08ee967df092f66
F ext/rbu/sqlite3rbu.c 6b7dc899b3980d4236bffa5048218f8dba85ac0a
F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2
F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d
F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920
F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
@ -1457,7 +1457,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ffc58d2c2576a5b6e1c2c7112612c5760e711afd
R b5c08658920cbd4a66ad9300ad32e622
P bf82321724d3b0feb51e26d9b76090e03cc3964a
R 14faa32cca0db38c921d7bc3da7dea4c
U dan
Z 25ebcfe6c5065c93d73006dcdc7ea2c0
Z 40a2b2dbafc34d98f3493240ec44ac36

2
manifest.uuid
View File

@ -1 +1 @@
bf82321724d3b0feb51e26d9b76090e03cc3964a
1a1b69e87eb7d18f76f5b733e44da75136a686b6