From 789780d8f6cd1c770ef8f39aed0ccd2a2f37d14b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Mar 2016 18:56:45 +0000 Subject: [PATCH] Change the name of the new API on this branch to "sqlite3_bp_progress". Add tests and documentation for the same. FossilOrigin-Name: 1a1b69e87eb7d18f76f5b733e44da75136a686b6 --- ext/rbu/rbuprogress.test | 145 +++++++++++++++++++++++++++++++-------- ext/rbu/sqlite3rbu.c | 49 ++++++++++++- ext/rbu/sqlite3rbu.h | 42 +++++++++++- ext/rbu/test_rbu.c | 6 +- manifest | 18 ++--- manifest.uuid | 2 +- 6 files changed, 217 insertions(+), 45 deletions(-) diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test index 005aec5b24..8459bc09e0 100644 --- a/ext/rbu/rbuprogress.test +++ b/ext/rbu/rbuprogress.test @@ -13,14 +13,20 @@ source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbuprogress + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + # Create a simple RBU database. That expects to write to a table: # # CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); # proc create_rbu1 {filename} { - forcedelete $filename - sqlite3 rbu1 $filename - rbu1 eval { + create_db_file $filename { CREATE TABLE data_t1(a, b, c, rbu_control); INSERT INTO data_t1 VALUES(1, 2, 3, 0); INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); @@ -29,7 +35,6 @@ proc create_rbu1 {filename} { CREATE TABLE rbu_count(tbl, cnt); INSERT INTO rbu_count VALUES('data_t1', 3); } - rbu1 close return $filename } @@ -41,16 +46,16 @@ do_execsql_test 1.0 { do_test 1.1 { create_rbu1 rbu.db sqlite3rbu rbu test.db rbu.db - rbu stage_progress + rbu bp_progress } {0 0} -do_test 1.2 { rbu step ; rbu stage_progress } {3333 0} -do_test 1.3 { rbu step ; rbu stage_progress } {6666 0} -do_test 1.4 { rbu step ; rbu stage_progress } {10000 0} -do_test 1.5 { rbu step ; rbu stage_progress } {10000 0} -do_test 1.6 { rbu step ; rbu stage_progress } {10000 0} -do_test 1.7 { rbu step ; rbu stage_progress } {10000 5000} -do_test 1.8 { rbu step ; rbu stage_progress } {10000 10000} -do_test 1.9 { rbu step ; rbu stage_progress } {10000 10000} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} do_test 1.10 { rbu close @@ -66,7 +71,7 @@ proc do_sp_test {tn bReopen target rbu reslist} { if {$bReopen} { sqlite3rbu rbu $target $rbu } set rc [rbu step] if {[set rc] != "SQLITE_OK"} { error "error 1" } - lappend res [lindex [rbu stage_progress] 0] + lappend res [lindex [rbu bp_progress] 0] if {[lindex [set res] end]==10000} break if {$bReopen} { rbu close } } @@ -79,26 +84,29 @@ proc do_sp_test {tn bReopen target rbu reslist} { # file to *-wal. After each of these steps, the progress remains # at "10000 0". # - rbu step - set res [rbu stage_progress] - if {[set res] != [list 10000 0]} { - error "2. reslist incorrect (expect=10000 0 got=[set res])" + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } } + rbu step - set res [rbu stage_progress] + set res [rbu bp_progress] if {[set res] != [list 10000 0]} { error "3. reslist incorrect (expect=10000 0 got=[set res])" } # Do the checkpoint. while {[rbu step]=="SQLITE_OK"} { - foreach {a b} [rbu stage_progress] {} + foreach {a b} [rbu bp_progress] {} if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" } } - set res [rbu stage_progress] + set res [rbu bp_progress] if {[set res] != [list 10000 10000]} { error "5. reslist is incorrect (expect=10000 10000 got=[set res])" } @@ -107,13 +115,6 @@ proc do_sp_test {tn bReopen target rbu reslist} { }] {SQLITE_DONE}] } -proc create_db_file {filename sql} { - forcedelete $filename - sqlite3 tmpdb $filename - tmpdb eval $sql - tmpdb close -} - foreach {bReopen} { 0 1 } { reset_db @@ -197,7 +198,95 @@ foreach {bReopen} { 0 1 } { } } {} do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} } +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 4b2529bf34..ab7b983ead 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -300,6 +300,43 @@ struct RbuFrame { /* ** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. */ struct sqlite3rbu { int eStage; /* Value of RBU_STATE_STAGE field */ @@ -2957,7 +2994,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ break; case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = (u32)sqlite3_column_int64(pStmt, 1); + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); break; default: @@ -3068,7 +3105,9 @@ static void rbuDeleteVfs(sqlite3rbu *p){ } /* -** +** This user-defined SQL function is invoked with a single argument - the +** name of a table expected to appear in the target database. It returns +** the number of auxilliary indexes on the table. */ static void rbuIndexCntFunc( sqlite3_context *pCtx, @@ -3371,7 +3410,11 @@ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } -void sqlite3rbu_stage_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ +/* +** Return permyriadage progress indications for the two main stages of +** an RBU update. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ const int MAX_PROGRESS = 10000; switch( p->eStage ){ case RBU_STAGE_OAL: diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index fb81c85a0a..f379bb5b41 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -400,7 +400,47 @@ int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); */ sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); -void sqlite3rbu_stage_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); +/* +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** progress indications for the two stages of an RBU update. This API may +** be useful for driving GUI progress indicators and similar. +** +** An RBU update is divided into two stages: +** +** * Stage 1, in which changes are accumulated in an oal/wal file, and +** * Stage 2, in which the contents of the wal file are copied into the +** main database. +** +** The update is visible to non-RBU clients during stage 2. During stage 1 +** non-RBU reader clients may see the original database. +** +** If this API is called during stage 2 of the update, output variable +** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) +** to a value between 0 and 10000 to indicate the permyriadage progress of +** stage 2. A value of 5000 indicates that stage 2 is half finished, +** 9000 indicates that it is 90% finished, and so on. +** +** If this API is called during stage 1 of the update, output variable +** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it +** exists, must contain the same columns as the following: +** +** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; +** +** There must be one row in the table for each source (data_xxx) table within +** the RBU database. The 'tbl' column should contain the name of the source +** table. The 'cnt' column should contain the number of rows within the +** source table. +** +** If the rbu_count table is present and populated correctly and this +** API is called during stage 1, the *pnOne output variable is set to the +** permyriadage progress of the same stage. If the rbu_count table does +** not exist, then (*pnOne) is set to -1 during stage 1. If the rbu_count +** table exists but is not correctly populated, the value of the *pnOne +** output variable during stage 1 is undefined. +*/ +void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); /* ** Create an RBU VFS named zName that accesses the underlying file-system diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index e35d76e745..5e8640a9f8 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -66,7 +66,7 @@ static int test_sqlite3rbu_cmd( {"create_rbu_delta", 2, ""}, /* 2 */ {"savestate", 2, ""}, /* 3 */ {"dbMain_eval", 3, "SQL"}, /* 4 */ - {"stage_progress", 2, ""}, /* 5 */ + {"bp_progress", 2, ""}, /* 5 */ {0,0,0} }; int iCmd; @@ -137,10 +137,10 @@ static int test_sqlite3rbu_cmd( break; } - case 5: /* stage_progress */ { + case 5: /* bp_progress */ { int one, two; Tcl_Obj *pObj; - sqlite3rbu_stage_progress(pRbu, &one, &two); + sqlite3rbu_bp_progress(pRbu, &one, &two); pObj = Tcl_NewObj(); Tcl_ListObjAppendElement(interp, pObj, Tcl_NewIntObj(one)); diff --git a/manifest b/manifest index 7cbf1886a9..106d447afa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sthe\schanges\son\sthis\sbranch.\sFix\sa\sproblem\swith\scalls\sto\sthe\snew\sprogress\sindicator\sAPI\smade\safter\san\srbu\supdate\shas\sbeen\sresumed. -D 2016-03-18T10:29:47.529 +C Change\sthe\sname\sof\sthe\snew\sAPI\son\sthis\sbranch\sto\s"sqlite3_bp_progress".\sAdd\stests\sand\sdocumentation\sfor\sthe\ssame. +D 2016-03-18T18:56:45.343 F Makefile.in f53429fb2f313c099283659d0df6f20f932c861f F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc df0bf9ff7f8b3f4dd9fb4cc43f92fe58f6ec5c66 @@ -242,11 +242,11 @@ F ext/rbu/rbudiff.test 6cc806dc36389292f2a8f5842d0103721df4a07d F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda -F ext/rbu/rbuprogress.test c4a9b3262bc0cafbf19709b56fbda0c3a9e69ac2 +F ext/rbu/rbuprogress.test 77fe3cd10d3c408bef364bc2e529310495ddd07e F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c c89f1e59eb09257e126d2cdcb3c5588e16a44fee -F ext/rbu/sqlite3rbu.h f8ee94f95fc80a35b7cb7ef3f2f5ea740b662477 -F ext/rbu/test_rbu.c 5b6d31af188193d929234d1cd08ee967df092f66 +F ext/rbu/sqlite3rbu.c 6b7dc899b3980d4236bffa5048218f8dba85ac0a +F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2 +F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1457,7 +1457,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ffc58d2c2576a5b6e1c2c7112612c5760e711afd -R b5c08658920cbd4a66ad9300ad32e622 +P bf82321724d3b0feb51e26d9b76090e03cc3964a +R 14faa32cca0db38c921d7bc3da7dea4c U dan -Z 25ebcfe6c5065c93d73006dcdc7ea2c0 +Z 40a2b2dbafc34d98f3493240ec44ac36 diff --git a/manifest.uuid b/manifest.uuid index a1b8da46e4..d6c18684be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf82321724d3b0feb51e26d9b76090e03cc3964a \ No newline at end of file +1a1b69e87eb7d18f76f5b733e44da75136a686b6 \ No newline at end of file