Fix a problem preventing resumption of RBU operations after recovering from a

process or system failure that occurs during the incremental-checkpoint phase.

FossilOrigin-Name: 97914266cb4ec63b0c9185ab139673139bd2f0ed

ext/rbu/rburesume.test

@@ -0,0 +1,254 @@
+# 2017 January 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests for resumption of RBU operations in the
+# case where the previous RBU process crashed.
+#
+
+source [file join [file dirname [info script]] rbu_common.tcl]
+set ::testprefix rburesume
+
+forcedelete test.db-shm test.db-oal
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a PRIMARY KEY, b, c);
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  CREATE INDEX t1c ON t1(c);
+  WITH s(i) AS (
+    VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50
+  )
+  INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s;
+}
+db_save_and_close
+
+do_test 1.1 {
+  list [file exists test.db] \
+       [file exists test.db-wal] \
+       [file exists test.db-shm] \
+       [file exists test.db-oal]
+} {1 0 0 0}
+
+# Each iteration of the following loop:
+#
+#   1. Restores the db to the state it was in following test case 1.0
+#   2. Opens an RBU vacuum and steps it $n times.
+#   3. Closes the RBU vacuum handled opened in (2).
+#   4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. 
+#
+# The loop runs until $n is large enough that step (2) vacuums the entire
+# database.
+#
+for {set n 1} {$n < 5000} {incr n} {
+  db_restore
+  forcedelete state.db
+  sqlite3rbu_vacuum rbu test.db state.db
+  for {set i 0} {$i<$n} {incr i} {
+    set rc [rbu step]
+    if {$rc == "SQLITE_DONE"} break
+  }
+  rbu close
+  if {$rc == "SQLITE_DONE"} break
+
+  do_test 1.2.$n.1 {
+    sqlite3rbu_vacuum rbu test.db state.db
+    while {[rbu step]=="SQLITE_OK"} {}
+    rbu close
+  } {SQLITE_DONE}
+
+  do_test 1.2.$n.2 {
+    sqlite3 db2 test.db
+    db2 eval { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    }
+  } {50 ok}
+  db2 close
+}
+
+# Each iteration of this loop:
+#
+#   1. Restores the db to the state it was in following test case 1.0
+#   2. Opens an RBU vacuum and steps it $n times.
+#   3. Takes a copy of all database files and the state db.
+#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
+#      vacuum op. 
+#
+# The loop runs until $n is large enough that step (2) vacuums the entire
+# database.
+#
+for {set n 1} {$n < 5000} {incr n} {
+  db_restore
+  forcedelete state.db state.db-shm state.db-oal state.db-wal
+  sqlite3rbu_vacuum rbu test.db state.db
+  for {set i 0} {$i<$n} {incr i} {
+    set rc [rbu step]
+    if {$rc == "SQLITE_DONE"} break
+  }
+  if {$rc == "SQLITE_DONE"} {
+    rbu close
+    break
+  }
+
+  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
+    set f2 [string map [list test.db test.db2] $f]
+    if {[file exists $f]} {
+      forcecopy $f $f2
+    } else {
+      forcedelete $f2
+    }
+  }
+  forcecopy state.db state.db2
+  rbu close
+
+  do_test 1.3.$n.1 {
+    sqlite3rbu_vacuum rbu test.db2 state.db2
+    while {[rbu step]=="SQLITE_OK"} {}
+    rbu close
+  } {SQLITE_DONE}
+
+  do_test 1.3.$n.2 {
+    sqlite3 db2 test.db2
+    db2 eval { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    }
+  } {50 ok}
+  db2 close
+}
+
+# Each iteration of this loop:
+#
+#   1. Restores the db to the state it was in following test case 1.0
+#   2. Opens an RBU vacuum and steps it 10 times. Then closes it.
+#   2. Opens an RBU vacuum and steps it $n times.
+#   3. Takes a copy of all database files and the state db.
+#   4. Opens a second RBU vacuum handle on the copy, resumes and completes the
+#      vacuum op. 
+#
+# The loop runs until $n is large enough that step (3) vacuums the entire
+# database.
+#
+for {set n 1} {$n < 5000} {incr n} {
+  db_restore
+  forcedelete state.db state.db-shm state.db-oal state.db-wal
+
+  sqlite3rbu_vacuum rbu test.db state.db
+  for {set i 0} {$i<10} {incr i} {
+    rbu step
+  }
+  rbu close
+
+  sqlite3rbu_vacuum rbu test.db state.db
+  for {set i 0} {$i<$n} {incr i} {
+    set rc [rbu step]
+    if {$rc == "SQLITE_DONE"} break
+  }
+  if {$rc == "SQLITE_DONE"} {
+    rbu close
+    break
+  }
+
+  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
+    set f2 [string map [list test.db test.db2] $f]
+    if {[file exists $f]} {
+      forcecopy $f $f2
+    } else {
+      forcedelete $f2
+    }
+  }
+  forcecopy state.db state.db2
+  rbu close
+
+  do_test 1.4.$n.1 {
+    sqlite3rbu_vacuum rbu test.db2 state.db2
+    while {[rbu step]=="SQLITE_OK"} {}
+    rbu close
+  } {SQLITE_DONE}
+
+  do_test 1.4.$n.2 {
+    sqlite3 db2 test.db2
+    db2 eval { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    }
+  } {50 ok}
+  db2 close
+}
+
+forcedelete rbu.db
+do_test 2.0 {
+  sqlite3 db2 rbu.db
+  db2 eval {
+    CREATE TABLE data_t1(a, b, c, rbu_control);
+    WITH s(i) AS (
+        VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10
+    )
+    INSERT INTO data_t1 
+      SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s;
+  }
+  db2 close
+} {}
+
+# Each iteration of this loop:
+#
+#   1. Restores the db to the state it was in following test case 1.0
+#   2. Opens an RBU handle to apply the RBU update created in test case 2.0.
+#   3. Steps the RBU handle $n times.
+#   4. Takes a copy of all database files and the state db.
+#   5. Opens a second RBU handle on the copy, resumes and completes the
+#      RBU op. Checks it worked as expected.
+#
+# The loop runs until $n is large enough that step (3) applies the entire
+# update.
+#
+for {set n 1} {$n < 5000} {incr n} {
+  db_restore
+  forcedelete state.db state.db-shm state.db-oal state.db-wal
+  sqlite3rbu rbu test.db rbu.db state.db
+
+  for {set i 0} {$i<$n} {incr i} {
+    set rc [rbu step]
+    if {$rc == "SQLITE_DONE"} break
+  }
+  if {$rc == "SQLITE_DONE"} {
+    rbu close
+    break
+  }
+
+  foreach f {test.db test.db-oal test.db-wal test.db-shm test.db-vacuum} {
+    set f2 [string map [list test.db test.db2] $f]
+    if {[file exists $f]} {
+      forcecopy $f $f2
+    } else {
+      forcedelete $f2
+    }
+  }
+  forcecopy state.db state.db2
+  rbu close
+
+  do_test 2.$n.1 {
+    sqlite3rbu rbu test.db2 rbu.db state.db2
+    while {[rbu step]=="SQLITE_OK"} {}
+    rbu close
+  } {SQLITE_DONE}
+
+  do_test 2.$n.2 {
+    sqlite3 db2 test.db2
+    db2 eval { 
+      SELECT count(*) FROM t1;
+      PRAGMA integrity_check;
+    }
+  } {60 ok}
+  db2 close
+}
+
+finish_test
+

ext/rbu/sqlite3rbu.c

@@ -2408,7 +2408,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){
     }else{
       RbuState *pState = rbuLoadState(p);
       if( pState ){
-        bOpen = (pState->eStage>RBU_STAGE_MOVE);
+        bOpen = (pState->eStage>=RBU_STAGE_MOVE);
         rbuFreeState(pState);
       }
     }

manifest

@@ -1,5 +1,5 @@
-C Fix\sthe\sbuild\sfor\sSQLITE_ENABLE_MEMORY_MANAGEMENT.
-D 2017-01-13T12:53:35.623
+C Fix\sa\sproblem\spreventing\sresumption\sof\sRBU\soperations\safter\srecovering\sfrom\sa\nprocess\sor\ssystem\sfailure\sthat\soccurs\sduring\sthe\sincremental-checkpoint\sphase.
+D 2017-01-13T18:24:37.297
 F Makefile.in 41bd4cad981487345c4a84081074bcdb876e4b2e
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc b8ca53350ae545e3562403d5da2a69cec79308da
@@ -254,10 +254,11 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda
 F ext/rbu/rbuprogress.test e3e25fb7622641b8f2df7c6b7a7eb6fddfc46a4b
+F ext/rbu/rburesume.test 8acb77f4a422ff55acfcfc9cc15a5cb210b1de83
 F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
 F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc
-F ext/rbu/sqlite3rbu.c e074c38798b90591f7f0cf0032d62f152ce5a95e
+F ext/rbu/sqlite3rbu.c 4ca89fbe9ca501da17f338d580d3f19d59fc6053
 F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba
 F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
@@ -1544,7 +1545,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 9acc72381ccd5e36f3ffdf7e7fbefc5a15701eb4
-R 538795cf3f379197890291f480545c18
-U drh
-Z 8d3190b0c41cbb057e375dc759a5947e
+P 8c85b8fdd7f0ba65fba83361d361a567b797a184
+R dc62936e86267740ae9736f88160b05f
+U dan
+Z f2366ce5ecd39574b5b5eb81b3b99a63

manifest.uuid

@@ -1 +1 @@
-8c85b8fdd7f0ba65fba83361d361a567b797a184
+97914266cb4ec63b0c9185ab139673139bd2f0ed