Add file test/tt3_checkpoint.c that adds a multi-threaded test for blocking checkpoints to threadtest3.

FossilOrigin-Name: 648dd157ef3b7b790764698fd4dd7107c25212c9

main.mk

@@ -530,8 +530,8 @@ test:	testfixture$(EXE) sqlite3$(EXE)
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
 # 
-threadtest3$(EXE): sqlite3.c $(TOP)/test/threadtest3.c
-	$(TCCX) -O2 sqlite3.c $(TOP)/test/threadtest3.c \
+threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
+	$(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
 		-o threadtest3$(EXE) $(THREADLIB)
 
 threadtest: threadtest3$(EXE)

manifest

@@ -1,5 +1,5 @@
-C Add\sextra\stest\scases\sfor\sblocking\scheckpoints.
-D 2010-11-19T07:17:10
+C Add\sfile\stest/tt3_checkpoint.c\sthat\sadds\sa\smulti-threaded\stest\sfor\sblocking\scheckpoints\sto\sthreadtest3.
+D 2010-11-19T09:58:11
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -99,7 +99,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
-F main.mk 497c8cb6ae132c88fa184e5e454b0e6336da5693
+F main.mk 2c360958d2981a394ce021a9dfc842577f7935df
 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -668,7 +668,7 @@ F test/thread2.test e08034b83fe9693ade77049732518e5b3d2d700d
 F test/thread_common.tcl 2aa6f2fdcd4d6e461169c3e5ca098eebf643b863
 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
 F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9
-F test/threadtest3.c d6d209190c7110f9a7e6a8154bdc3260efdbf8b7
+F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef
 F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab
 F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660
 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28
@@ -794,6 +794,7 @@ F test/triggerA.test eaf11a29db2a11967d2d4b49d37f92bce598194e
 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe
 F test/triggerC.test 2a23edcc00684d084902ba5ec93e721775c3a70a
 F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7
+F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af
 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff
 F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84
 F test/types3.test a0f66bf12f80fad89493535474f7a6d16fa58150
@@ -828,7 +829,7 @@ F test/wal.test f060cae4b2164c4375109a8f803873187234661d
 F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7
 F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4
 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30
-F test/wal5.test b467d39f88ce0e814a3cf381f0f6a014664d2e85
+F test/wal5.test 1f99651d856c8b9e1376781c981d1b903e93a478
 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
 F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4
 F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0
@@ -888,7 +889,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P 4c663a4dcc77e00558edd94f58410605b95db33a
-R d43362578e7138fb690b169b023fbca4
+P ac348ae25cb0239dc525bb473cc83cdb1b3de205
+R d259a6023fb0ad6db4b616b7c9bd726d
 U dan
-Z c818aa76c45599e0e0a501a870acd353
+Z 699d2f3824e67293f11a7075160f739b

manifest.uuid

@@ -1 +1 @@
-ac348ae25cb0239dc525bb473cc83cdb1b3de205
+648dd157ef3b7b790764698fd4dd7107c25212c9

test/threadtest3.c

@@ -1394,6 +1394,7 @@ static void dynamic_triggers(int nMs){
   print_and_free_err(&err);
 }
 
+#include "tt3_checkpoint.c"
 
 int main(int argc, char **argv){
   struct ThreadTest {
@@ -1408,8 +1409,11 @@ int main(int argc, char **argv){
     { walthread5, "walthread5",  1000 },
     { walthread5, "walthread5",  1000 },
     
-    { cgt_pager_1, "cgt_pager_1", 0 },
+    { cgt_pager_1,      "cgt_pager_1", 0 },
     { dynamic_triggers, "dynamic_triggers", 20000 },
+
+    { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
+    { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },
   };
 
   int i;

test/tt3_checkpoint.c

@@ -0,0 +1,150 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the test program "threadtest3". Despite being a C
+** file it is not compiled separately, but included by threadtest3.c using
+** the #include directive normally used with header files.
+**
+** This file contains the implementation of test cases:
+**
+**     checkpoint_starvation_1
+**     checkpoint_starvation_2
+*/
+
+/*
+** Both test cases involve 1 writer/checkpointer thread and N reader threads.
+** 
+** Each reader thread performs a series of read transactions, one after 
+** another. Each read transaction lasts for 100 ms.
+**
+** The writer writes transactions as fast as possible. It uses a callback
+** registered with sqlite3_wal_hook() to try to keep the WAL-size limited to 
+** around 50 pages.
+**
+** In test case checkpoint_starvation_1, the auto-checkpoint uses 
+** SQLITE_CHECKPOINT_PASSIVE. In checkpoint_starvation_2, it uses RESTART.
+** The expectation is that in the first case the WAL file will grow very 
+** large, and in the second will be limited to the 50 pages or thereabouts.
+** However, the overall transaction throughput will be lower for 
+** checkpoint_starvation_2, as every checkpoint will block for up to 200 ms
+** waiting for readers to clear.
+*/
+
+/* Frame limit used by the WAL hook for these tests. */
+#define CHECKPOINT_STARVATION_FRAMELIMIT 50
+
+/* Duration in ms of each read transaction */
+#define CHECKPOINT_STARVATION_READMS    100
+
+struct CheckpointStarvationCtx {
+  int eMode;
+  int nMaxFrame;
+};
+typedef struct CheckpointStarvationCtx CheckpointStarvationCtx;
+
+static int checkpoint_starvation_walhook(
+  void *pCtx, 
+  sqlite3 *db, 
+  const char *zDb, 
+  int nFrame
+){
+  CheckpointStarvationCtx *p = (CheckpointStarvationCtx *)pCtx;
+  if( nFrame>p->nMaxFrame ){
+    p->nMaxFrame = nFrame;
+  }
+  if( nFrame>=CHECKPOINT_STARVATION_FRAMELIMIT ){
+    sqlite3_wal_checkpoint_v2(db, zDb, p->eMode, 0, 0);
+  }
+  return SQLITE_OK;
+}
+
+static char *checkpoint_starvation_reader(int iTid, int iArg){
+  Error err = {0};
+  Sqlite db = {0};
+
+  opendb(&err, &db, "test.db", 0);
+  while( !timetostop(&err) ){
+    i64 iCount1, iCount2;
+    sql_script(&err, &db, "BEGIN");
+    iCount1 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
+    usleep(CHECKPOINT_STARVATION_READMS*1000);
+    iCount2 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
+    sql_script(&err, &db, "COMMIT");
+
+    if( iCount1!=iCount2 ){
+      test_error(&err, "Isolation failure - %lld %lld", iCount1, iCount2);
+    }
+  }
+  closedb(&err, &db);
+
+  print_and_free_err(&err);
+  return 0;
+}
+
+static void checkpoint_starvation_main(int nMs, CheckpointStarvationCtx *p){
+  Error err = {0};
+  Sqlite db = {0};
+  Threadset threads = {0};
+  int nInsert = 0;
+  int i;
+
+  opendb(&err, &db, "test.db", 1);
+  sql_script(&err, &db, 
+      "PRAGMA page_size = 1024;"
+      "PRAGMA journal_mode = WAL;"
+      "CREATE TABLE t1(x);"
+  );
+
+  setstoptime(&err, nMs);
+
+  for(i=0; i<4; i++){
+    launch_thread(&err, &threads, checkpoint_starvation_reader, 0);
+    usleep(CHECKPOINT_STARVATION_READMS*1000/4);
+  }
+
+  sqlite3_wal_hook(db.db, checkpoint_starvation_walhook, (void *)p);
+  while( !timetostop(&err) ){
+    sql_script(&err, &db, "INSERT INTO t1 VALUES(randomblob(1200))");
+    nInsert++;
+  }
+
+  printf(" Checkpoint mode  : %s\n",
+      p->eMode==SQLITE_CHECKPOINT_PASSIVE ? "PASSIVE" : "RESTART"
+  );
+  printf(" Peak WAL         : %d frames\n", p->nMaxFrame);
+  printf(" Transaction count: %d transactions\n", nInsert);
+
+  join_all_threads(&err, &threads);
+  closedb(&err, &db);
+  print_and_free_err(&err);
+}
+
+static void checkpoint_starvation_1(int nMs){
+  Error err = {0};
+  CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_PASSIVE, 0 };
+  checkpoint_starvation_main(nMs, &ctx);
+  if( ctx.nMaxFrame<(CHECKPOINT_STARVATION_FRAMELIMIT*10) ){
+    test_error(&err, "WAL failed to grow - %d frames", ctx.nMaxFrame);
+  }
+  print_and_free_err(&err);
+}
+
+static void checkpoint_starvation_2(int nMs){
+  Error err = {0};
+  CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_RESTART, 0 };
+  checkpoint_starvation_main(nMs, &ctx);
+  if( ctx.nMaxFrame>CHECKPOINT_STARVATION_FRAMELIMIT+10 ){
+    test_error(&err, "WAL grew too large - %d frames", ctx.nMaxFrame);
+  }
+  print_and_free_err(&err);
+}
+
+

test/wal5.test

@@ -207,6 +207,9 @@ do_multiclient_test tn {
 #      database file, RESTART checkpoints block until readers using any part
 #      of the log file have finished.
 #
+# This test case involves running a checkpoint while there exist other 
+# processes holding all three types of locks.
+#
 foreach {tn1 checkpoint busy_on ckpt_expected expected} {
   1   PASSIVE   -   {0 5 5}   -
   2   TYPO      -   {0 5 5}   -