From 348bb5d6c8e534c1258b3cd6fe202399a2739b02 Mon Sep 17 00:00:00 2001 From: danielk1977 Date: Sat, 18 Oct 2003 09:37:26 +0000 Subject: [PATCH] Add sqlite_progress_handler() API for specifying an progress callback (CVS 1111) FossilOrigin-Name: ddb364635a207658664ea92fc677cf16a143a938 --- manifest | 25 +++++----- manifest.uuid | 2 +- src/main.c | 27 +++++++++- src/sqlite.h.in | 29 ++++++++++- src/sqliteInt.h | 8 ++- src/tclsqlite.c | 63 ++++++++++++++++++++++- src/vdbe.c | 22 ++++++++- test/progress.test | 118 ++++++++++++++++++++++++++++++++++++++++++++ www/c_interface.tcl | 28 ++++++++++- 9 files changed, 302 insertions(+), 20 deletions(-) create mode 100755 test/progress.test diff --git a/manifest b/manifest index 3988e9226f..cc6ca48e0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sto\sthe\sdate\sfunctions.\s(CVS\s1110) -D 2003-10-10T02:09:57 +C Add\ssqlite_progress_handler()\sAPI\sfor\sspecifying\san\sprogress\scallback\s(CVS\s1111) +D 2003-10-18T09:37:26 F Makefile.in ab585a91e34bc33928a1b6181fa2f6ebd4fb17e1 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd @@ -35,7 +35,7 @@ F src/func.c fce558b4c1d895e81091d6d5e7d86a192fc8e84c F src/hash.c 058f077c1f36f266581aa16f907a3903abf64aa3 F src/hash.h cd0433998bc1a3759d244e1637fe5a3c13b53bf8 F src/insert.c dc200ae04a36bd36e575272a069e20c528b7fbdf -F src/main.c ae92469674db9987de2848e373cd41a394621e32 +F src/main.c 9422005bb4411cc08c2986fde3278ac5b87068a0 F src/md5.c fe4f9c9c6f71dfc26af8da63e4d04489b1430565 F src/os.c 97df440bc71f65e22df5d3d920ce39551c0a5f5a F src/os.h 729395fefcca4b81ae056aa9ff67b72bb40dd9e0 @@ -48,10 +48,10 @@ F src/random.c 19e8e00fe0df32a742f115773f57651be327cabe F src/select.c d79ac60ba1595ff3c94b12892e87098329776482 F src/shell.c c2ba26c850874964f5ec1ebf6c43406f28e44c4a F src/shell.tcl 27ecbd63dd88396ad16d81ab44f73e6c0ea9d20e -F src/sqlite.h.in f8ae61546942e5a81df0ce3118048bec8dc87be4 -F src/sqliteInt.h 5f706313beafcc2da8102c807c35e18b2b0a3572 +F src/sqlite.h.in e6cfff01fafc8a82ce82cd8c932af421dc9adb54 +F src/sqliteInt.h 74dc7989c9f2b46b50485d0455a8ef8d4f178708 F src/table.c 4301926464d88d2c2c7cd21c3360aa75bf068b95 -F src/tclsqlite.c ec9e5b796bf9ec1483927e986828a205d4a7422a +F src/tclsqlite.c 3efac6b5861ac149c41251d4d4c420c94be5ba6a F src/test1.c f9d5816610f7ec4168ab7b098d5207a5708712b6 F src/test2.c 5014337d8576b731cce5b5a14bec4f0daf432700 F src/test3.c 30985ebdfaf3ee1462a9b0652d3efbdc8d9798f5 @@ -61,7 +61,7 @@ F src/trigger.c ce83e017b407d046e909d05373d7f8ee70f9f7f9 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397 F src/util.c f16efa2d60bfd4e31ae06b07ed149557e828d294 F src/vacuum.c e4724eade07e4cf8897060a8cf632dbd92408eeb -F src/vdbe.c a9923a38a24ee86dd2e237c9f7e9d0116e329394 +F src/vdbe.c 0928a242ced0b5d26292f3949fdab26fa4dc327d F src/vdbe.h 3957844e46fea71fd030e78f6a3bd2f7e320fb43 F src/vdbeInt.h 2824bf88895b901b3a8c9e44527c67530e1c0dcb F src/vdbeaux.c 31abb8e3e57866913360381947e267a51fed92c6 @@ -109,6 +109,7 @@ F test/null.test c14d0f4739f21e929b8115b72bf0c765b6bb1721 F test/pager.test dd31da9bee94a82e2e87e58cf286cfe809f8fc5f F test/pragma.test e7cb7ffd765c9158868b0b7a3771d54a0d5f5072 F test/printf.test 3ed02f1361402c0767492cd5cef4650e61df8308 +F test/progress.test 701b6115c2613128ececdfe1398a1bd0e1a4cfb3 x F test/quick.test c527bdb899b12a8cd8ceecce45f72922099f4095 F test/quote.test 08f23385c685d3dc7914ec760d492cacea7f6e3d F test/rowid.test 1936d0d866a8105ab53cf6cb40a549b6664d06ce @@ -153,7 +154,7 @@ F www/arch.fig d5f9752a4dbf242e9cfffffd3f5762b6c63b3bcf F www/arch.png 82ef36db1143828a7abc88b1e308a5f55d4336f4 F www/arch.tcl 44b589fc01d6829d43447ab40588b00aec5b9734 F www/audit.tcl 90e09d580f79c7efec0c7d6f447b7ec5c2dce5c0 -F www/c_interface.tcl acacd31d4441de900e09ee48b5ffdef0162d8dc3 +F www/c_interface.tcl 17d8bd9e7b4fbdca47c30c8b9bcb728c351d55c0 F www/changes.tcl 1188dd0e79f9a8c48996ff44e4d9e81789bf1503 F www/conflict.tcl 81dd21f9a679e60aae049e9dd8ab53d59570cda2 F www/datatypes.tcl 0cb28565580554fa7e03e8fcb303e87ce57757ae @@ -173,7 +174,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1 -P 54aa0fb236d17b53b194a667d68c71007c8e7687 -R 78b719cb974e90d3bbfc839a1c83b68b -U drh -Z 6362f853c8fee8f874044b89744bb342 +P 06d4e88394217fb1390b069bad82d6ac71981f72 +R cde37f606921798e43de3b54f3499f6d +U danielk1977 +Z a955533f397af1a9a7db7d26a39908ea diff --git a/manifest.uuid b/manifest.uuid index eae8b6bc94..a066bc7a39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06d4e88394217fb1390b069bad82d6ac71981f72 \ No newline at end of file +ddb364635a207658664ea92fc677cf16a143a938 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 1b0c692e8a..45ff1b06fe 100644 --- a/src/main.c +++ b/src/main.c @@ -14,7 +14,7 @@ ** other files are for internal use by SQLite and should not be ** accessed by users of the library. ** -** $Id: main.c,v 1.142 2003/09/06 22:18:08 drh Exp $ +** $Id: main.c,v 1.143 2003/10/18 09:37:26 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -826,6 +826,31 @@ void sqlite_busy_handler( db->pBusyArg = pArg; } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This routine sets the progress callback for an Sqlite database to the +** given callback function with the given argument. The progress callback will +** be invoked every nOps opcodes. +*/ +void sqlite_progress_handler( + sqlite *db, + int nOps, + int (*xProgress)(void*), + void *pArg +){ + if( nOps>0 ){ + db->xProgress = xProgress; + db->nProgressOps = nOps; + db->pProgressArg = pArg; + }else{ + db->xProgress = 0; + db->nProgressOps = 0; + db->pProgressArg = 0; + } +} +#endif + + /* ** This routine installs a default busy handler that waits for the ** specified number of milliseconds before returning 0. diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3832ee6b9d..6f8712145e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -12,7 +12,7 @@ ** This header file defines the interface that the SQLite library ** presents to client programs. ** -** @(#) $Id: sqlite.h.in,v 1.52 2003/09/06 22:18:08 drh Exp $ +** @(#) $Id: sqlite.h.in,v 1.53 2003/10/18 09:37:26 danielk1977 Exp $ */ #ifndef _SQLITE_H_ #define _SQLITE_H_ @@ -729,6 +729,33 @@ int sqlite_reset(sqlite_vm*, char **pzErrMsg); */ int sqlite_bind(sqlite_vm*, int idx, const char *value, int len, int copy); +/* +** This routine configures a callback function - the progress callback - that +** is invoked periodically during long running calls to sqlite_exec(), +** sqlite_step() and sqlite_get_table(). An example use for this API is to keep +** a GUI updated during a large query. +** +** The progress callback is invoked once for every N virtual machine opcodes, +** where N is the second argument to this function. The progress callback +** itself is identified by the third argument to this function. The fourth +** argument to this function is a void pointer passed to the progress callback +** function each time it is invoked. +** +** If a call to sqlite_exec(), sqlite_step() or sqlite_get_table() results +** in less than N opcodes being executed, then the progress callback is not +** invoked. +** +** Calling this routine overwrites any previously installed progress callback. +** To remove the progress callback altogether, pass NULL as the third +** argument to this function. +** +** If the progress callback returns a result other than 0, then the current +** query is immediately terminated and any database changes rolled back. If the +** query was part of a larger transaction, then the transaction is not rolled +** back and remains active. The sqlite_exec() call returns SQLITE_ABORT. +*/ +void sqlite_progress_handler(sqlite*, int, int(*)(void*), void*); + #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1b9322b052..11d14cc211 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -11,7 +11,7 @@ ************************************************************************* ** Internal interface definitions for SQLite. ** -** @(#) $Id: sqliteInt.h,v 1.199 2003/09/27 13:39:39 drh Exp $ +** @(#) $Id: sqliteInt.h,v 1.200 2003/10/18 09:37:26 danielk1977 Exp $ */ #include "config.h" #include "sqlite.h" @@ -88,6 +88,7 @@ /* #define SQLITE_OMIT_INMEMORYDB 1 */ /* #define SQLITE_OMIT_VACUUM 1 */ /* #define SQLITE_OMIT_TIMEDATE_FUNCS 1 */ +/* #define SQLITE_OMIT_PROGRESS_CALLBACK 1 */ /* ** Integers of known sizes. These typedefs might change for architectures @@ -326,6 +327,11 @@ struct sqlite { /* Access authorization function */ void *pAuthArg; /* 1st argument to the access auth function */ #endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int (*xProgress)(void *); /* The progress callback */ + void *pProgressArg; /* Argument to the progress callback */ + int nProgressOps; /* Number of opcodes for progress callback */ +#endif }; /* diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 6f186028fd..ee7cf6f61f 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -11,7 +11,7 @@ ************************************************************************* ** A TCL Interface to SQLite ** -** $Id: tclsqlite.c,v 1.50 2003/08/19 14:31:02 drh Exp $ +** $Id: tclsqlite.c,v 1.51 2003/10/18 09:37:26 danielk1977 Exp $ */ #ifndef NO_TCL /* Omit this whole file if TCL is unavailable */ @@ -52,6 +52,7 @@ struct SqliteDb { Tcl_Interp *interp; /* The interpreter used for this database */ char *zBusy; /* The busy callback routine */ char *zTrace; /* The trace callback routine */ + char *zProgress; /* The progress callback routine */ char *zAuth; /* The authorization callback routine */ SqlFunc *pFunc; /* List of SQL functions */ int rc; /* Return code of most recent sqlite_exec() */ @@ -325,6 +326,21 @@ static int DbBusyHandler(void *cd, const char *zTable, int nTries){ return 1; } +/* +** This routine is invoked as the 'progress callback' for the database. +*/ +static int DbProgressHandler(void *cd){ + SqliteDb *pDb = (SqliteDb*)cd; + int rc; + + assert( pDb->zProgress ); + rc = Tcl_Eval(pDb->interp, pDb->zProgress); + if( rc!=TCL_OK || atoi(Tcl_GetStringResult(pDb->interp)) ){ + return 1; + } + return 0; +} + /* ** This routine is called by the SQLite trace handler whenever a new ** block of SQL is executed. The TCL script in pDb->zTrace is executed. @@ -457,13 +473,14 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ "close", "complete", "errorcode", "eval", "function", "last_insert_rowid", "onecolumn", "timeout", "trace", - 0 + "progress", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BUSY, DB_CHANGES, DB_CLOSE, DB_COMPLETE, DB_ERRORCODE, DB_EVAL, DB_FUNCTION, DB_LAST_INSERT_ROWID, DB_ONECOLUMN, DB_TIMEOUT, DB_TRACE, + DB_PROGRESS, }; if( objc<2 ){ @@ -562,6 +579,48 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ break; } + /* $db progress ?N CALLBACK? + ** + ** Invoke the given callback every N virtual machine opcodes while executing + ** queries. + */ + case DB_PROGRESS: { + if( objc==2 ){ + if( pDb->zProgress ){ + Tcl_AppendResult(interp, pDb->zProgress, 0); + } + }else if( objc==4 ){ + char *zProgress; + int len; + int N; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &N) ){ + return TCL_ERROR; + }; + if( pDb->zProgress ){ + Tcl_Free(pDb->zProgress); + } + zProgress = Tcl_GetStringFromObj(objv[3], &len); + if( zProgress && len>0 ){ + pDb->zProgress = Tcl_Alloc( len + 1 ); + strcpy(pDb->zProgress, zProgress); + }else{ + pDb->zProgress = 0; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( pDb->zProgress ){ + pDb->interp = interp; + sqlite_progress_handler(pDb->db, N, DbProgressHandler, pDb); + }else{ + sqlite_progress_handler(pDb->db, 0, 0, 0); + } +#endif + }else{ + Tcl_WrongNumArgs(interp, 2, objv, "N CALLBACK"); + return TCL_ERROR; + } + break; + } + /* ** $db changes ** diff --git a/src/vdbe.c b/src/vdbe.c index 3019a7757b..ab5a2264bd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -43,7 +43,7 @@ ** in this file for details. If in doubt, do not deviate from existing ** commenting and indentation practices when changing or adding code. ** -** $Id: vdbe.c,v 1.241 2003/09/27 00:56:32 drh Exp $ +** $Id: vdbe.c,v 1.242 2003/10/18 09:37:26 danielk1977 Exp $ */ #include "sqliteInt.h" #include "os.h" @@ -529,6 +529,9 @@ int sqliteVdbeExec( unsigned long long start; /* CPU clock count at start of opcode */ int origPc; /* Program counter at start of opcode */ #endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + int nProgressOps = 0; /* Opcodes executed since progress callback. */ +#endif if( p->magic!=VDBE_MAGIC_RUN ) return SQLITE_MISUSE; assert( db->magic==SQLITE_MAGIC_BUSY ); @@ -556,6 +559,23 @@ int sqliteVdbeExec( } #endif +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + /* Call the progress callback if it is configured and the required number + ** of VDBE ops have been executed (either since this invocation of + ** sqliteVdbeExec() or since last time the progress callback was called). + ** If the progress callback returns non-zero, exit the virtual machine with + ** a return code SQLITE_ABORT. + */ + if( db->xProgress && (db->nProgressOps==nProgressOps) ){ + if( db->xProgress(db->pProgressArg)!=0 ){ + rc = SQLITE_ABORT; + continue; /* skip to the next iteration of the for loop */ + } + nProgressOps = 0; + } + nProgressOps++; +#endif + switch( pOp->opcode ){ /***************************************************************************** diff --git a/test/progress.test b/test/progress.test new file mode 100755 index 0000000000..15d769d496 --- /dev/null +++ b/test/progress.test @@ -0,0 +1,118 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the 'progress callback'. +# +# $Id: progress.test,v 1.1 2003/10/18 09:37:27 danielk1977 Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# Build some test data +# +execsql { + BEGIN; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(7); + INSERT INTO t1 VALUES(8); + INSERT INTO t1 VALUES(9); + INSERT INTO t1 VALUES(10); + COMMIT; +} + + +# Test that the progress callback is invoked. +do_test progress-1.0 { + set counter 0 + db progress 1 "[namespace code {incr counter}] ; expr 0" + execsql { + SELECT * FROM t1 + } + expr $counter > 1 +} 1 + +# Test that the query is abandoned when the progress callback returns non-zero +do_test progress1.1 { + set counter 0 + db progress 1 "[namespace code {incr counter}] ; expr 1" + execsql { + SELECT * FROM t1 + } + set counter +} 1 + +# Test that the query is rolled back when the progress callback returns +# non-zero. +do_test progress1.2 { + + # This figures out how many opcodes it takes to copy 5 extra rows into t1. + db progress 1 "[namespace code {incr five_rows}] ; expr 0" + set five_rows 0 + execsql { + INSERT INTO t1 SELECT a+10 FROM t1 WHERE a < 6 + } + db progress 0 "" + execsql { + DELETE FROM t1 WHERE a > 10 + } + + # Now set up the progress callback to abandon the query after the number of + # opcodes to copy 5 rows. That way, when we try to copy 6 rows, we know + # some data will have been inserted into the table by the time the progress + # callback abandons the query. + db progress $five_rows "expr 1" + execsql { + INSERT INTO t1 SELECT a+10 FROM t1 WHERE a < 7 + } + execsql { + SELECT count(*) FROM t1 + } +} 10 + +# Test that an active transaction remains active and not rolled back after the +# progress query abandons a query. +do_test progress1.3 { + + db progress 0 "" + execsql BEGIN + execsql { + INSERT INTO t1 VALUES(11) + } + db progress 1 "expr 1" + execsql { + INSERT INTO t1 VALUES(12) + } + db progress 0 "" + execsql COMMIT + execsql { + SELECT count(*) FROM t1 + } +} 11 + +# Check that a value of 0 for N means no progress callback +do_test progress1.4 { + set counter 0 + db progress 0 "[namespace code {incr counter}] ; expr 0" + execsql { + SELECT * FROM t1; + } + set counter +} 0 + +db progress 0 "" + +finish_test diff --git a/www/c_interface.tcl b/www/c_interface.tcl index 48bcaecd76..2a6f5fe396 100644 --- a/www/c_interface.tcl +++ b/www/c_interface.tcl @@ -1,7 +1,7 @@ # # Run this Tcl script to generate the sqlite.html file. # -set rcsid {$Id: c_interface.tcl,v 1.38 2003/07/08 23:42:25 drh Exp $} +set rcsid {$Id: c_interface.tcl,v 1.39 2003/10/18 09:37:27 danielk1977 Exp $} puts { @@ -635,6 +635,8 @@ char *sqlite_vmprintf(const char *zFormat, va_list); void sqlite_freemem(char*); +void sqlite_progress_handler(sqlite*, int, int (*)(void*), void*); +

All of the above definitions are included in the "sqlite.h" @@ -979,6 +981,30 @@ routine. The string pointer that these routines return should be freed by passing it to sqlite_freemem().

+

3.10 Performing background jobs during large queries

+ +

The sqlite_progress_handler() routine can be used to register a +callback routine with an SQLite database to be invoked periodically during long +running calls to sqlite_exec(), sqlite_step() and the various +wrapper functions. +

+ +

The callback is invoked every N virtual machine operations, where N is +supplied as the second argument to sqlite_progress_handler(). The third +and fourth arguments to sqlite_progress_handler() are a pointer to the +routine to be invoked and a void pointer to be passed as the first argument to +it. +

+ +

The time taken to execute each virtual machine operation can vary based on +many factors. A typical value for a 1 GHz PC is between half and three million +per second but may be much higher or lower, depending on the query. As such it +is difficult to schedule background operations based on virtual machine +operations. Instead, it is recommended that a callback be scheduled relatively +frequently (say every 1000 instructions) and external timer routines used to +determine whether or not background jobs need to be run. +

+

4.0 Adding New SQL Functions