From f51446a38cb0db8dcfeade1e30da43de0f80d266 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Jul 2012 19:40:42 +0000 Subject: [PATCH 01/99] Add an internal interface that allows the code to take advantage of multiple cores by pushing subcomputations off into separate threads. The interface is not currently used. FossilOrigin-Name: 0e4d977a4a07d6de50acbf022c7dd947998b8d96 --- Makefile.in | 6 +- Makefile.msc | 6 +- main.mk | 3 +- manifest | 28 +++++---- manifest.uuid | 2 +- src/sqliteInt.h | 7 +++ src/threads.c | 120 ++++++++++++++++++++++++++++++++++++++ tool/mksqlite3c-noext.tcl | 1 + tool/mksqlite3c.tcl | 1 + 9 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 src/threads.c diff --git a/Makefile.in b/Makefile.in index df424cb190..894b352d09 100644 --- a/Makefile.in +++ b/Makefile.in @@ -176,7 +176,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -260,6 +260,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tclsqlite.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ @@ -708,6 +709,9 @@ status.lo: $(TOP)/src/status.c $(HDR) table.lo: $(TOP)/src/table.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c +threads.lo: $(TOP)/src/threads.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c + tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c diff --git a/Makefile.msc b/Makefile.msc index e16ced35e4..33cde181c5 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -376,7 +376,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.o tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -463,6 +463,7 @@ SRC = \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\table.c \ + $(TOP)\src\threads.c \ $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\trigger.c \ @@ -896,6 +897,9 @@ status.lo: $(TOP)\src\status.c $(HDR) table.lo: $(TOP)\src\table.c $(HDR) $(LTCOMPILE) -c $(TOP)\src\table.c +threads.lo: $(TOP)\src\threads.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\threads.c + tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) -c $(TOP)\src\tokenize.c diff --git a/main.mk b/main.mk index 8682961881..7a7be4b9db 100644 --- a/main.mk +++ b/main.mk @@ -64,7 +64,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ notify.o opcodes.o os.o os_unix.o os_win.o \ pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ - table.o tokenize.o trigger.o \ + table.o threads.o tokenize.o trigger.o \ update.o util.o vacuum.o \ vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \ vdbetrace.o wal.o walker.o where.o utf.o vtab.o @@ -142,6 +142,7 @@ SRC = \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ $(TOP)/src/tclsqlite.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ diff --git a/manifest b/manifest index 8576e56420..7e6d5fef2a 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Ensure\sthat\sthere\sis\salways\sat\sleast\sone\saReadMark\sslot\susable\sby\san\sunprivileged\sreader\swhile\sa\scheckpoint\sis\srunning.\sAlso,\sif\sone\sor\smore\stransactions\sare\srecovered\sfrom\sa\slog\sfile,\sinitialize\sone\sof\sthe\saReadMark\sslots\sto\scontain\smxFrame\sas\spart\sof\sthe\srecovery\sprocess. -D 2012-07-17T14:37:12.494 +C Add\san\sinternal\sinterface\sthat\sallows\sthe\scode\sto\stake\sadvantage\sof\smultiple\ncores\sby\spushing\ssubcomputations\soff\sinto\sseparate\sthreads.\s\sThe\sinterface\nis\snot\scurrently\sused. +D 2012-07-21T19:40:42.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 8f6d858bf3df9978ba43df19985146a1173025e4 +F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 56ff0fcc3fc3b275aec7f6acb34b3c0526c684bc +F Makefile.msc 332c166048570b65127db33450bca22c53b4f0a2 F Makefile.vxworks 879f034a64062a364b21000266bbd5bc6e0c19b9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION a71848df48082f1d6585d4b0819d530fc455485d @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d109a9342d1fe135d3900aca9f5563f9480a991d +F main.mk dd63e2de89cdcb9be441e8f046a0acce3fbae2a8 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -180,7 +180,7 @@ F src/select.c f6c4833c4d8e94714761d99013d74f381e084f1d F src/shell.c 076e1c90d594644f36027c8ecff9a392cf2d3a06 F src/sqlite.h.in 310ae7e538883fa1619ab0638c775ce11ad43015 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 2bc2ebc2ff1a2b530ee5ed9ffd46c6fce93b244c +F src/sqliteInt.h 273e9d00296d3a76519c0446698c23787d8951f9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -231,6 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 +F src/threads.c df23fc534a54fe7e1ff935628fa14a9318d27ae3 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -978,8 +979,8 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl 589c7f44e990be1b8443cfe4808dce392b0327fa +F tool/mksqlite3c-noext.tcl 752f1a9d3287f6c0ef5738b1c4add0b96fbe0854 +F tool/mksqlite3c.tcl d4923e8e75b7710ddbe4eb37f83dda5eadef63d8 F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 @@ -1005,7 +1006,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 8c9ee1d78f99394eef73a177141ca9e1c67e4e07 -R 21a0c6942de3e9593e3e93f462c443c7 -U dan -Z a98d8c358c03bd601d60af308961371e +P e4163596339c2166f9c4356ab824fff8bda8d0b0 +R e05d278833a84eb0e1d0c443cf6ae5d1 +T *branch * threads +T *sym-threads * +T -sym-trunk * +U drh +Z d49696325d34c649ba05b5124d6547e7 diff --git a/manifest.uuid b/manifest.uuid index dad8b25eaa..250952bf9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4163596339c2166f9c4356ab824fff8bda8d0b0 \ No newline at end of file +0e4d977a4a07d6de50acbf022c7dd947998b8d96 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f7f5fd386e..f0ccc5cd28 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -661,6 +661,7 @@ typedef struct Parse Parse; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; @@ -3307,4 +3308,10 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ +/* +** Threading interface +*/ +int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); +int sqlite3ThreadJoin(SQLiteThread*, void**); + #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c new file mode 100644 index 0000000000..ed98015510 --- /dev/null +++ b/src/threads.c @@ -0,0 +1,120 @@ +/* +** 2012 July 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file presents a simple cross-platform threading interface for +** use internally by SQLite. +** +** A "thread" can be created using sqlite3ThreadCreate(). This thread +** runs independently of its creator until it is joined using +** sqlite3ThreadJoin(), at which point it terminates. +** +** Threads do not have to be real. It could be that the work of the +** "thread" is done by the main thread at either the sqlite3ThreadCreate() +** or sqlite3ThreadJoin() call. This is, in fact, what happens in +** single threaded systems. Nothing in SQLite requires multiple threads. +** This interface exists so that applications that want to take advantage +** of multiple cores can do so, while also allowing applications to stay +** single-threaded if desired. +*/ +#include "sqliteInt.h" + +/********************************* Unix Pthreads ****************************/ +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + pthread_t tid; +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + int rc; + + *ppThread = p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_OK; + rc = pthread_create(&p->tid, 0, xTask, pIn); + if( rc ){ + sqlite3_free(p); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + int rc; + if( p==0 ) return SQLITE_NOMEM; + rc = pthread_join(p->tid, ppOut); + sqlite3_free(p); + return rc ? SQLITE_ERROR : SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ +/******************************** End Unix Pthreads *************************/ + + +/********************************* Single-Threaded **************************/ +#ifndef SQLITE_THREADS_IMPLEMENTED +/* +** This implementation does not actually create a new thread. It does the +** work of the thread in the main thread, when either the thread is created +** or when it is joined +*/ + +/* A running thread */ +struct SQLiteThread { + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + *ppThread = p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ + p->xTask = xTask; + p->pIn = pIn; + }else{ + p->xTask = 0; + p->pResult = xTask(pIn); + } + return p; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + if( p==0 ) return SQLITE_NOMEM; + if( p->xTask ){ + *ppOut = = p->xTask(p->pIn); + }else{ + *ppOut = p->pResult; + } + sqlite3_free(p); + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ +/****************************** End Single-Threaded *************************/ diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index 017ad6292f..3a0138b3b2 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -237,6 +237,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 2c569d7a4e..4885728f95 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -242,6 +242,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c From da0e47109e6ba93ab92023a8e7a9ddd7afa5d8b2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 21 Jul 2012 22:49:08 +0000 Subject: [PATCH 02/99] Add Win32 support to the internal threads interface. Also, add several asserts and fix a few typos. FossilOrigin-Name: 793195d37109c75eba84f7190c8fe0b8722f76f7 --- manifest | 21 +++++------- manifest.uuid | 2 +- src/os_win.c | 4 +++ src/sqliteInt.h | 7 ++++ src/threads.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 7e6d5fef2a..dcf166e469 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinternal\sinterface\sthat\sallows\sthe\scode\sto\stake\sadvantage\sof\smultiple\ncores\sby\spushing\ssubcomputations\soff\sinto\sseparate\sthreads.\s\sThe\sinterface\nis\snot\scurrently\sused. -D 2012-07-21T19:40:42.441 +C Add\sWin32\ssupport\sto\sthe\sinternal\sthreads\sinterface.\s\sAlso,\sadd\sseveral\sasserts\sand\sfix\sa\sfew\stypos. +D 2012-07-21T22:49:08.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h c2ebd26a68a4223fe170b003852b97d9e7211498 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d7c96b5d140f550f07345870112fae5d7ef99757 -F src/os_win.c e3d3d3e26b65a35d4293d753137a58510bd3299b +F src/os_win.c 899783012ed47a756cd4358b43ecfa139cf14ace F src/pager.c e381c118b77dc22021a1a59d3fec24815e91df78 F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -180,7 +180,7 @@ F src/select.c f6c4833c4d8e94714761d99013d74f381e084f1d F src/shell.c 076e1c90d594644f36027c8ecff9a392cf2d3a06 F src/sqlite.h.in 310ae7e538883fa1619ab0638c775ce11ad43015 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 273e9d00296d3a76519c0446698c23787d8951f9 +F src/sqliteInt.h 4bb99c73090cde71ea5534d0f9c3ee299a1c10e7 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -231,7 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c df23fc534a54fe7e1ff935628fa14a9318d27ae3 +F src/threads.c 82ea90092f36d02f887e85f7559efc1519f9edd3 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -1006,10 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e4163596339c2166f9c4356ab824fff8bda8d0b0 -R e05d278833a84eb0e1d0c443cf6ae5d1 -T *branch * threads -T *sym-threads * -T -sym-trunk * -U drh -Z d49696325d34c649ba05b5124d6547e7 +P 0e4d977a4a07d6de50acbf022c7dd947998b8d96 +R a23a964d0745aaa13c7e5378646293f9 +U mistachkin +Z d6738fe175fa5cec50ba76dfb911d019 diff --git a/manifest.uuid b/manifest.uuid index 250952bf9a..25fa463af5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e4d977a4a07d6de50acbf022c7dd947998b8d96 \ No newline at end of file +793195d37109c75eba84f7190c8fe0b8722f76f7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8509e9272d..d3f9671809 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -945,6 +945,10 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +DWORD sqlite3Win32Wait(HANDLE hObject){ + return osWaitForSingleObjectEx(hObject, INFINITE, TRUE); +} + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f0ccc5cd28..559a552de0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3314,4 +3314,11 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); +/* +** Win32 interface +*/ +#if SQLITE_OS_WIN + DWORD sqlite3Win32Wait(HANDLE hObject); +#endif + #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c index ed98015510..dfb44a7c2f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -47,19 +47,25 @@ int sqlite3ThreadCreate( SQLiteThread *p; int rc; - *ppThread = p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_OK; + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; rc = pthread_create(&p->tid, 0, xTask, pIn); if( rc ){ sqlite3_free(p); return SQLITE_ERROR; } + *ppThread = p; return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; + + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; rc = pthread_join(p->tid, ppOut); sqlite3_free(p); @@ -70,6 +76,71 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /******************************** End Unix Pthreads *************************/ +/********************************* Win32 Threads ****************************/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINRT + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + uintptr_t tid; /* The thread handle */ + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Thread procedure Win32 compatibility shim */ +static void sqlite3ThreadProc( + void *pArg /* IN: Pointer to the SQLiteThread structure */ +){ + SQLiteThread *p = (SQLiteThread *)pArg; + + assert( p!=0 ); + assert( p->xTask!=0 ); + p->pResult = p->xTask(p->pIn); + _endthread(); +} + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->xTask = xTask; p->pIn = pIn; + p->tid = _beginthread(sqlite3ThreadProc, 0, p); + if( p->tid==(uintptr_t)-1 ){ + sqlite3_free(p); + return SQLITE_ERROR; + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + DWORD rc; + + assert( ppOut!=0 ); + if( p==0 ) return SQLITE_NOMEM; + rc = sqlite3Win32Wait((HANDLE)p->tid); + if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; + sqlite3_free(p); + return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; +} + +#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINRT */ +/******************************** End Win32 Threads *************************/ + + /********************************* Single-Threaded **************************/ #ifndef SQLITE_THREADS_IMPLEMENTED /* @@ -92,7 +163,11 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; - *ppThread = p = sqlite3Malloc(sizeof(*p)); + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ p->xTask = xTask; @@ -101,14 +176,16 @@ int sqlite3ThreadCreate( p->xTask = 0; p->pResult = xTask(pIn); } - return p; + *ppThread = p; + return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ - *ppOut = = p->xTask(p->pIn); + *ppOut = p->xTask(p->pIn); }else{ *ppOut = p->pResult; } From 572df3b1e4a307e2657579658c041f567442f50c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Jul 2012 02:00:38 +0000 Subject: [PATCH 03/99] Enhance implementation of the Win32 thread wait function. FossilOrigin-Name: 049b04117353c3e163ffc87916cbe121403a2821 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dcf166e469..da25a187c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sWin32\ssupport\sto\sthe\sinternal\sthreads\sinterface.\s\sAlso,\sadd\sseveral\sasserts\sand\sfix\sa\sfew\stypos. -D 2012-07-21T22:49:08.403 +C Enhance\simplementation\sof\sthe\sWin32\sthread\swait\sfunction. +D 2012-07-23T02:00:38.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h c2ebd26a68a4223fe170b003852b97d9e7211498 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d7c96b5d140f550f07345870112fae5d7ef99757 -F src/os_win.c 899783012ed47a756cd4358b43ecfa139cf14ace +F src/os_win.c f9f2a4569f9a6d2415170261641e6e3ee6ed8121 F src/pager.c e381c118b77dc22021a1a59d3fec24815e91df78 F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1006,7 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 0e4d977a4a07d6de50acbf022c7dd947998b8d96 -R a23a964d0745aaa13c7e5378646293f9 +P 793195d37109c75eba84f7190c8fe0b8722f76f7 +R 49885bc11b122527dda4cd8bcfe1cfa0 U mistachkin -Z d6738fe175fa5cec50ba76dfb911d019 +Z b0f357746aebd8329bde52b662a0497e diff --git a/manifest.uuid b/manifest.uuid index 25fa463af5..1d07b1a32a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793195d37109c75eba84f7190c8fe0b8722f76f7 \ No newline at end of file +049b04117353c3e163ffc87916cbe121403a2821 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d3f9671809..75d0edcddb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -946,7 +946,10 @@ void sqlite3_win32_sleep(DWORD milliseconds){ } DWORD sqlite3Win32Wait(HANDLE hObject){ - return osWaitForSingleObjectEx(hObject, INFINITE, TRUE); + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; } /* From a806475f86220a2bb19033eb8bf304c1601f4994 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Jul 2012 06:47:30 +0000 Subject: [PATCH 04/99] Add an assert() to help verify the return code from the Win32 thread wait function. FossilOrigin-Name: ed3dc7a89f3416622fcd741ae5fba437929d06d6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index da25a187c6..e4cf469b13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\simplementation\sof\sthe\sWin32\sthread\swait\sfunction. -D 2012-07-23T02:00:38.699 +C Add\san\sassert()\sto\shelp\sverify\sthe\sreturn\scode\sfrom\sthe\sWin32\sthread\swait\sfunction. +D 2012-07-23T06:47:30.102 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -231,7 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 82ea90092f36d02f887e85f7559efc1519f9edd3 +F src/threads.c cde5bd24ab5b33bb694c59f4236ecd56ad8da9b5 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -1006,7 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 793195d37109c75eba84f7190c8fe0b8722f76f7 -R 49885bc11b122527dda4cd8bcfe1cfa0 +P 049b04117353c3e163ffc87916cbe121403a2821 +R 8ad4cfcece99f2543c3fc25ff90ff8da U mistachkin -Z b0f357746aebd8329bde52b662a0497e +Z c7ca4753dded1e68bd82c1792e2c5272 diff --git a/manifest.uuid b/manifest.uuid index 1d07b1a32a..acb6e11d72 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -049b04117353c3e163ffc87916cbe121403a2821 \ No newline at end of file +ed3dc7a89f3416622fcd741ae5fba437929d06d6 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index dfb44a7c2f..61e58f494b 100644 --- a/src/threads.c +++ b/src/threads.c @@ -132,6 +132,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; sqlite3_free(p); return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; From daa3ce4b0805753abb473a216b2057d08326023b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Aug 2012 13:08:15 +0000 Subject: [PATCH 05/99] Update the spellfix virtual table so that all OOM errors are reported out to the application. FossilOrigin-Name: c2cf498513c2633bd2b08372772eaa0f3b3ab25f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_spellfix.c | 32 ++++++++++++++++++++++---------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 20486b8f7e..4e50d4dc33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_DISABLE_FTS4_DEFERRED\scompile\stime\soption. -D 2012-08-20T17:24:48.768 +C Update\sthe\sspellfix\svirtual\stable\sso\sthat\sall\sOOM\serrors\sare\sreported\sout\nto\sthe\sapplication. +D 2012-08-21T13:08:15.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in abd5c10d21d1395f140d9e50ea999df8fa4d6376 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_spellfix.c deab0f9caf853d2ddbee7c4a680ad27621adf1bf +F src/test_spellfix.c 5cc2bfe3fe4b3e6d7b0b0807d36a485a3a15a1ae F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -1011,7 +1011,7 @@ F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P be1faadebd9464f1c7d4cc26104f219ed35384b8 -R 9d3902f63cfac6bb8ee3fe0483a0e736 -U dan -Z a16ab6913bdfea9d4fad9944795f8c23 +P e799222f3b8246e65657a758437914ece7069ba9 +R 41231569c08717226d41316ed2c512be +U drh +Z 09392d5c35b740ca7443d8f9f40e84b9 diff --git a/manifest.uuid b/manifest.uuid index 61af4bab2b..51c86e8c1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e799222f3b8246e65657a758437914ece7069ba9 \ No newline at end of file +c2cf498513c2633bd2b08372772eaa0f3b3ab25f \ No newline at end of file diff --git a/src/test_spellfix.c b/src/test_spellfix.c index ec73b0b3cf..fe6992e535 100644 --- a/src/test_spellfix.c +++ b/src/test_spellfix.c @@ -2508,15 +2508,17 @@ static int spellfix1Filter( */ static int spellfix1Next(sqlite3_vtab_cursor *cur){ spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + rc = SQLITE_OK; if( pCur->iRow < pCur->nRow ){ if( pCur->pFullScan ){ - int rc = sqlite3_step(pCur->pFullScan); + rc = sqlite3_step(pCur->pFullScan); if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; }else{ pCur->iRow++; } } - return SQLITE_OK; + return rc; } /* @@ -2773,25 +2775,35 @@ static sqlite3_module spellfix1Module = { ** Register the various functions and the virtual table. */ static int spellfix1Register(sqlite3 *db){ - int nErr = 0; + int rc = SQLITE_OK; int i; - nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, transliterateSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, editdistSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, phoneticHashSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, scriptCodeSqlFunc, 0, 0); - nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); - nErr += editDist3Install(db); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); + } + if( rc==SQLITE_OK ){ + rc = editDist3Install(db); + } /* Verify sanity of the translit[] table */ for(i=0; i Date: Mon, 17 Mar 2014 15:43:05 +0000 Subject: [PATCH 06/99] Add an experimental multi-threaded capability to vdbesorter.c. FossilOrigin-Name: ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 --- manifest | 22 +- manifest.uuid | 2 +- src/threads.c | 1 - src/vdbesort.c | 812 +++++++++++++++++++++++++++++++++---------------- 4 files changed, 567 insertions(+), 270 deletions(-) diff --git a/manifest b/manifest index 4f0c8a0be4..2de90048c0 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2014-03-13T15:41:09.146 +C Add\san\sexperimental\smulti-threaded\scapability\sto\svdbesorter.c. +D 2014-03-17T15:43:05.543 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 153eb9b9725bc7b8e4dbe963219298e0c4a644b0 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 -F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 w README +F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION c3b0d47c3c5cf25c5bd4ff9e6f3af2f9d7934ea6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 @@ -108,17 +108,17 @@ F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 -F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d w src/test_fuzzer.c +F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed w src/test_spellfix.c +F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e -F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 w src/test_wholenumber.c +F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 2b918d1f4f0b0831e8f41c49bcaa097f01490120 +F src/threads.c cde9d885fd562b5427f89a42a8829085f88b17df F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 1d973fcd0e00c77836e9d505e7c45df02601ffc2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c92b0fe1371e7c20a5fbdf5fa96e30da14c40880 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 -R c6806fc7f1eff740ed4af1293ffc55d0 +P d17231b63d48c1f9c4dee109c90cec112e2f0fd4 +R 3c79acd4eecb6926efd889b756a27898 U dan -Z b3b09b0f0acb184b570781ded35d947c +Z 226b72b7faaae8e6d7b6586f32d5c5a0 diff --git a/manifest.uuid b/manifest.uuid index c9d158cd10..bf397f3d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d17231b63d48c1f9c4dee109c90cec112e2f0fd4 \ No newline at end of file +ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 33781a7aac..7cc964250d 100644 --- a/src/threads.c +++ b/src/threads.c @@ -47,7 +47,6 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; - int rc; assert( ppThread!=0 ); assert( xTask!=0 ); diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..e84a33fccc 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -20,9 +20,83 @@ typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterThread SorterThread; typedef struct SorterRecord SorterRecord; +typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; + +/* +** Maximum number of threads to use. Setting this value to 1 forces all +** operations to be single-threaded. +*/ +#ifndef SQLITE_MAX_SORTER_THREAD +# define SQLITE_MAX_SORTER_THREAD 1 +#endif + +/* +** Candidate values for SorterThread.eWork +*/ +#define SORTER_THREAD_SORT 1 +#define SORTER_THREAD_TO_PMA 2 +#define SORTER_THREAD_CONS 3 + +/* +** Much of the work performed in this module to sort the list of records is +** broken down into smaller units that may be peformed in parallel. In order +** to perform such a unit of work, an instance of the following structure +** is configured and passed to vdbeSorterThreadMain() - either directly by +** the main thread or via a background thread. +** +** Exactly SQLITE_MAX_SORTER_THREAD instances of this structure are allocated +** as part of each VdbeSorter object. Instances are never allocated any other +** way. +** +** When a background thread is launched to perform work, SorterThread.bDone +** is set to 0 and the SorterThread.pThread variable set to point to the +** thread handle. SorterThread.bDone is set to 1 (to indicate to the main +** thread that joining SorterThread.pThread will not block) before the thread +** exits. SorterThread.pThread and bDone are always cleared after the +** background thread has been joined. +** +** One object (specifically, VdbeSorter.aThread[SQLITE_MAX_SORTER_THREAD-1]) +** is reserved for the foreground thread. +** +** The nature of the work performed is determined by SorterThread.eWork, +** as follows: +** +** SORTER_THREAD_SORT: +** Sort the linked list of records at SorterThread.pList. +** +** SORTER_THREAD_TO_PMA: +** Sort the linked list of records at SorterThread.pList, and write +** the results to a new PMA in temp file SorterThread.pTemp1. Open +** the temp file if it is not already open. +** +** SORTER_THREAD_CONS: +** Merge existing PMAs until SorterThread.nConsolidate or fewer +** remain in temp file SorterThread.pTemp1. +*/ +struct SorterThread { + SQLiteThread *pThread; /* Thread handle, or NULL */ + int bDone; /* Set to true by pThread when finished */ + + sqlite3_vfs *pVfs; /* VFS used to open temporary files */ + KeyInfo *pKeyInfo; /* How to compare records */ + UnpackedRecord *pUnpacked; /* Space to unpack a record */ + int pgsz; /* Main database page size */ + + u8 eWork; /* One of the SORTER_THREAD_* constants */ + int nConsolidate; /* For THREAD_CONS, max final PMAs */ + SorterRecord *pList; /* List of records for pThread to sort */ + int nInMemory; /* Expected size of PMA based on pList */ + + int nPMA; /* Number of PMAs currently in pTemp1 */ + i64 iTemp1Off; /* Offset to write to in pTemp1 */ + sqlite3_file *pTemp1; /* File to write PMAs to, or NULL */ +}; + + /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: ** @@ -92,19 +166,24 @@ typedef struct FileWriter FileWriter; ** key comparison operations are required, where N is the number of segments ** being merged (rounded up to the next power of 2). */ -struct VdbeSorter { - i64 iWriteOff; /* Current write offset within file pTemp1 */ - i64 iReadOff; /* Current read offset within file pTemp1 */ - int nInMemory; /* Current size of pRecord list as PMA */ +struct SorterMerger { int nTree; /* Used size of aTree/aIter (power of 2) */ - int nPMA; /* Number of PMAs stored in pTemp1 */ + int *aTree; /* Current state of incremental merge */ + VdbeSorterIter *aIter; /* Array of iterators to merge data from */ +}; + +/* +** Main sorter structure. A single instance of this is allocated for each +** sorter cursor created by the VDBE. +*/ +struct VdbeSorter { + int nInMemory; /* Current size of pRecord list as PMA */ int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ - VdbeSorterIter *aIter; /* Array of iterators to merge */ - int *aTree; /* Current state of incremental merge */ - sqlite3_file *pTemp1; /* PMA file 1 */ + int bUsePMA; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - UnpackedRecord *pUnpacked; /* Used to unpack keys */ + SorterMerger *pMerger; /* For final merge of PMAs (by caller) */ + SorterThread aThread[SQLITE_MAX_SORTER_THREAD]; }; /* @@ -150,7 +229,8 @@ struct SorterRecord { SorterRecord *pNext; }; -/* Minimum allowable value for the VdbeSorter.nWorking variable */ +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. */ #define SORTER_MIN_WORKING 10 /* Maximum number of segments to merge in a single pass. */ @@ -160,9 +240,9 @@ struct SorterRecord { ** Free all memory belonging to the VdbeSorterIter object passed as the second ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ - sqlite3DbFree(db, pIter->aAlloc); - sqlite3DbFree(db, pIter->aBuffer); +static void vdbeSorterIterZero(VdbeSorterIter *pIter){ + sqlite3_free(pIter->aAlloc); + sqlite3_free(pIter->aBuffer); memset(pIter, 0, sizeof(VdbeSorterIter)); } @@ -176,7 +256,6 @@ static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ ** next call to this function. */ static int vdbeSorterIterRead( - sqlite3 *db, /* Database handle (for malloc) */ VdbeSorterIter *p, /* Iterator */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ @@ -222,11 +301,13 @@ static int vdbeSorterIterRead( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; while( nByte>nNew ) nNew = nNew*2; - p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); - if( !p->aAlloc ) return SQLITE_NOMEM; + aNew = sqlite3Realloc(p->aAlloc, nNew); + if( !aNew ) return SQLITE_NOMEM; p->nAlloc = nNew; + p->aAlloc = aNew; } /* Copy as much data as is available in the buffer into the start of @@ -244,7 +325,7 @@ static int vdbeSorterIterRead( nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + rc = vdbeSorterIterRead(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -261,7 +342,7 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ +static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ int iBuf; iBuf = p->iReadOff % p->nBuffer; @@ -271,7 +352,7 @@ static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ u8 aVarint[16], *a; int i = 0, rc; do{ - rc = vdbeSorterIterRead(db, p, 1, &a); + rc = vdbeSorterIterRead(p, 1, &a); if( rc ) return rc; aVarint[(i++)&0xf] = a[0]; }while( (a[0]&0x80)!=0 ); @@ -286,23 +367,20 @@ static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ - VdbeSorterIter *pIter /* Iterator to advance */ -){ +static int vdbeSorterIterNext(VdbeSorterIter *pIter){ int rc; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ - vdbeSorterIterZero(db, pIter); + vdbeSorterIterZero(pIter); return SQLITE_OK; } - rc = vdbeSorterIterVarint(db, pIter, &nRec); + rc = vdbeSorterIterVarint(pIter, &nRec); if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); + rc = vdbeSorterIterRead(pIter, (int)nRec, &pIter->aKey); } return rc; @@ -315,26 +393,23 @@ static int vdbeSorterIterNext( ** PMA is empty). */ static int vdbeSorterIterInit( - sqlite3 *db, /* Database handle */ - const VdbeSorter *pSorter, /* Sorter object */ - i64 iStart, /* Start offset in pFile */ + SorterThread *pThread, /* Thread context */ + i64 iStart, /* Start offset in pThread->pTemp1 */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc = SQLITE_OK; - int nBuf; + int nBuf = pThread->pgsz; - nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - - assert( pSorter->iWriteOff>iStart ); + assert( pThread->iTemp1Off>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pSorter->pTemp1; + pIter->pFile = pThread->pTemp1; pIter->iReadOff = iStart; pIter->nAlloc = 128; - pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); + pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !pIter->aBuffer ){ rc = SQLITE_NOMEM; @@ -344,26 +419,26 @@ static int vdbeSorterIterInit( iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( (iStart + nRead) > pThread->iTemp1Off ){ + nRead = (int)(pThread->iTemp1Off - iStart); } rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); + u64 nByte; + pIter->iEof = pThread->iTemp1Off; + rc = vdbeSorterIterVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; } } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(db, pIter); + rc = vdbeSorterIterNext(pIter); } return rc; } @@ -385,15 +460,14 @@ static int vdbeSorterIterInit( ** has been allocated and contains an unpacked record that is used as key2. */ static void vdbeSorterCompare( - const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + SorterThread *pThread, /* Thread context (for pKeyInfo) */ int nIgnore, /* Ignore the last nIgnore fields */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ ){ - KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; + KeyInfo *pKeyInfo = pThread->pKeyInfo; + UnpackedRecord *r2 = pThread->pUnpacked; int i; if( pKey2 ){ @@ -420,26 +494,29 @@ static void vdbeSorterCompare( ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ -static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ - VdbeSorter *pSorter = pCsr->pSorter; +static int vdbeSorterDoCompare( + SorterThread *pThread, + SorterMerger *pMerger, + int iOut +){ int i1; int i2; int iRes; VdbeSorterIter *p1; VdbeSorterIter *p2; - assert( iOutnTree && iOut>0 ); + assert( iOutnTree && iOut>0 ); - if( iOut>=(pSorter->nTree/2) ){ - i1 = (iOut - pSorter->nTree/2) * 2; + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; i2 = i1 + 1; }else{ - i1 = pSorter->aTree[iOut*2]; - i2 = pSorter->aTree[iOut*2+1]; + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; } - p1 = &pSorter->aIter[i1]; - p2 = &pSorter->aIter[i2]; + p1 = &pMerger->aIter[i1]; + p2 = &pMerger->aIter[i2]; if( p1->pFile==0 ){ iRes = i2; @@ -447,9 +524,9 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ iRes = i1; }else{ int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); if( res<=0 ){ iRes = i1; @@ -458,7 +535,7 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ } } - pSorter->aTree[iOut] = iRes; + pMerger->aTree[iOut] = iRes; return SQLITE_OK; } @@ -467,22 +544,32 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ + int i; /* Used to iterate through aThread[] */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ - char *d; /* Dummy */ + KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ + int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo); + pCsr->pSorter = pSorter; if( pSorter==0 ){ return SQLITE_NOMEM; } - - pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); - if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; - assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + pKeyInfo = (KeyInfo*)&pSorter[1]; + memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); + pKeyInfo->db = 0; + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + + for(i=0; iaThread[i]; + pThread->pKeyInfo = pKeyInfo; + pThread->pVfs = db->pVfs; + pThread->pgsz = pgsz; + } if( !sqlite3TempInMemory(db) ){ - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; if( mxCachepUnpacked); + vdbeSorterRecordFree(0, pThread->pList); + if( pThread->pTemp1 ){ + sqlite3OsCloseFree(pThread->pTemp1); + } + memset(pThread, 0, sizeof(SorterThread)); +} + +/* +** Join all threads. +*/ +static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ + int rc = rcin; + int i; + for(i=0; iaThread[i]; + if( pThread->pThread ){ + void *pRet; + int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); + pThread->pThread = 0; + pThread->bDone = 0; + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + } + } + return rc; +} + +/* +** Allocate a new SorterMerger object with space for nIter iterators. +*/ +static SorterMerger *vdbeSorterMergerNew(int nIter){ + int N = 2; /* Smallest power of two >= nIter */ + int nByte; /* Total bytes of space to allocate */ + SorterMerger *pNew; /* Pointer to allocated object to return */ + + assert( nIter<=SORTER_MAX_MERGE_COUNT ); + while( NnTree = N; + pNew->aIter = (VdbeSorterIter*)&pNew[1]; + pNew->aTree = (int*)&pNew->aIter[N]; + } + + return pNew; +} + +/* +** Free the SorterMerger object passed as the only argument. +*/ +static void vdbeSorterMergerFree(SorterMerger *pMerger){ + if( pMerger ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(&pMerger->aIter[i]); + } + sqlite3_free(pMerger); + } +} + /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); + int i; + vdbeSorterJoinAll(pSorter, SQLITE_OK); + for(i=0; iaThread[i]; + vdbeSorterThreadCleanup(db, pThread); } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - } - vdbeSorterRecordFree(db, pSorter->pRecord); - sqlite3DbFree(db, pSorter->pUnpacked); + + vdbeSorterRecordFree(0, pSorter->pRecord); + vdbeSorterMergerFree(pSorter->pMerger); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -532,9 +684,9 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ +static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + return sqlite3OsOpenMalloc(pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy @@ -546,7 +698,7 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - const VdbeCursor *pCsr, /* For pKeyInfo */ + SorterThread *pThread, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -557,7 +709,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pThread, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); if( res<=0 ){ *pp = p1; pp = &p1->pNext; @@ -576,27 +728,26 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK -** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error -** occurs. +** Sort the linked list of records headed at pThread->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** an error occurs. */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ +static int vdbeSorterSort(SorterThread *pThread){ int i; SorterRecord **aSlot; SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pSorter->pRecord; + p = pThread->pList; while( p ){ SorterRecord *pNext = p->pNext; p->pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pThread, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -605,9 +756,9 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pThread, p, aSlot[i], &p); } - pSorter->pRecord = p; + pThread->pList = p; sqlite3_free(aSlot); return SQLITE_OK; @@ -617,15 +768,13 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ ** Initialize a file-writer object. */ static void fileWriterInit( - sqlite3 *db, /* Database (for malloc) */ sqlite3_file *pFile, /* File to write to */ FileWriter *p, /* Object to populate */ + int nBuf, /* Buffer size */ i64 iStart /* Offset of pFile to begin writing at */ ){ - int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - memset(p, 0, sizeof(FileWriter)); - p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ p->eFWErr = SQLITE_NOMEM; }else{ @@ -673,7 +822,7 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ +static int fileWriterFinish(FileWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFile, @@ -682,7 +831,7 @@ static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ ); } *piEof = (p->iWriteOff + p->iBufEnd); - sqlite3DbFree(db, p->aBuffer); + sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(FileWriter)); return rc; @@ -712,43 +861,234 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(SorterThread *pThread){ int rc = SQLITE_OK; /* Return code */ - VdbeSorter *pSorter = pCsr->pSorter; - FileWriter writer; + FileWriter writer; /* Object used to write to the file */ memset(&writer, 0, sizeof(FileWriter)); - - if( pSorter->nInMemory==0 ){ - assert( pSorter->pRecord==0 ); - return rc; - } - - rc = vdbeSorterSort(pCsr); + assert( pThread->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); - assert( rc!=SQLITE_OK || pSorter->pTemp1 ); - assert( pSorter->iWriteOff==0 ); - assert( pSorter->nPMA==0 ); + if( pThread->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(pThread->pVfs, &pThread->pTemp1); + assert( rc!=SQLITE_OK || pThread->pTemp1 ); + assert( pThread->iTemp1Off==0 ); + assert( pThread->nPMA==0 ); } if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); - pSorter->nPMA++; - fileWriterWriteVarint(&writer, pSorter->nInMemory); - for(p=pSorter->pRecord; p; p=pNext){ + fileWriterInit(pThread->pTemp1, &writer, pThread->pgsz, pThread->iTemp1Off); + pThread->nPMA++; + fileWriterWriteVarint(&writer, pThread->nInMemory); + for(p=pThread->pList; p; p=pNext){ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + sqlite3_free(p); + } + pThread->pList = p; + rc = fileWriterFinish(&writer, &pThread->iTemp1Off); + } + + return rc; +} + +/* +** Advance the SorterMerger iterator passed as the second argument to +** the next entry. Set *pbEof to true if this means the iterator has +** reached EOF. +** +** Return SQLITE_OK if successful or an error code if an error occurs. +*/ +static int vdbeSorterNext( + SorterThread *pThread, + SorterMerger *pMerger, + int *pbEof +){ + int rc; + int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ + + /* Advance the current iterator */ + rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); + + /* Update contents of aTree[] */ + for(i=(pMerger->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pThread, pMerger, i); + } + + *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); + return rc; +} + +/* +** The main routine for sorter-thread operations. +*/ +static void *vdbeSorterThreadMain(void *pCtx){ + int rc = SQLITE_OK; + SorterThread *pThread = (SorterThread*)pCtx; + + assert( pThread->eWork==SORTER_THREAD_SORT + || pThread->eWork==SORTER_THREAD_TO_PMA + || pThread->eWork==SORTER_THREAD_CONS + ); + assert( pThread->bDone==0 ); + + if( pThread->pUnpacked==0 ){ + char *pFree; + pThread->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pThread->pKeyInfo, 0, 0, &pFree + ); + assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ){ + rc = SQLITE_NOMEM; + goto thread_out; + } + } + + if( pThread->eWork==SORTER_THREAD_CONS ){ + assert( pThread->pList==0 ); + while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ + int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); + sqlite3_file *pTemp2 = 0; /* Second temp file to use */ + SorterMerger *pMerger; /* Object for reading/merging PMA data */ + i64 iReadOff = 0; /* Offset in pTemp1 to read from */ + i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ + int i; + + /* Allocate a merger object to merge PMAs together. */ + pMerger = vdbeSorterMergerNew(nIter); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + break; + } + + /* Open a second temp file to write merged data to */ + rc = vdbeSorterOpenTempFile(pThread->pVfs, &pTemp2); + if( rc!=SQLITE_OK ){ + vdbeSorterMergerFree(pMerger); + break; + } + + /* This loop runs once for each output PMA. Each output PMA is made + ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ + for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + FileWriter writer; /* Object for writing data to pTemp2 */ + i64 nOut = 0; /* Bytes of data in output PMA */ + int bEof = 0; + int rc2; + + /* Configure the merger object to read and merge data from the next + ** SORTER_MAX_MERGE_COUNT PMAs in pTemp1 (or from all remaining PMAs, + ** if that is fewer). */ + int iIter; + for(iIter=0; iIteraIter[iIter]; + rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nOut); + iReadOff = pIter->iEof; + if( iReadOff>=pThread->iTemp1Off || rc!=SQLITE_OK ) break; + } + for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ + rc = vdbeSorterDoCompare(pThread, pMerger, iIter); + } + + fileWriterInit(pTemp2, &writer, pThread->pgsz, iWriteOff); + fileWriterWriteVarint(&writer, nOut); + while( rc==SQLITE_OK && bEof==0 ){ + VdbeSorterIter *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; + assert( pIter->pFile!=0 ); /* pIter is not at EOF */ + fileWriterWriteVarint(&writer, pIter->nKey); + fileWriterWrite(&writer, pIter->aKey, pIter->nKey); + rc = vdbeSorterNext(pThread, pMerger, &bEof); + } + rc2 = fileWriterFinish(&writer, &iWriteOff); + if( rc==SQLITE_OK ) rc = rc2; + } + + vdbeSorterMergerFree(pMerger); + sqlite3OsCloseFree(pThread->pTemp1); + pThread->pTemp1 = pTemp2; + pThread->nPMA = (i / SORTER_MAX_MERGE_COUNT); + pThread->iTemp1Off = iWriteOff; + } + }else{ + /* Sort the pThread->pList list */ + rc = vdbeSorterSort(pThread); + + /* If required, write the list out to a PMA. */ + if( rc==SQLITE_OK && pThread->eWork==SORTER_THREAD_TO_PMA ){ +#ifdef SQLITE_DEBUG + i64 nExpect = pThread->nInMemory + + sqlite3VarintLen(pThread->nInMemory) + + pThread->iTemp1Off; +#endif + rc = vdbeSorterListToPMA(pThread); + assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); + } + } + + thread_out: + pThread->bDone = 1; + return SQLITE_INT_TO_PTR(rc); +} + +/* +** Run the activity scheduled by the object passed as the only argument +** in the current thread. +*/ +static int vdbeSorterRunThread(SorterThread *pThread){ + int rc = SQLITE_PTR_TO_INT( vdbeSorterThreadMain((void*)pThread) ); + assert( pThread->bDone ); + pThread->bDone = 0; + return rc; +} + +/* +** Flush the current contents of VdbeSorter.pRecord to a new PMA, possibly +** using a background thread. +** +** If argument bFg is non-zero, the operation always uses the calling thread. +*/ +static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc = SQLITE_OK; + int i; + SorterThread *pThread; /* Thread context used to create new PMA */ + + pSorter->bUsePMA = 1; + for(i=0; ALWAYS( iaThread[i]; + if( pThread->bDone ){ + void *pRet; + assert( pThread->pThread ); + rc = sqlite3ThreadJoin(pThread->pThread, &pRet); + pThread->pThread = 0; + pThread->bDone = 0; + if( rc==SQLITE_OK ){ + rc = SQLITE_PTR_TO_INT(pRet); + } + } + if( pThread->pThread==0 ) break; + } + + if( rc==SQLITE_OK ){ + assert( pThread->pThread==0 && pThread->bDone==0 ); + pThread->eWork = SORTER_THREAD_TO_PMA; + pThread->pList = pSorter->pRecord; + pThread->nInMemory = pSorter->nInMemory; + pSorter->nInMemory = 0; + pSorter->pRecord = 0; + + if( bFg || i<(SQLITE_MAX_SORTER_THREAD-1) ){ + void *pCtx = (void*)pThread; + rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); + }else{ + /* Use the foreground thread for this operation */ + rc = vdbeSorterRunThread(pThread); } - pSorter->pRecord = p; - rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; @@ -769,7 +1109,7 @@ int sqlite3VdbeSorterWrite( assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3Malloc(pVal->n + sizeof(SorterRecord)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -793,48 +1133,22 @@ int sqlite3VdbeSorterWrite( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; -#endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + rc = vdbeSorterFlushPMA(db, pCsr, 0); } return rc; } /* -** Helper function for sqlite3VdbeSorterRewind(). +** Return the total number of PMAs in all temporary files. */ -static int vdbeSorterInitMerge( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Cursor handle for this sorter */ - i64 *pnByte /* Sum of bytes in all opened PMAs */ -){ - VdbeSorter *pSorter = pCsr->pSorter; - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterator through aIter[] */ - i64 nByte = 0; /* Total bytes in all opened PMAs */ - - /* Initialize the iterators. */ - for(i=0; iaIter[i]; - rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); - pSorter->iReadOff = pIter->iEof; - assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); - if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; +static int vdbeSorterCountPMA(VdbeSorter *pSorter){ + int nPMA = 0; + int i; + for(i=0; iaThread[i].nPMA; } - - /* Initialize the aTree[] array. */ - for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pCsr, i); - } - - *pnByte = nByte; - return rc; + return nPMA; } /* @@ -843,107 +1157,98 @@ static int vdbeSorterInitMerge( */ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; - int rc; /* Return code */ - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - i64 iWrite2 = 0; /* Write offset for pTemp2 */ - int nIter; /* Number of iterators used */ - int nByte; /* Bytes of space required for aIter/aTree */ - int N = 2; /* Power of 2 >= nIter */ + int rc = SQLITE_OK; /* Return code */ assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ - if( pSorter->nPMA==0 ){ - *pbEof = !pSorter->pRecord; - assert( pSorter->aTree==0 ); - return vdbeSorterSort(pCsr); + if( pSorter->bUsePMA==0 ){ + if( pSorter->pRecord ){ + SorterThread *pThread = &pSorter->aThread[0]; + *pbEof = 0; + pThread->pList = pSorter->pRecord; + pThread->eWork = SORTER_THREAD_SORT; + rc = vdbeSorterRunThread(pThread); + pSorter->pRecord = pThread->pList; + pThread->pList = 0; + }else{ + *pbEof = 1; + } + return rc; } /* Write the current in-memory list to a PMA. */ - rc = vdbeSorterListToPMA(db, pCsr); - if( rc!=SQLITE_OK ) return rc; - - /* Allocate space for aIter[] and aTree[]. */ - nIter = pSorter->nPMA; - if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; - assert( nIter>0 ); - while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); - if( !pSorter->aIter ) return SQLITE_NOMEM; - pSorter->aTree = (int *)&pSorter->aIter[N]; - pSorter->nTree = N; - - do { - int iNew; /* Index of new, merged, PMA */ - - for(iNew=0; - rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; - iNew++ - ){ - int rc2; /* Return code from fileWriterFinish() */ - FileWriter writer; /* Object used to write to disk */ - i64 nWrite; /* Number of bytes in new PMA */ - - memset(&writer, 0, sizeof(FileWriter)); - - /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, - ** initialize an iterator for each of them and break out of the loop. - ** These iterators will be incrementally merged as the VDBE layer calls - ** sqlite3VdbeSorterNext(). - ** - ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, - ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs - ** are merged into a single PMA that is written to file pTemp2. - */ - rc = vdbeSorterInitMerge(db, pCsr, &nWrite); - assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); - if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - } - - /* Open the second temp file, if it is not already open. */ - if( pTemp2==0 ){ - assert( iWrite2==0 ); - rc = vdbeSorterOpenTempFile(db, &pTemp2); - } - - if( rc==SQLITE_OK ){ - int bEof = 0; - fileWriterInit(db, pTemp2, &writer, iWrite2); - fileWriterWriteVarint(&writer, nWrite); - while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - assert( pIter->pFile ); - - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); - } - rc2 = fileWriterFinish(db, &writer, &iWrite2); - if( rc==SQLITE_OK ) rc = rc2; - } - } - - if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - }else{ - sqlite3_file *pTmp = pSorter->pTemp1; - pSorter->nPMA = iNew; - pSorter->pTemp1 = pTemp2; - pTemp2 = pTmp; - pSorter->iWriteOff = iWrite2; - pSorter->iReadOff = 0; - iWrite2 = 0; - } - }while( rc==SQLITE_OK ); - - if( pTemp2 ){ - sqlite3OsCloseFree(pTemp2); + if( pSorter->pRecord ){ + rc = vdbeSorterFlushPMA(db, pCsr, 1); + } + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge + ** some of them together so that this is no longer the case. */ + assert( SORTER_MAX_MERGE_COUNT>=SQLITE_MAX_SORTER_THREAD ); + if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ + int i; + for(i=0; rc==SQLITE_OK && iaThread[i]; + if( pThread->pTemp1 ){ + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/SQLITE_MAX_SORTER_THREAD; + pThread->eWork = SORTER_THREAD_CONS; + + if( i<(SQLITE_MAX_SORTER_THREAD-1) ){ + void *pCtx = (void*)pThread; + rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); + }else{ + rc = vdbeSorterRunThread(pThread); + } + } + } + } + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + /* Assuming no errors have occurred, set up a merger structure to read + ** and merge all remaining PMAs. */ + assert( pSorter->pMerger==0 ); + if( rc==SQLITE_OK ){ + int nIter = 0; /* Number of iterators used */ + int i; + SorterMerger *pMerger; + for(i=0; iaThread[i].nPMA; + } + + pSorter->pMerger = pMerger = vdbeSorterMergerNew(nIter); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + }else{ + int iIter = 0; + int iThread = 0; + for(iThread=0; iThreadaThread[iThread]; + for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ + i64 nDummy = 0; + VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; + rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nDummy); + iReadOff = pIter->iEof; + } + } + + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(&pSorter->aThread[0], pMerger, i); + } + } + } + + if( rc==SQLITE_OK ){ + *pbEof = (pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]].pFile==0); } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); return rc; } @@ -954,16 +1259,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->aTree ){ - int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ - - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } - - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + if( pSorter->pMerger ){ + rc = vdbeSorterNext(&pSorter->aThread[0], pSorter->pMerger, pbEof); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; @@ -984,9 +1281,9 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->aTree ){ + if( pSorter->pMerger ){ VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; *pnKey = pIter->nKey; pKey = pIter->aKey; }else{ @@ -1031,9 +1328,10 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; + SorterThread *pMain = &pSorter->aThread[0]; void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } From f45f2326a28870f9533d5e75f8daa7562708b96a Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Mar 2014 17:45:03 +0000 Subject: [PATCH 07/99] Use only a single OP_MakeRecord instead of two when constructing entries to go onto a sorter. FossilOrigin-Name: d696cdedacd39075aa7fc407ab7c7e50f01d9f39 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/select.c | 82 ++++++++++++++++++++++++++----------------------- test/tester.tcl | 2 ++ 4 files changed, 53 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index d2ff62a057..1724c9322b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sOFFSET-on-query-without-FROM\sfix\sfrom\strunk. -D 2014-03-21T18:45:19.215 +C Use\sonly\sa\ssingle\sOP_MakeRecord\sinstead\sof\stwo\swhen\sconstructing\sentries\nto\sgo\sonto\sa\ssorter. +D 2014-03-23T17:45:03.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 596b8098be41e6256968f167d1d8ece2be08d082 +F src/select.c b1e0ac15d846c1d584c26f95bd92e887313b55df F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -845,7 +845,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744 +F test/tester.tcl bc0889a2f86d9c17307992ca1e70391794780265 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e70cfa28aa393661ccc742ecd5e672d807bdd0a9 179ef81648b0ad557df78b7712f216b876b6fb65 -R 654a6218c0c034c3e6462d9f9d0a06b2 +P 71e9ae72c272dc86720b2bfe719f57de437c400b +R d93a4c6f8aefee44f30ed73f4e05ae09 U drh -Z 7a7ab7eaa2946af0174662f0ca0e6009 +Z 5ef6faddb00682067ed8bb051e6b9892 diff --git a/manifest.uuid b/manifest.uuid index d1f85a0601..b62e4b84bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71e9ae72c272dc86720b2bfe719f57de437c400b \ No newline at end of file +d696cdedacd39075aa7fc407ab7c7e50f01d9f39 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f81a5ef3e7..6a3dff28a9 100644 --- a/src/select.c +++ b/src/select.c @@ -455,26 +455,29 @@ static KeyInfo *keyInfoFromExprList( ); /* -** Insert code into "v" that will push the record in register regData -** into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int nData /* Number of elements in the data array */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); - int nOBSat = pSort->nOBSat; - int op; + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ + int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -509,7 +512,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + sqlite3ReleaseTempRange(pParse, regBase, nBase); } if( pSelect->iLimit ){ int addr1, addr2; @@ -773,7 +776,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1); + pushOntoSorter(pParse, pSort, p, r1, 1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -799,7 +802,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -825,7 +828,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -839,10 +842,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pSort, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -1129,6 +1129,10 @@ static void generateSortTail( int regRow; int regRowid; int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + u8 p5; /* p5 parameter for 1st OP_Column */ + int i; if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1142,26 +1146,35 @@ static void generateSortTail( pseudoTab = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; - int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + iSortTab = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + p5 = OPFLAG_CLEARCACHE; }else{ if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + iSortTab = iTab; + p5 = 0; + } + for(i=0; iiSdst+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); @@ -1210,9 +1215,10 @@ static void generateSortTail( break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -1223,9 +1229,6 @@ static void generateSortTail( } if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); - } } /* @@ -4760,8 +4763,9 @@ int sqlite3Select( sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ sSort.addrSortIndex = -1; } diff --git a/test/tester.tcl b/test/tester.tcl index 1c4e93937c..4d55042b43 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1076,6 +1076,7 @@ proc explain_i {sql {db db}} { foreach opcode { Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind NoConflict Next Prev VNext VPrev VFilter + SorterSort SorterNext } { set color($opcode) $B } @@ -1098,6 +1099,7 @@ proc explain_i {sql {db db}} { if {$opcode=="Next" || $opcode=="Prev" || $opcode=="VNext" || $opcode=="VPrev" + || $opcode=="SorterNext" } { for {set i $p2} {$i<$addr} {incr i} { incr x($i) 2 From 70f624c3a9ed4a9d1efc61ddd3a6744d741be886 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 01:43:50 +0000 Subject: [PATCH 08/99] Further enhancements to geneverated VDBE code for ORDER BY. FossilOrigin-Name: e7188fad87ec82d36a39b80ccaf8006bf45a9bcd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 140d5c124c..979df5c0f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C merge\sfixes\sfrom\strunk -D 2014-03-23T18:47:00.384 +C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. +D 2014-03-24T01:43:50.641 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 09fef04ec0746d168ddcff37031ee804ac19dd0e +F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d696cdedacd39075aa7fc407ab7c7e50f01d9f39 641408a1395bfc911ca619ef9e5f073b913d856b -R e64080ab6e1cc6573ca05b14626c10fb +P faf7f9caf526ab33a6fdb9c89b45a0483510db21 +R 55d45b1a91856051dc771a7feb58050e U drh -Z 0df19c139e23d451d51cd0e8cf9cddd3 +Z 450407adee455925374bbdaef08e9b36 diff --git a/manifest.uuid b/manifest.uuid index 49e295d769..61b2c1f687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faf7f9caf526ab33a6fdb9c89b45a0483510db21 \ No newline at end of file +e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1acc6d1020..5ececf1540 100644 --- a/src/select.c +++ b/src/select.c @@ -473,8 +473,7 @@ static void pushOntoSorter( int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -1127,7 +1126,6 @@ static void generateSortTail( int addr; int addrOnce = 0; int iTab; - int pseudoTab = 0; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; @@ -1138,6 +1136,9 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1148,8 +1149,6 @@ static void generateSortTail( iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; @@ -1180,6 +1179,7 @@ static void generateSortTail( for(i=0; i Date: Mon, 24 Mar 2014 02:20:53 +0000 Subject: [PATCH 09/99] Remove a pointless OP_Once operation in ORDER BY clauses with LIMIT. FossilOrigin-Name: e6c59d23316c83b318b1a94d9b28a5d321737fa5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 979df5c0f1..e046917d18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. -D 2014-03-24T01:43:50.641 +C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. +D 2014-03-24T02:20:53.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b +F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P faf7f9caf526ab33a6fdb9c89b45a0483510db21 -R 55d45b1a91856051dc771a7feb58050e +P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd +R 7cf4844b34fb28597f11c358a9ceae37 U drh -Z 450407adee455925374bbdaef08e9b36 +Z 5144b0c0a9c74c80111921622c3f3ee8 diff --git a/manifest.uuid b/manifest.uuid index 61b2c1f687..0c4d7e566f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file +e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 5ececf1540..013feb019f 100644 --- a/src/select.c +++ b/src/select.c @@ -1144,7 +1144,6 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); @@ -1161,6 +1160,9 @@ static void generateSortTail( if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); @@ -1169,10 +1171,8 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; }else{ - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); iSortTab = iTab; p5 = 0; } From 3f802ebce2d497fce3ba9330fb8bc4c0ab1cf539 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 09:34:58 +0000 Subject: [PATCH 10/99] Remove an unnecessary temporary register allocation. FossilOrigin-Name: 5d506743f541b022cde04a9606baa4680cdfd70b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e046917d18..feb6340a70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. -D 2014-03-24T02:20:53.278 +C Remove\san\sunnecessary\stemporary\sregister\sallocation. +D 2014-03-24T09:34:58.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 +F src/select.c a088183774c4efae4105076355ac4010c62390a8 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd -R 7cf4844b34fb28597f11c358a9ceae37 -U drh -Z 5144b0c0a9c74c80111921622c3f3ee8 +P e6c59d23316c83b318b1a94d9b28a5d321737fa5 +R ab9aa4f9f7d5ea8c370ac77061403538 +U dan +Z 631d6a70f4aeb5d6a455a81761c8aefd diff --git a/manifest.uuid b/manifest.uuid index 0c4d7e566f..36b51c0a51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file +5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 013feb019f..641a54faa2 100644 --- a/src/select.c +++ b/src/select.c @@ -1146,7 +1146,6 @@ static void generateSortTail( sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ regRowid = 0; regRow = pDest->iSdst; From fd0a2f9756745401f68fc0e76b868c682194d023 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 18:08:15 +0000 Subject: [PATCH 11/99] Avoid unnecessary moving of content between registers during an ORDER BY. FossilOrigin-Name: 4f472accf072d9cb64f209923924b26f21b13d27 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 38 +++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index feb6340a70..a1b753f05e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\stemporary\sregister\sallocation. -D 2014-03-24T09:34:58.396 +C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. +D 2014-03-24T18:08:15.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a088183774c4efae4105076355ac4010c62390a8 +F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6c59d23316c83b318b1a94d9b28a5d321737fa5 -R ab9aa4f9f7d5ea8c370ac77061403538 -U dan -Z 631d6a70f4aeb5d6a455a81761c8aefd +P 5d506743f541b022cde04a9606baa4680cdfd70b +R edfc832bbac864fb9d0e2cb38ae7f098 +U drh +Z a04be0daeb9646c039a07b2f019c65a9 diff --git a/manifest.uuid b/manifest.uuid index 36b51c0a51..1c7384bd0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file +4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 641a54faa2..40fd59daab 100644 --- a/src/select.c +++ b/src/select.c @@ -463,19 +463,28 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ - int nData /* Number of elements in the data array */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + 1 + nData; /* Fields in sorter record */ - int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regBase; /* Regs for sorter record */ int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+1 ); + regBase = regData - nExpr - 1; + }else{ + regBase = sqlite3GetTempRange(pParse, nBase); + } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -515,7 +524,9 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nBase); + if( nPrefixReg==0 ){ + sqlite3ReleaseTempRange(pParse, regBase, nBase); + } } if( pSelect->iLimit ){ int addr1, addr2; @@ -631,6 +642,7 @@ static void selectInnerLoop( int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); assert( pEList!=0 ); @@ -646,6 +658,10 @@ static void selectInnerLoop( nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr + 1; + pParse->nMem += nPrefixReg; + } pDest->iSdst = pParse->nMem+1; pParse->nMem += nResultCol; }else if( pDest->iSdst+nResultCol > pParse->nMem ){ @@ -762,10 +778,10 @@ static void selectInnerLoop( case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -780,7 +796,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1, 1); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -788,7 +804,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -806,7 +822,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -832,7 +848,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -846,7 +862,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ From dd23c6bfb42cc87dc5919657a972d84afff2162a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 20:19:07 +0000 Subject: [PATCH 12/99] Omit the sequence value from sorter records used by GROUP BY queries that cannot use an index. FossilOrigin-Name: 3f90abddc31ac20739778c235a834c33f7057997 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 9 ++++----- src/vdbesort.c | 1 + 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a1b753f05e..63d2130c2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. -D 2014-03-24T18:08:15.960 +C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. +D 2014-03-24T20:19:07.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e +F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 +F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5d506743f541b022cde04a9606baa4680cdfd70b -R edfc832bbac864fb9d0e2cb38ae7f098 -U drh -Z a04be0daeb9646c039a07b2f019c65a9 +P 4f472accf072d9cb64f209923924b26f21b13d27 +R e1309a603202b4d83715c783fe89f3da +U dan +Z 8e3a9a2ddbbe8575a70cbc6b0dba434a diff --git a/manifest.uuid b/manifest.uuid index 1c7384bd0a..0ff8edcdd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file +3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 40fd59daab..7e0085dd36 100644 --- a/src/select.c +++ b/src/select.c @@ -4910,7 +4910,7 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); @@ -5002,8 +5002,8 @@ int sqlite3Select( groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -5013,8 +5013,7 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index d1b726b727..d55156a2c1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -480,6 +480,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + pSorter->pUnpacked->nField = pCsr->pKeyInfo->nField; if( !sqlite3TempInMemory(db) ){ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); From 3c863634ef70229b61573d4e98165ac564e44658 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 14:12:16 +0000 Subject: [PATCH 13/99] Enable four sorting threads by default in the command-line shell. FossilOrigin-Name: 1cab83577c814feb35b4fb91af0d52a9751d99bc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c | 1 - src/threads.c | 4 ++-- src/vdbesort.c | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7ccdd5b9a5..d29d622a2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sfixes\sand\senhancements\sfrom\strunk. -D 2014-03-25T13:17:41.050 +C Enable\sfour\ssorting\sthreads\sby\sdefault\sin\sthe\scommand-line\sshell. +D 2014-03-25T14:12:16.989 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 269c3e31a450fce642a10569221a49180348c88e -F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf +F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c cde9d885fd562b5427f89a42a8829085f88b17df +F src/threads.c b96d62f88c06d4fa980a4a92685d1b130c4c84d3 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0fbaf5b3ec3e779d81c4db4eb2f0ae5f44fbb02c +F src/vdbesort.c c3e427de848b78e9e9feaa25f68fb64686bab6cd F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 e6798871ce94961135762669af418cd78540c121 -R 5887d7b59f581a1221f99e7d95aac2b8 +P b415dfb6cb0df0c69992ca2bb700c15664f158e6 +R a4b6dbfcec045a7fea4740daec225662 U drh -Z 5e51466328d6bc63d3b26ee7aec599ce +Z a12cd4438cfae19c4441f02ed27b5761 diff --git a/manifest.uuid b/manifest.uuid index 628a8f1b85..e0e5369892 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b415dfb6cb0df0c69992ca2bb700c15664f158e6 \ No newline at end of file +1cab83577c814feb35b4fb91af0d52a9751d99bc \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1313112709..c9abd9ec7d 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3532,7 +3532,6 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } /* diff --git a/src/threads.c b/src/threads.c index 7cc964250d..cb148b6c6f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -28,7 +28,7 @@ #include "sqliteInt.h" /********************************* Unix Pthreads ****************************/ -#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include @@ -85,7 +85,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /********************************* Win32 Threads ****************************/ -#if SQLITE_OS_WIN && !SQLITE_OS_WINRT +#if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include diff --git a/src/vdbesort.c b/src/vdbesort.c index 3c8e91d987..a3f7aba10e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -31,7 +31,7 @@ typedef struct FileWriter FileWriter; ** operations to be single-threaded. */ #ifndef SQLITE_MAX_SORTER_THREAD -# define SQLITE_MAX_SORTER_THREAD 1 +# define SQLITE_MAX_SORTER_THREAD 4 #endif /* From 78d5843245b2ef9c1fb92213d0c0e4bb49e6c70d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 15:04:07 +0000 Subject: [PATCH 14/99] Remove the sequence values from sorter records used by ORDER BY as well. FossilOrigin-Name: c3ae3697832a00d4d5758988a8766bdbb691e6b8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 40 +++++++++++++++++++++++++++------------- src/vdbe.c | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 63d2130c2b..2690ecef90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. -D 2014-03-24T20:19:07.793 +C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. +D 2014-03-25T15:04:07.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 +F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d +F src/vdbe.c 42177064bd02fc55984aabc3a241368fda01e8b4 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4f472accf072d9cb64f209923924b26f21b13d27 -R e1309a603202b4d83715c783fe89f3da +P 3f90abddc31ac20739778c235a834c33f7057997 +R 743d88a284437e5e0babc8e33ec37e61 U dan -Z 8e3a9a2ddbbe8575a70cbc6b0dba434a +Z 5024d9c780855d9abccbd4c6c23f3054 diff --git a/manifest.uuid b/manifest.uuid index 0ff8edcdd3..b187241ae4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file +c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7e0085dd36..7480eeb94b 100644 --- a/src/select.c +++ b/src/select.c @@ -467,24 +467,29 @@ static void pushOntoSorter( int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ - int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ - int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ - int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ - int op; /* Opcode to add sorter record to sorter */ + int regRecord = sqlite3GetTempReg(pParse); /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + assert( bSeq==0 || bSeq==1 ); if( nPrefixReg ){ - assert( nPrefixReg==nExpr+1 ); - regBase = regData - nExpr - 1; + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; }else{ regBase = sqlite3GetTempRange(pParse, nBase); } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); - sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - if( nPrefixReg==0 ){ - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+bSeq, nData); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -496,8 +501,13 @@ static void pushOntoSorter( regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; - nKey = nExpr - pSort->nOBSat + 1; - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; @@ -659,7 +669,8 @@ static void selectInnerLoop( if( pDest->iSdst==0 ){ if( pSort ){ - nPrefixReg = pSort->pOrderBy->nExpr + 1; + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; pParse->nMem += nPrefixReg; } pDest->iSdst = pParse->nMem+1; @@ -1152,6 +1163,7 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; + int bSeq; /* True if sorter record includes seq. no. */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; #endif @@ -1185,14 +1197,16 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; + bSeq = 0; }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; p5 = 0; + bSeq = 1; } for(i=0; ip1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC->pSorter ); + if( (pC->seqCount++)==0 ){ + pc = pOp->p2 - 1; + } + break; +} + /* Opcode: OpenPseudo P1 P2 P3 * * ** Synopsis: P3 columns in r[P2] ** From ab1dcc1a4be2d1636bb43685525ab60c2e2e6ee8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 17:07:48 +0000 Subject: [PATCH 15/99] Fix a problem in the code added by [707ea170b3] causing vdbesort.c to sort unstably. FossilOrigin-Name: d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 20 ++++++++++++++++---- test/sort.test | 23 +++++++++++++++++++++++ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2690ecef90..eb635d79a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. -D 2014-03-25T15:04:07.777 +C Fix\sa\sproblem\sin\sthe\scode\sadded\sby\s[707ea170b3]\scausing\svdbesort.c\sto\ssort\sunstably. +D 2014-03-25T17:07:48.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf +F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -816,7 +816,7 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 +F test/sort.test cb76a6e9db897b6871ef4dbc206ebc6dbc033bf4 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3f90abddc31ac20739778c235a834c33f7057997 -R 743d88a284437e5e0babc8e33ec37e61 +P c3ae3697832a00d4d5758988a8766bdbb691e6b8 +R eb749164f2115a0b3ef6dbb32fd74b6a U dan -Z 5024d9c780855d9abccbd4c6c23f3054 +Z ecf1eb93a5e3757595a49ddb23f9c671 diff --git a/manifest.uuid b/manifest.uuid index b187241ae4..bb3b1adce9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file +d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d55156a2c1..7b43554282 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -562,6 +562,9 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. +** +** In cases where key values are equal, keys from list p1 are considered +** to be smaller than list p2. */ static void vdbeSorterMerge( const VdbeCursor *pCsr, /* For pKeyInfo */ @@ -597,6 +600,11 @@ static void vdbeSorterMerge( ** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK ** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error ** occurs. +** +** The sort is required to be stable - if two elements compare as equal +** then the one added to the sorter first is considered the smaller. +** Currently, the list is sorted from newest to oldest - pSorter->pRecord +** points to the most recently added sort key. */ static int vdbeSorterSort(const VdbeCursor *pCsr){ int i; @@ -837,7 +845,7 @@ static int vdbeSorterInitMerge( int i; /* Used to iterator through aIter[] */ i64 nByte = 0; /* Total bytes in all opened PMAs */ - /* Initialize the iterators. */ + /* Initialize the iterators. Iterator 0 contains the oldest data. */ for(i=0; iaIter[i]; rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); @@ -1009,8 +1017,13 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() ** was actually called above, then pSorter->pUnpacked now contains ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. */ - if( iRes<=0 ){ + ** vdbeSorterCompare() from decoding pIter2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aIter[] array + ** is sorted from oldest to newest, so pIter1 contains older values + ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pSorter->aIter); pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; pKey2 = pIter2->aKey; @@ -1019,7 +1032,6 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; } - } *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); } diff --git a/test/sort.test b/test/sort.test index 08d496b259..b543ffad24 100644 --- a/test/sort.test +++ b/test/sort.test @@ -464,4 +464,27 @@ do_test sort-12.1 { } } {1 2 xxx 1 3 yyy 1 1 zzz} + +#------------------------------------------------------------------------- +# Check that the sorter in vdbesort.c sorts in a stable fashion. +# +do_execsql_test sort-13.0 { + CREATE TABLE t10(a, b); +} +do_test sort-13.1 { + db transaction { + for {set i 0} {$i < 100000} {incr i} { + execsql { INSERT INTO t10 VALUES( $i/10, $i%10 ) } + } + } +} {} +do_execsql_test sort-13.2 { + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +do_execsql_test sort-13.3 { + PRAGMA cache_size = 5; + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] + + finish_test From face0872125725b1d104b5a5d8e0f04079c9e7a6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 17:23:41 +0000 Subject: [PATCH 16/99] Use xFetch() to access temporary files in vdbesort.c. Use a single large allocation instead of many small allocations when accumulating records in vdbesort.c. This is an interim commit - it allocates a buffer the size of the page-cache every time data is sorted. FossilOrigin-Name: f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 --- manifest | 14 ++-- manifest.uuid | 2 +- src/vdbesort.c | 204 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 50b1f52a8e..dc6ca295c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sand\sthe\sfix\sfor\sthe\scrash\son\sa\scorrupt\ndatabase. -D 2014-03-27T00:09:00.185 +C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. +D 2014-03-27T17:23:41.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc +F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e005f2d6dd9faf38cc8fdb9428b5aa6192a6adae f585f5d7a0f9bf8c590388654a3638231eba8892 -R 9012cf758152dd3775ad665fff626f11 -U drh -Z ffbdc2f043e4deddac6c4b509aff14ac +P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc +R 8f417577dbf4ab5b5ce5a802872c70f0 +U dan +Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 diff --git a/manifest.uuid b/manifest.uuid index f8b9980e3d..853b60d3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b35346c32dba14963c85ec178f2b46aa2bbf6dc \ No newline at end of file +f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 7b43554282..2a15168fd2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,6 +105,8 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ + u8* aMemory; /* Block to allocate records from */ + int iMemory; /* Offset of free space in aMemory */ }; /* @@ -121,6 +123,7 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ u8 *aBuffer; /* Current read buffer */ int nBuffer; /* Size of read buffer in bytes */ + u8 *aMap; /* Pointer to mapping of pFile */ }; /* @@ -163,6 +166,7 @@ struct SorterRecord { static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ sqlite3DbFree(db, pIter->aAlloc); sqlite3DbFree(db, pIter->aBuffer); + if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); memset(pIter, 0, sizeof(VdbeSorterIter)); } @@ -183,6 +187,13 @@ static int vdbeSorterIterRead( ){ int iBuf; /* Offset within buffer to read from */ int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + assert( p->aBuffer ); /* If there is no more data to be read from the buffer, read the next @@ -264,18 +275,22 @@ static int vdbeSorterIterRead( static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ int iBuf; - iBuf = p->iReadOff % p->nBuffer; - if( iBuf && (p->nBuffer-iBuf)>=9 ){ - p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); }else{ - u8 aVarint[16], *a; - int i = 0, rc; - do{ - rc = vdbeSorterIterRead(db, p, 1, &a); - if( rc ) return rc; - aVarint[(i++)&0xf] = a[0]; - }while( (a[0]&0x80)!=0 ); - sqlite3GetVarint(aVarint, pnOut); + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbeSorterIterRead(db, p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } } return SQLITE_OK; @@ -323,6 +338,7 @@ static int vdbeSorterIterInit( ){ int rc = SQLITE_OK; int nBuf; + void *pMap; nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); @@ -333,33 +349,41 @@ static int vdbeSorterIterInit( pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; + /* See if this PMA can be read using xFetch. */ + rc = sqlite3OsFetch(pIter->pFile, 0, pSorter->iWriteOff, &pMap); + if( rc!=SQLITE_OK ) return rc; + if( pMap ){ + pIter->aMap = (u8*)pMap; }else{ - int iBuf; + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; + }else{ + int iBuf; + + iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pSorter->iWriteOff ){ + nRead = (int)(pSorter->iWriteOff - iStart); + } + rc = sqlite3OsRead( + pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } - rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); } + } - if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; - *pnByte += nByte; - } + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + pIter->iEof = pSorter->iWriteOff; + rc = vdbeSorterIterVarint(db, pIter, &nByte); + pIter->iEof = pIter->iReadOff + nByte; + *pnByte += nByte; } if( rc==SQLITE_OK ){ @@ -488,6 +512,10 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + + pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); + assert( pSorter->iMemory==0 ); + if( !pSorter->aMemory ) return SQLITE_NOMEM; } return SQLITE_OK; @@ -521,7 +549,9 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ sqlite3OsCloseFree(pSorter->pTemp1); pSorter->pTemp1 = 0; } - vdbeSorterRecordFree(db, pSorter->pRecord); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pSorter->pRecord); + } pSorter->pRecord = 0; pSorter->iWriteOff = 0; pSorter->iReadOff = 0; @@ -529,6 +559,7 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ pSorter->nTree = 0; pSorter->nPMA = 0; pSorter->aTree = 0; + pSorter->iMemory = 0; } @@ -540,6 +571,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3DbFree(db, pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -551,12 +583,17 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ - int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + int rc; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + } + return rc; } /* @@ -725,6 +762,29 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ fileWriterWrite(p, aByte, nByte); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ + int rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } + return rc; +} +#else +# define vdbeSorterExtendFile(x,y) SQLITE_OK +#endif + /* ** Write the current contents of the in-memory linked-list to a PMA. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. @@ -760,6 +820,13 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ assert( pSorter->nPMA==0 ); } + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + ); + } + if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; @@ -771,12 +838,14 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } + if( pSorter->aMemory ) pSorter->pRecord = 0; + assert( pSorter->pRecord==0 || rc!=SQLITE_OK ); return rc; } @@ -789,23 +858,37 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; + SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( pSorter->aMemory ){ + int nReq = sizeof(SorterRecord) + pVal->n; + if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ + pNew = &sRecord; + pNew->pVal = pVal->z; + }else{ + pNew = &pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + } }else{ - pNew->pVal = (void *)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } } + if( pNew!=&sRecord ){ + pNew->pVal = (void*)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + } + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + /* See if the contents of the sorter should now be written out. They ** are written out when either of the following are true: ** @@ -815,18 +898,22 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( - (pSorter->nInMemory>pSorter->mxPmaSize) - || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - )){ + if( pSorter->mxPmaSize>0 ){ + if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + (pSorter->nInMemory > pSorter->mxPmaSize) + || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ))){ #ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; #endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } } return rc; @@ -934,6 +1021,9 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( pTemp2==0 ){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + } } if( rc==SQLITE_OK ){ @@ -1039,7 +1129,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; pFree->pNext = 0; - vdbeSorterRecordFree(db, pFree); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pFree); + } *pbEof = !pSorter->pRecord; rc = SQLITE_OK; } From 6971952c650cdf1d4e76e63fdb89165bac5c1af0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 19:25:02 +0000 Subject: [PATCH 17/99] Instead of allocating a single large buffer at the beginning of each sort operation, start with a small buffer and extend it using realloc() as required. FossilOrigin-Name: 81987c8ceb64f051528a6ca42673821d9ab7c0ff --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 191 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 134 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index dc6ca295c7..986addf1fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. -D 2014-03-27T17:23:41.403 +C Instead\sof\sallocating\sa\ssingle\slarge\sbuffer\sat\sthe\sbeginning\sof\seach\ssort\soperation,\sstart\swith\sa\ssmall\sbuffer\sand\sextend\sit\susing\srealloc()\sas\srequired. +D 2014-03-27T19:25:02.222 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf +F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc -R 8f417577dbf4ab5b5ce5a802872c70f0 +P f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 +R 5fefa7763a5f0d580046ba5219de21a5 U dan -Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 +Z 8f7301353335dd4530b24cac1ad9c37a diff --git a/manifest.uuid b/manifest.uuid index 853b60d3d3..cf2496b0b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file +81987c8ceb64f051528a6ca42673821d9ab7c0ff \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 2a15168fd2..f229b1f18b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -107,6 +107,7 @@ struct VdbeSorter { UnpackedRecord *pUnpacked; /* Used to unpack keys */ u8* aMemory; /* Block to allocate records from */ int iMemory; /* Offset of free space in aMemory */ + int nMemory; /* Current size of allocation at aMemory */ }; /* @@ -144,15 +145,37 @@ struct FileWriter { /* ** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord using the -** SorterRecord.pNext pointer. +** together into a linked list headed at VdbeSorter.pRecord. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. */ struct SorterRecord { - void *pVal; int nVal; - SorterRecord *pNext; + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; }; +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -513,9 +536,15 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ if( mxCachemxPmaSize = mxCache * pgsz; - pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); - assert( pSorter->iMemory==0 ); - if( !pSorter->aMemory ) return SQLITE_NOMEM; + /* If the application is using memsys3 or memsys5, use a separate + ** allocation for each sort-key in memory. Otherwise, use a single big + ** allocation at pSorter->aMemory for all sort-keys. */ + if( sqlite3GlobalConfig.pHeap==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->aMemory = (u8*)sqlite3Malloc(pSorter->nMemory); + if( !pSorter->aMemory ) return SQLITE_NOMEM; + } } return SQLITE_OK; @@ -528,7 +557,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *p; SorterRecord *pNext; for(p=pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; sqlite3DbFree(db, p); } } @@ -611,22 +640,22 @@ static void vdbeSorterMerge( ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? p2->pVal : 0; + void *pVal2 = p2 ? SRVAL(p2) : 0; while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pCsr, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); if( res<=0 ){ *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; + pp = &p1->u.pNext; + p1 = p1->u.pNext; pVal2 = 0; }else{ *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; + pp = &p2->u.pNext; + p2 = p2->u.pNext; if( p2==0 ) break; - pVal2 = p2->pVal; + pVal2 = SRVAL(p2); } } *pp = p1 ? p1 : p2; @@ -656,8 +685,18 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = pSorter->pRecord; while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; + SorterRecord *pNext; + if( pSorter->aMemory ){ + assert( p->u.iNextnMemory ); + if( (u8*)p==pSorter->aMemory ){ + pNext = 0; + }else{ + pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + p->u.pNext = 0; for(i=0; aSlot[i]; i++){ vdbeSorterMerge(pCsr, p, aSlot[i], &p); aSlot[i] = 0; @@ -835,9 +874,9 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pSorter->nPMA++; fileWriterWriteVarint(&writer, pSorter->nInMemory); for(p=pSorter->pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, p->pVal, p->nVal); + fileWriterWrite(&writer, SRVAL(p), p->nVal); if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; @@ -858,39 +897,24 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + assert( pSorter ); - pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - if( pSorter->aMemory ){ - int nReq = sizeof(SorterRecord) + pVal->n; - if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ - pNew = &sRecord; - pNew->pVal = pVal->z; - }else{ - pNew = &pSorter->aMemory[pSorter->iMemory]; - pSorter->iMemory += ROUND8(nReq); - } - }else{ - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); - if( pNew==0 ){ - return SQLITE_NOMEM; - } - } - - if( pNew!=&sRecord ){ - pNew->pVal = (void*)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - } - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - - /* See if the contents of the sorter should now be written out. They - ** are written out when either of the following are true: + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: ** ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or @@ -898,23 +922,62 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( pSorter->mxPmaSize>0 ){ - if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( (pSorter->nInMemory > pSorter->mxPmaSize) || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - ))){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; -#endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); - } + ); } + if( bFlush ){ +#ifdef SQLITE_DEBUG + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; +#endif + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } + + pSorter->nInMemory += nPMA; + + if( pSorter->aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + int nNew = pSorter->nMemory * 2; + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + + aNew = sqlite3Realloc(pSorter->aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM; + pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; + }else{ + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pNew->u.pNext = pSorter->pRecord; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->pRecord = pNew; return rc; } @@ -1127,8 +1190,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ } }else{ SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->pNext; - pFree->pNext = 0; + pSorter->pRecord = pFree->u.pNext; + pFree->u.pNext = 0; if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(db, pFree); } @@ -1154,7 +1217,7 @@ static void *vdbeSorterRowkey( pKey = pIter->aKey; }else{ *pnKey = pSorter->pRecord->nVal; - pKey = pSorter->pRecord->pVal; + pKey = SRVAL(pSorter->pRecord); } return pKey; } From 73e7d8b2bbccfc462e57973939c2acd887e1067b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 19:47:19 +0000 Subject: [PATCH 18/99] Fix a compiler warning and an after-OOM memory leak. FossilOrigin-Name: 58f7ca29303c280229f86d01f418e1de5f5aebba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a3dc17de19..e6d729925d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\schanges\sfrom\strunk. -D 2014-03-28T18:35:39.779 +C Fix\sa\scompiler\swarning\sand\san\safter-OOM\smemory\sleak. +D 2014-03-28T19:47:19.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b +F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 81987c8ceb64f051528a6ca42673821d9ab7c0ff 27deb6e49bcc76714dbdc61b34748603155ac770 -R 79fcad4572db3d584834fdf0de4170a6 +P 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e +R 911f6d7d99d9350a186d106d9b5cf527 U drh -Z 871d90dec169e4d494b7ff32475fff97 +Z 55f6b91a865b632e99503eb67df5d777 diff --git a/manifest.uuid b/manifest.uuid index 518ac58c2a..cf96f01707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e \ No newline at end of file +58f7ca29303c280229f86d01f418e1de5f5aebba \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index f229b1f18b..791a2465ca 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -959,7 +959,8 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*) + (aNew + ((u8*)pSorter->pRecord - pSorter->aMemory)); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1052,7 +1053,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree = (int *)&pSorter->aIter[N]; pSorter->nTree = N; - do { + while(1){ int iNew; /* Index of new, merged, PMA */ for(iNew=0; @@ -1105,6 +1106,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( rc==SQLITE_OK ) rc = rc2; } } + if( rc ) break; if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ break; @@ -1117,7 +1119,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->iReadOff = 0; iWrite2 = 0; } - }while( rc==SQLITE_OK ); + } if( pTemp2 ){ sqlite3OsCloseFree(pTemp2); From ff9fce4d607a92715636898af6bb96ea53a3f84d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 06:27:35 +0000 Subject: [PATCH 19/99] Add the optimization to avoid some unnecessary calls to sqlite3VdbeRecordUnpack() added to the trunk by [707ea170b3]. FossilOrigin-Name: fc4d04e6b039ea5aeb47739e38c5926e63a4b01b --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 52c9a03b24..a8a84cabb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\sorderby-planning\sbranch. -D 2014-03-28T19:18:16.969 +C Add\sthe\soptimization\sto\savoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sadded\sto\sthe\strunk\sby\s[707ea170b3]. +D 2014-03-29T06:27:35.162 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 01068b89364fa2bffeba9b929367ed04661e97f7 +F src/vdbesort.c d7ef3c431d7996907815a13579952506bf2bdf8b F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8cb2b02baa7ef9aa96319e977f0315328f944237 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e -R 132477cf9a18d083088d044c20454723 +P 4c7fb5423430f3b936befaa7c309f8e1968ee7d8 +R c555f8c440b397229a1b1d80867cea18 U dan -Z b189ca5a1e8d41a36f80415af9011f68 +Z 5b7820939497d4f0728d4d29380b4189 diff --git a/manifest.uuid b/manifest.uuid index a600160612..d7c0207d3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c7fb5423430f3b936befaa7c309f8e1968ee7d8 \ No newline at end of file +fc4d04e6b039ea5aeb47739e38c5926e63a4b01b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e01790d600..e6fbbdd2df 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1052,11 +1052,59 @@ static int vdbeSorterNext( rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); /* Update contents of aTree[] */ - for(i=(pMerger->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pThread, pMerger, i); + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pMerger->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pMerger->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pThread, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pThread->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pThread->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aIter[] array + ** is sorted from oldest to newest, so pIter1 contains older values + ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pMerger->aIter); + pIter2 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); + pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + } + } + *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } - *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); return rc; } From e7c84cc7e30a90fdefc8508b5ef6fc2bed48e115 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 09:34:45 +0000 Subject: [PATCH 20/99] Fix a problem in vdbesort.c causing spurious SQLITE_NOMEM errors when using memsys3 or memsys5. FossilOrigin-Name: a683c05f6250389e84b980b16559e162ba1a27c2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 44 ++++++++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index a8a84cabb7..5859cf8e2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\soptimization\sto\savoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sadded\sto\sthe\strunk\sby\s[707ea170b3]. -D 2014-03-29T06:27:35.162 +C Fix\sa\sproblem\sin\svdbesort.c\scausing\sspurious\sSQLITE_NOMEM\serrors\swhen\susing\smemsys3\sor\smemsys5. +D 2014-03-29T09:34:45.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d7ef3c431d7996907815a13579952506bf2bdf8b +F src/vdbesort.c 80812ceb596febe778d506587d24bef6fabc00d4 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4c7fb5423430f3b936befaa7c309f8e1968ee7d8 -R c555f8c440b397229a1b1d80867cea18 +P fc4d04e6b039ea5aeb47739e38c5926e63a4b01b +R 58d9f269ef794b7845b0943176283ea0 U dan -Z 5b7820939497d4f0728d4d29380b4189 +Z 849635fb9198ef253e10fe8fcdddee0e diff --git a/manifest.uuid b/manifest.uuid index d7c0207d3a..0e4e18ad35 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc4d04e6b039ea5aeb47739e38c5926e63a4b01b \ No newline at end of file +a683c05f6250389e84b980b16559e162ba1a27c2 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e6fbbdd2df..d9c26f209e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1046,7 +1046,6 @@ static int vdbeSorterNext( ){ int rc; int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ /* Advance the current iterator */ rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); @@ -1281,11 +1280,14 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; - if( pSorter->aMemory==0 ){ - pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); - if( pSorter->aMemory==0 ) return SQLITE_NOMEM; - }else{ - pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); + assert( pSorter->aMemory==0 || pThread->aListMemory==0 ); + if( pThread->aListMemory ){ + if( pSorter->aMemory==0 ){ + pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); + if( pSorter->aMemory==0 ) return SQLITE_NOMEM; + }else{ + pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); + } } rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); }else{ @@ -1337,19 +1339,21 @@ int sqlite3VdbeSorterWrite( */ nReq = pVal->n + sizeof(SorterRecord); nPMA = pVal->n + sqlite3VarintLen(pVal->n); - if( pSorter->aMemory ){ - bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; - }else{ - bFlush = ( - (pSorter->nInMemory > pSorter->mxPmaSize) - || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - ); - } - if( bFlush ){ - rc = vdbeSorterFlushPMA(db, pCsr, 0); - pSorter->nInMemory = 0; - pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + if( pSorter->mxPmaSize ){ + if( pSorter->aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( + (pSorter->nInMemory > pSorter->mxPmaSize) + || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ); + } + if( bFlush ){ + rc = vdbeSorterFlushPMA(db, pCsr, 0); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } } pSorter->nInMemory += nPMA; @@ -1375,7 +1379,7 @@ int sqlite3VdbeSorterWrite( pSorter->iMemory += ROUND8(nReq); pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; }else{ - pNew = (SorterRecord *)sqlite3Malloc(pVal->n+sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ return SQLITE_NOMEM; } From 853c4a7621b37ecb3c4cab8ca1fb7fc06fb1a533 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 10:01:58 +0000 Subject: [PATCH 21/99] Fix a broken assert() in vdbesort.c. FossilOrigin-Name: 18d1b402f2dbe78f1a1113bb356b710e348365ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5859cf8e2e..0b2bd63f50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\svdbesort.c\scausing\sspurious\sSQLITE_NOMEM\serrors\swhen\susing\smemsys3\sor\smemsys5. -D 2014-03-29T09:34:45.457 +C Fix\sa\sbroken\sassert()\sin\svdbesort.c. +D 2014-03-29T10:01:58.802 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 80812ceb596febe778d506587d24bef6fabc00d4 +F src/vdbesort.c 2881297f4acdba5908078c5d7f00635288a1ca08 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fc4d04e6b039ea5aeb47739e38c5926e63a4b01b -R 58d9f269ef794b7845b0943176283ea0 +P a683c05f6250389e84b980b16559e162ba1a27c2 +R d8e63408790a442d33eec2a57272c54d U dan -Z 849635fb9198ef253e10fe8fcdddee0e +Z 6c5e71d99f167a1ded5f09353b3e0513 diff --git a/manifest.uuid b/manifest.uuid index 0e4e18ad35..1bf7c269ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a683c05f6250389e84b980b16559e162ba1a27c2 \ No newline at end of file +18d1b402f2dbe78f1a1113bb356b710e348365ef \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d9c26f209e..c6927f87a7 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1280,7 +1280,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; - assert( pSorter->aMemory==0 || pThread->aListMemory==0 ); + assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); if( pThread->aListMemory ){ if( pSorter->aMemory==0 ){ pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); From b3f56fdb69201329c96ef9bb904a5df46fc62c0d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 31 Mar 2014 19:57:34 +0000 Subject: [PATCH 22/99] Add the SQLITE_MAX_WORKER_THREADS compile time option. And the SQLITE_CONFIG_WORKER_THREADS sqlite3_config() switch. FossilOrigin-Name: 2774710df8cd2bfaca49888c69f1b01c0ddadf9a --- manifest | 22 +++++++-------- manifest.uuid | 2 +- src/global.c | 1 + src/main.c | 7 +++++ src/sqlite.h.in | 11 ++++++++ src/sqliteInt.h | 14 ++++++++++ src/threads.c | 3 +++ src/vdbesort.c | 72 ++++++++++++++++++++++++++++--------------------- 8 files changed, 90 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 0b2bd63f50..4a4f37cc87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\svdbesort.c. -D 2014-03-29T10:01:58.802 +C Add\sthe\sSQLITE_MAX_WORKER_THREADS\scompile\stime\soption.\sAnd\sthe\sSQLITE_CONFIG_WORKER_THREADS\ssqlite3_config()\sswitch. +D 2014-03-31T19:57:34.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 +F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db +F src/main.c d3655832585baef4c2356529a5c6ca5ca3bd7c1f F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -219,10 +219,10 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df -F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 +F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3f5190a4e07ca227035334da8d66ebe227071528 +F src/sqliteInt.h 7f42c2792b951db22fa189bbed828a5e3b38789c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b96d62f88c06d4fa980a4a92685d1b130c4c84d3 +F src/threads.c 6992f70cab8d5d8451a6b5641a9256d1749af87b F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2881297f4acdba5908078c5d7f00635288a1ca08 +F src/vdbesort.c b4d6133bada297e118492420346f83cd76c6da31 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a683c05f6250389e84b980b16559e162ba1a27c2 -R d8e63408790a442d33eec2a57272c54d +P 18d1b402f2dbe78f1a1113bb356b710e348365ef +R 8b3347b8372cc17a3226330524ab6da6 U dan -Z 6c5e71d99f167a1ded5f09353b3e0513 +Z 11cb5db8cbfdddf6047ddaf9c26850df diff --git a/manifest.uuid b/manifest.uuid index 1bf7c269ad..096bf0eab2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d1b402f2dbe78f1a1113bb356b710e348365ef \ No newline at end of file +2774710df8cd2bfaca49888c69f1b01c0ddadf9a \ No newline at end of file diff --git a/src/global.c b/src/global.c index 1ee3f6436f..4ed597d292 100644 --- a/src/global.c +++ b/src/global.c @@ -167,6 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ + SQLITE_MAX_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/main.c b/src/main.c index 9e83d4963a..c1eaa6849a 100644 --- a/src/main.c +++ b/src/main.c @@ -515,6 +515,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_WORKER_THREADS: { + int n = va_arg(ap, int); + if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; + if( n>=0 ) sqlite3GlobalConfig.nWorker = n; + break; + } + default: { rc = SQLITE_ERROR; break; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5d2c87552c..78aa9c36e1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1715,6 +1715,16 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_WORKER_THREADS]] +**
SQLITE_CONFIG_WORKER_THREADS +**
^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int. +** It is used to set the number of background worker threads that may be +** launched when sorting large amounts of data. A value of 0 means launch +** no background threads at all. The maximum number of background threads +** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS +** pre-processor option. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1739,6 +1749,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_WORKER_THREADS 24 /* int nWorker */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 41219c710a..5f85e80d82 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -422,6 +422,19 @@ # define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 +# undef SQLITE_MAX_WORKER_THREADS +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif + + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -2685,6 +2698,7 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ + int nWorker; /* Number of worker threads to use */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ diff --git a/src/threads.c b/src/threads.c index cb148b6c6f..96cb17cef7 100644 --- a/src/threads.c +++ b/src/threads.c @@ -27,6 +27,8 @@ */ #include "sqliteInt.h" +#if SQLITE_MAX_WORKER_THREADS>0 + /********************************* Unix Pthreads ****************************/ #if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 @@ -215,3 +217,4 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ #endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ /****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c6927f87a7..84c6fadc94 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -26,14 +26,6 @@ typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; -/* -** Maximum number of threads to use. Setting this value to 1 forces all -** operations to be single-threaded. -*/ -#ifndef SQLITE_MAX_SORTER_THREAD -# define SQLITE_MAX_SORTER_THREAD 4 -#endif - /* ** Candidate values for SorterThread.eWork */ @@ -48,9 +40,10 @@ typedef struct FileWriter FileWriter; ** is configured and passed to vdbeSorterThreadMain() - either directly by ** the main thread or via a background thread. ** -** Exactly SQLITE_MAX_SORTER_THREAD instances of this structure are allocated +** Exactly SorterThread.nThread instances of this structure are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. +** way. SorterThread.nThread is set to the number of worker threads allowed +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SorterThread.bDone ** is set to 0 and the SorterThread.pThread variable set to point to the @@ -59,7 +52,7 @@ typedef struct FileWriter FileWriter; ** exits. SorterThread.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SQLITE_MAX_SORTER_THREAD-1]) +** One object (specifically, VdbeSorter.aThread[SorterThread.nThread-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SorterThread.eWork, @@ -187,7 +180,8 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ - SorterThread aThread[SQLITE_MAX_SORTER_THREAD]; + int nThread; /* Size of aThread[] array */ + SorterThread aThread[1]; }; /* @@ -572,7 +566,7 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ vdbeSorterCompare( pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); @@ -597,21 +591,26 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; + int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); - pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SorterThread); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->pSorter = pSorter; if( pSorter==0 ){ rc = SQLITE_NOMEM; }else{ - pKeyInfo = (KeyInfo*)&pSorter[1]; + pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - for(i=0; inThread = nWorker + 1; + for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; pThread->pKeyInfo = pKeyInfo; pThread->pVfs = db->pVfs; @@ -674,10 +673,11 @@ static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){ /* ** Join all threads. */ +#if SQLITE_MAX_WORKER_THREADS>0 static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pThread ){ void *pRet; @@ -690,6 +690,9 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ } return rc; } +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +#endif /* ** Allocate a new SorterMerger object with space for nIter iterators. @@ -739,7 +742,7 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; vdbeSorterThreadCleanup(db, pThread); } @@ -1246,8 +1249,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ SorterThread *pThread; /* Thread context used to create new PMA */ pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ pThread = &pSorter->aThread[i]; +#if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; assert( pThread->pThread ); @@ -1258,11 +1262,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ rc = SQLITE_PTR_TO_INT(pRet); } } +#endif if( pThread->pThread==0 ) break; } if( rc==SQLITE_OK ){ - int bUseFg = (bFg || i==(SQLITE_MAX_SORTER_THREAD-1)); + int bUseFg = (bFg || i==(pSorter->nThread-1)); assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; @@ -1277,6 +1282,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->aMemory = aMem; } +#if SQLITE_MAX_WORKER_THREADS>0 if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; @@ -1290,7 +1296,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } } rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); - }else{ + }else +#endif + { /* Use the foreground thread for this operation */ u8 *aMem; rc = vdbeSorterRunThread(pThread); @@ -1370,7 +1378,9 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*)( + aNew + ((u8*)pSorter->pRecord - pSorter->aMemory) + ); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1399,7 +1409,7 @@ int sqlite3VdbeSorterWrite( static int vdbeSorterCountPMA(VdbeSorter *pSorter){ int nPMA = 0; int i; - for(i=0; inThread; i++){ nPMA += pSorter->aThread[i].nPMA; } return nPMA; @@ -1446,19 +1456,21 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge ** some of them together so that this is no longer the case. */ - assert( SORTER_MAX_MERGE_COUNT>=SQLITE_MAX_SORTER_THREAD ); if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; - for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/SQLITE_MAX_SORTER_THREAD; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; - if( i<(SQLITE_MAX_SORTER_THREAD-1) ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( i<(pSorter->nThread-1) ){ void *pCtx = (void*)pThread; rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); - }else{ + }else +#endif + { rc = vdbeSorterRunThread(pThread); } } @@ -1475,7 +1487,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ int nIter = 0; /* Number of iterators used */ int i; SorterMerger *pMerger; - for(i=0; inThread; i++){ nIter += pSorter->aThread[i].nPMA; } @@ -1485,7 +1497,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else{ int iIter = 0; int iThread = 0; - for(iThread=0; iThreadnThread; iThread++){ int iPMA; i64 iReadOff = 0; SorterThread *pThread = &pSorter->aThread[iThread]; From 38fdead890bc6006ee4d550b9694aa495ddc9798 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 10:19:02 +0000 Subject: [PATCH 23/99] Fix a problem with OOM handling in the sorter code. FossilOrigin-Name: 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/btree.c | 9 ++++++--- src/sqliteInt.h | 2 +- src/test1.c | 41 +++++++++++++++++++++++++++++++++++++++++ src/vdbeaux.c | 26 ++++++++++++++++---------- src/vdbesort.c | 9 ++++++--- test/malloc.test | 22 ++++++++++++++++++++++ 8 files changed, 104 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 4a4f37cc87..b819b3428f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_MAX_WORKER_THREADS\scompile\stime\soption.\sAnd\sthe\sSQLITE_CONFIG_WORKER_THREADS\ssqlite3_config()\sswitch. -D 2014-03-31T19:57:34.075 +C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode. +D 2014-04-01T10:19:02.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578 +F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -222,12 +222,12 @@ F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 7f42c2792b951db22fa189bbed828a5e3b38789c +F src/sqliteInt.h 3ed0fedb5b64ece395a2114b7c73417678f3e420 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 2401eee14a4309a7cfe2aeb2f30ad517a1d9c299 +F src/test1.c 31596bf8a9c0629f88e514a4ec864847c8946c4e F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -283,10 +283,10 @@ F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d +F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b4d6133bada297e118492420346f83cd76c6da31 +F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -663,7 +663,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test fd368e31fe98d4779ed80442f311ed9f03bcd1f7 +F test/malloc.test 26ae08a09cc15a98d147ee63925e3a66048e71c9 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 18d1b402f2dbe78f1a1113bb356b710e348365ef -R 8b3347b8372cc17a3226330524ab6da6 +P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a +R a6fe6d7d652069432584e1a691602cb5 U dan -Z 11cb5db8cbfdddf6047ddaf9c26850df +Z ac080311c2ad2c3d7bf733448ffc0f03 diff --git a/manifest.uuid b/manifest.uuid index 096bf0eab2..7a6826cad1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2774710df8cd2bfaca49888c69f1b01c0ddadf9a \ No newline at end of file +59cd5229e2b5be5272cf57c7e7d09e97d16a5425 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c3055836c9..fb49746567 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4588,7 +4588,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - pIdxKey->isCorrupt = 0; + pIdxKey->errCode = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4712,7 +4712,10 @@ int sqlite3BtreeMovetoUnpacked( c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } - assert( pIdxKey->isCorrupt==0 || c==0 ); + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || !pCur->pBtree->db->mallocFailed) + ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4722,7 +4725,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5f85e80d82..0763f085ab 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1641,7 +1641,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ - u8 isCorrupt; /* Corruption detected by xRecordCompare() */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ diff --git a/src/test1.c b/src/test1.c index 3000288c7d..4b485ce741 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2703,6 +2703,46 @@ bad_args: return TCL_ERROR; } +/* +** Usage: add_test_utf16bin_collate +** +** Add a utf-16 collation sequence named "utf16bin" to the database +** handle. This collation sequence compares arguments in the same way as the +** built-in collation "binary". +*/ +static int test_utf16bin_collate_func( + void *pCtx, + int nA, const void *zA, + int nB, const void *zB +){ + int nCmp = (nA>nB ? nB : nA); + int res = memcmp(zA, zB, nCmp); + if( res==0 ) res = nA - nB; + return res; +} +static int test_utf16bin_collate( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + + if( objc!=2 ) goto bad_args; + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + + rc = sqlite3_create_collation(db, "utf16bin", SQLITE_UTF16, 0, + test_utf16bin_collate_func + ); + if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; + return TCL_OK; + +bad_args: + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; +} + /* ** When the collation needed callback is invoked, record the name of ** the requested collating function here. The recorded name is linked @@ -6481,6 +6521,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "add_test_collate", test_collate, 0 }, { "add_test_collate_needed", test_collate_needed, 0 }, { "add_test_function", test_function, 0 }, + { "add_test_utf16bin_collate", test_utf16bin_collate, 0 }, #endif { "sqlite3_test_errstr", test_errstr, 0 }, { "tcl_variable_type", tcl_variable_type, 0 }, diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0ce21378d5..7194dea36a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3229,7 +3229,8 @@ static int vdbeRecordCompareDebug( static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, - const CollSeq *pColl + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ ){ if( pMem1->enc==pColl->enc ){ /* The strings are already in the correct encoding. Call the @@ -3252,6 +3253,7 @@ static int vdbeCompareMemString( rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } @@ -3334,7 +3336,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ assert( !pColl || pColl->xCmp ); if( pColl ){ - return vdbeCompareMemString(pMem1, pMem2, pColl); + return vdbeCompareMemString(pMem1, pMem2, pColl, 0); } /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ @@ -3406,8 +3408,10 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** -** If database corruption is discovered, set pPKey2->isCorrupt to non-zero -** and return 0. +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the +** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ @@ -3438,7 +3442,7 @@ int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } i = 0; @@ -3517,14 +3521,16 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; mem1.z = (char*)&aKey1[d1]; - rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + rc = vdbeCompareMemString( + &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode + ); }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3544,7 +3550,7 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); @@ -3564,7 +3570,7 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB + assert( CORRUPT_DB || pKeyInfo->db==0 || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || pKeyInfo->db->mallocFailed @@ -3724,7 +3730,7 @@ static int vdbeRecordCompareString( nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } nCmp = MIN( pPKey2->aMem[0].n, nStr ); diff --git a/src/vdbesort.c b/src/vdbesort.c index 84c6fadc94..60a023c8f5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1134,6 +1134,7 @@ static void *vdbeSorterThreadMain(void *pCtx){ goto thread_out; } pThread->pUnpacked->nField = pThread->pKeyInfo->nField; + pThread->pUnpacked->errCode = 0; } if( pThread->eWork==SORTER_THREAD_CONS ){ @@ -1222,6 +1223,10 @@ static void *vdbeSorterThreadMain(void *pCtx){ thread_out: pThread->bDone = 1; + if( rc==SQLITE_OK && pThread->pUnpacked->errCode ){ + assert( pThread->pUnpacked->errCode==SQLITE_NOMEM ); + rc = SQLITE_NOMEM; + } return SQLITE_INT_TO_PTR(rc); } @@ -1267,8 +1272,6 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } if( rc==SQLITE_OK ){ - int bUseFg = (bFg || i==(pSorter->nThread-1)); - assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; pThread->pList = pSorter->pRecord; @@ -1283,7 +1286,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #if SQLITE_MAX_WORKER_THREADS>0 - if( bUseFg==0 ){ + if( bFg || i==(pSorter->nThread-1) ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); diff --git a/test/malloc.test b/test/malloc.test index 5d03aa8fe8..10d2a18c96 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -880,6 +880,28 @@ do_malloc_test 39 -tclprep { db close } +reset_db +add_test_utf16bin_collate db +do_execsql_test 40.1 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES('fghij'); + INSERT INTO t1 VALUES('pqrst'); + INSERT INTO t1 VALUES('abcde'); + INSERT INTO t1 VALUES('uvwxy'); + INSERT INTO t1 VALUES('klmno'); +} +do_execsql_test 40.2 { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; +} {abcde fghij klmno pqrst uvwxy} +do_faultsim_test 40.3 -faults oom-trans* -body { + execsql { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; + } +} -test { + faultsim_test_result {0 {abcde fghij klmno pqrst uvwxy}} + faultsim_integrity_check +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 578e1ca8d7d00805fb4410210ba317d2b8f9e846 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 15:38:44 +0000 Subject: [PATCH 24/99] Even if compile time option SQLITE_MAX_WORKER_THREADS is set to one or greater, set the default number of worker threads to zero. Distribute data more evenly between threads in sqlite3VdbeSorterWrite() to improve performance when sorting large amounts of data. Add new test file sort2.test. FossilOrigin-Name: 643c86a056168e39fcb7f39b8a72731f1eb246db --- manifest | 21 +++++++++++---------- manifest.uuid | 2 +- src/global.c | 2 +- src/shell.c | 2 ++ src/test_config.c | 6 ++++++ src/test_malloc.c | 27 +++++++++++++++++++++++++++ src/vdbesort.c | 18 +++++++++++++----- test/sort2.test | 41 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 test/sort2.test diff --git a/manifest b/manifest index b819b3428f..73c52a7c23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode. -D 2014-04-01T10:19:02.635 +C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. +D 2014-04-01T15:38:44.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 +F src/global.c deadd872189b92aca4ee2566332a86315839f811 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa -F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df +F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -240,7 +240,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 +F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -251,7 +251,7 @@ F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 +F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307 F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e +F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,6 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a +F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1160,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a -R a6fe6d7d652069432584e1a691602cb5 +P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 +R 574aa31e9a2482d16cbf0b6e42831c35 U dan -Z ac080311c2ad2c3d7bf733448ffc0f03 +Z 53e6ec40073ce4b7a6ae1cb9efd4c01b diff --git a/manifest.uuid b/manifest.uuid index 7a6826cad1..957ecbe9a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59cd5229e2b5be5272cf57c7e7d09e97d16a5425 \ No newline at end of file +643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file diff --git a/src/global.c b/src/global.c index 4ed597d292..091b750803 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - SQLITE_MAX_WORKER_THREADS, /* nWorker */ + 0, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/shell.c b/src/shell.c index c9abd9ec7d..6fc85ae394 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3532,6 +3532,8 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3); } /* diff --git a/src/test_config.c b/src/test_config.c index 1db8198641..2201fba007 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,6 +99,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif +#if SQLITE_MAX_WORKER_THREADS>0 + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/test_malloc.c b/src/test_malloc.c index e3cfcaa9f0..6ac030f23c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1253,6 +1253,32 @@ static int test_config_cis( return TCL_OK; } +/* +** Usage: sqlite3_config_worker_threads N +*/ +static int test_config_worker_threads( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + int nThread; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "N"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){ + return TCL_ERROR; + } + + rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + + return TCL_OK; +} + /* ** Usage: sqlite3_dump_memsys3 FILENAME ** sqlite3_dump_memsys5 FILENAME @@ -1506,6 +1532,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_config_cis", test_config_cis ,0 }, + { "sqlite3_config_worker_threads", test_config_worker_threads ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/vdbesort.c b/src/vdbesort.c index 60a023c8f5..746618cc25 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -180,6 +180,7 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ + int iPrev; /* Previous thread used to flush PMA */ int nThread; /* Size of aThread[] array */ SorterThread aThread[1]; }; @@ -1251,11 +1252,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SorterThread *pThread; /* Thread context used to create new PMA */ + SorterThread *pThread = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nThread-1); pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ - pThread = &pSorter->aThread[i]; + for(i=0; iiPrev + i + 1) % nWorker; + pThread = &pSorter->aThread[iTest]; #if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; @@ -1269,7 +1272,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #endif if( pThread->pThread==0 ) break; + pThread = 0; } + if( pThread==0 ){ + pThread = &pSorter->aThread[nWorker]; + } + pSorter->iPrev = (pThread - pSorter->aThread); if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); @@ -1286,7 +1294,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #if SQLITE_MAX_WORKER_THREADS>0 - if( bFg || i==(pSorter->nThread-1) ){ + if( !bFg && pThread!=&pSorter->aThread[nWorker] ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); @@ -1464,7 +1472,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; #if SQLITE_MAX_WORKER_THREADS>0 diff --git a/test/sort2.test b/test/sort2.test new file mode 100644 index 0000000000..2a77b61ed1 --- /dev/null +++ b/test/sort2.test @@ -0,0 +1,41 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort2 + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 7 +reset_db + +do_execsql_test 1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) +} { + 20000 100 20000 100 20000 100 20000 100 20000 100 +} + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 0 +sqlite3_initialize +finish_test + From fad9f9a8a6920c795223d84cc3923920fe2269a9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 18:41:51 +0000 Subject: [PATCH 25/99] When sorting data for a CREATE INDEX statement in single-threaded mode, assume that keys are delivered to the sorter in primary key order. Also fix various comments that had fallen out of date. FossilOrigin-Name: 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 --- manifest | 20 ++++++------- manifest.uuid | 2 +- src/build.c | 2 +- src/vdbe.c | 8 +++-- src/vdbeInt.h | 2 +- src/vdbesort.c | 77 ++++++++++++++++++++++++------------------------- test/sort2.test | 14 +++++++++ 7 files changed, 70 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index 73c52a7c23..d0ef35d91d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. -D 2014-04-01T15:38:44.990 +C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate. +D 2014-04-01T18:41:51.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 +F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b +F src/vdbe.c eed2230017ca8ac5ca8fc221b38be036bba48499 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 +F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 +F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 +F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 -R 574aa31e9a2482d16cbf0b6e42831c35 +P 643c86a056168e39fcb7f39b8a72731f1eb246db +R c2ac1c0b48f44ef7c812322485a83f86 U dan -Z 53e6ec40073ce4b7a6ae1cb9efd4c01b +Z 23ba0058a55a91ed976e2b1f4e943278 diff --git a/manifest.uuid b/manifest.uuid index 957ecbe9a9..823d08f629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file +821d1ac4504243fa13b9e3c0d56361ad9fb80d78 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 10077e5018..26c2922ebd 100644 --- a/src/build.c +++ b/src/build.c @@ -2669,7 +2669,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) sqlite3KeyInfoRef(pKey), P4_KEYINFO); /* Open the table. Loop through all rows of the table, inserting index diff --git a/src/vdbe.c b/src/vdbe.c index de02d1f638..21da5007ab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3390,11 +3390,15 @@ case OP_OpenEphemeral: { break; } -/* Opcode: SorterOpen P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 P3 P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. */ case OP_SorterOpen: { VdbeCursor *pCx; @@ -3406,7 +3410,7 @@ case OP_SorterOpen: { pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); - rc = sqlite3VdbeSorterInit(db, pCx); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 043b56da59..03a303ca1b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -437,7 +437,7 @@ void sqlite3VdbeFrameDelete(VdbeFrame*); int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); -int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); diff --git a/src/vdbesort.c b/src/vdbesort.c index 746618cc25..81b2ff5c2b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -489,47 +489,27 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Argument pKeyInfo supplies the collation functions -** used by the comparison. If an error occurs, return an SQLite error code. -** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive -** value, depending on whether key1 is smaller, equal to or larger than key2. +** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. ** -** If the bOmitRowid argument is non-zero, assume both keys end in a rowid -** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid -** is true and key1 contains even a single NULL value, it is considered to -** be less than key2. Even if key2 also contains NULL values. +** Before returning, object (pThread->pUnpacked) is populated with the +** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it +** is assumed that the (pThread->pUnpacked) structure already contains the +** unpacked key to use as key2. ** -** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace -** has been allocated and contains an unpacked record that is used as key2. +** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. */ -static void vdbeSorterCompare( +static int vdbeSorterCompare( SorterThread *pThread, /* Thread context (for pKeyInfo) */ - int nIgnore, /* Ignore the last nIgnore fields */ const void *pKey1, int nKey1, /* Left side of comparison */ - const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ ){ - KeyInfo *pKeyInfo = pThread->pKeyInfo; UnpackedRecord *r2 = pThread->pUnpacked; - int i; - if( pKey2 ){ - sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2); } - - if( nIgnore ){ - r2->nField = pKeyInfo->nField - nIgnore; - assert( r2->nField>0 ); - for(i=0; inField; i++){ - if( r2->aMem[i].flags & MEM_Null ){ - *pRes = -1; - return; - } - } - assert( r2->default_rc==0 ); - } - - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } /* @@ -568,8 +548,8 @@ static int vdbeSorterDoCompare( }else{ int res; assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ - vdbeSorterCompare( - pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + res = vdbeSorterCompare( + pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -585,7 +565,7 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. */ -int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ int i; /* Used to iterate through aThread[] */ int mxCache; /* Cache size */ @@ -608,6 +588,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; + if( nField && nWorker==0 ) pKeyInfo->nField = nField; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nThread = nWorker + 1; @@ -806,7 +787,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pThread, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); + res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; @@ -1075,8 +1056,8 @@ static int vdbeSorterNext( }else if( pIter2->pFile==0 ){ iRes = -1; }else{ - vdbeSorterCompare(pThread, 0, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + iRes = vdbeSorterCompare(pThread, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey ); } @@ -1596,6 +1577,9 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** passed as the first argument currently points to. For the purposes of ** the comparison, ignore the rowid field at the end of each record. ** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Evn if pVal also contains NULL values. +** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter @@ -1608,10 +1592,23 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterThread *pMain = &pSorter->aThread[0]; + UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + assert( r2->nField>=pKeyInfo->nField-nIgnore ); + r2->nField = pKeyInfo->nField-nIgnore; + pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; inField; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2, 0); return SQLITE_OK; } diff --git a/test/sort2.test b/test/sort2.test index 2a77b61ed1..f8bfb0fe51 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -33,6 +33,20 @@ do_execsql_test 1 { 20000 100 20000 100 20000 100 20000 100 20000 100 } +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; +} + +do_execsql_test 2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); +} + db close sqlite3_shutdown sqlite3_config_worker_threads 0 From 8d8f56296b1101784d2223f18808d50a245c8fec Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 2 Apr 2014 14:38:14 +0000 Subject: [PATCH 26/99] Change the name of the SorterThread object to "SortSubtask" to avoid confusion with the SQLiteThread object. FossilOrigin-Name: 4ee2d910fbbed8d4def15e4e99ee225839f3a739 --- manifest | 14 ++++---- manifest.uuid | 2 +- src/vdbesort.c | 92 +++++++++++++++++++++++++------------------------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index d0ef35d91d..8bc41fb375 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate. -D 2014-04-01T18:41:51.759 +C Change\sthe\sname\sof\sthe\sSorterThread\sobject\sto\s"SortSubtask"\sto\savoid\sconfusion\nwith\sthe\sSQLiteThread\sobject. +D 2014-04-02T14:38:14.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c +F src/vdbesort.c 0cb83fc36f8a56cc6e4ecdefc3c7d4155169cef2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 643c86a056168e39fcb7f39b8a72731f1eb246db -R c2ac1c0b48f44ef7c812322485a83f86 -U dan -Z 23ba0058a55a91ed976e2b1f4e943278 +P 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 +R 2283517ddbfd0881db16b2410c51a15c +U drh +Z e6ea0be46d70fe73f5ea59fa8b577076 diff --git a/manifest.uuid b/manifest.uuid index 823d08f629..0fe5750cf3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -821d1ac4504243fa13b9e3c0d56361ad9fb80d78 \ No newline at end of file +4ee2d910fbbed8d4def15e4e99ee225839f3a739 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 81b2ff5c2b..ff315494e6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -20,57 +20,57 @@ typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SorterThread SorterThread; +typedef struct SortSubtask SortSubtask; typedef struct SorterRecord SorterRecord; typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; /* -** Candidate values for SorterThread.eWork +** Candidate values for SortSubtask.eWork */ -#define SORTER_THREAD_SORT 1 -#define SORTER_THREAD_TO_PMA 2 -#define SORTER_THREAD_CONS 3 +#define SORTER_THREAD_SORT 1 /* Sort records on pList */ +#define SORTER_THREAD_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pFile */ +#define SORTER_THREAD_CONS 3 /* Consolidate multiple PMAs */ /* ** Much of the work performed in this module to sort the list of records is ** broken down into smaller units that may be peformed in parallel. In order ** to perform such a unit of work, an instance of the following structure -** is configured and passed to vdbeSorterThreadMain() - either directly by +** is configured and passed to vdbeSortSubtaskMain() - either directly by ** the main thread or via a background thread. ** -** Exactly SorterThread.nThread instances of this structure are allocated +** Exactly SortSubtask.nThread instances of this structure are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. SorterThread.nThread is set to the number of worker threads allowed +** way. SortSubtask.nThread is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** -** When a background thread is launched to perform work, SorterThread.bDone -** is set to 0 and the SorterThread.pThread variable set to point to the -** thread handle. SorterThread.bDone is set to 1 (to indicate to the main -** thread that joining SorterThread.pThread will not block) before the thread -** exits. SorterThread.pThread and bDone are always cleared after the +** When a background thread is launched to perform work, SortSubtask.bDone +** is set to 0 and the SortSubtask.pThread variable set to point to the +** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main +** thread that joining SortSubtask.pThread will not block) before the thread +** exits. SortSubtask.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SorterThread.nThread-1]) +** One object (specifically, VdbeSorter.aThread[SortSubtask.nThread-1]) ** is reserved for the foreground thread. ** -** The nature of the work performed is determined by SorterThread.eWork, +** The nature of the work performed is determined by SortSubtask.eWork, ** as follows: ** ** SORTER_THREAD_SORT: -** Sort the linked list of records at SorterThread.pList. +** Sort the linked list of records at SortSubtask.pList. ** ** SORTER_THREAD_TO_PMA: -** Sort the linked list of records at SorterThread.pList, and write -** the results to a new PMA in temp file SorterThread.pTemp1. Open +** Sort the linked list of records at SortSubtask.pList, and write +** the results to a new PMA in temp file SortSubtask.pTemp1. Open ** the temp file if it is not already open. ** ** SORTER_THREAD_CONS: -** Merge existing PMAs until SorterThread.nConsolidate or fewer -** remain in temp file SorterThread.pTemp1. +** Merge existing PMAs until SortSubtask.nConsolidate or fewer +** remain in temp file SortSubtask.pTemp1. */ -struct SorterThread { +struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ int bDone; /* Set to true by pThread when finished */ @@ -182,7 +182,7 @@ struct VdbeSorter { int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ int nThread; /* Size of aThread[] array */ - SorterThread aThread[1]; + SortSubtask aThread[1]; }; /* @@ -427,7 +427,7 @@ static int vdbeSorterIterNext(VdbeSorterIter *pIter){ ** PMA is empty). */ static int vdbeSorterIterInit( - SorterThread *pThread, /* Thread context */ + SortSubtask *pThread, /* Thread context */ i64 iStart, /* Start offset in pThread->pTemp1 */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ @@ -501,7 +501,7 @@ static int vdbeSorterIterInit( ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( - SorterThread *pThread, /* Thread context (for pKeyInfo) */ + SortSubtask *pThread, /* Thread context (for pKeyInfo) */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ @@ -518,7 +518,7 @@ static int vdbeSorterCompare( ** value to recalculate. */ static int vdbeSorterDoCompare( - SorterThread *pThread, + SortSubtask *pThread, SorterMerger *pMerger, int iOut ){ @@ -547,7 +547,7 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ res = vdbeSorterCompare( pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); @@ -578,7 +578,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); - sz = sizeof(VdbeSorter) + nWorker * sizeof(SorterThread); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->pSorter = pSorter; @@ -593,7 +593,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ pSorter->nThread = nWorker + 1; for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; pThread->pKeyInfo = pKeyInfo; pThread->pVfs = db->pVfs; pThread->pgsz = pgsz; @@ -636,7 +636,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ ** Free all resources owned by the object indicated by argument pThread. All ** fields of *pThread are zeroed before returning. */ -static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){ +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ sqlite3DbFree(db, pThread->pUnpacked); pThread->pUnpacked = 0; if( pThread->aListMemory==0 ){ @@ -660,7 +660,7 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pThread ){ void *pRet; int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); @@ -725,8 +725,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; - vdbeSorterThreadCleanup(db, pThread); + SortSubtask *pThread = &pSorter->aThread[i]; + vdbeSortSubtaskCleanup(db, pThread); } if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); @@ -776,7 +776,7 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - SorterThread *pThread, /* Calling thread context */ + SortSubtask *pThread, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -810,7 +810,7 @@ static void vdbeSorterMerge( ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SorterThread *pThread){ +static int vdbeSorterSort(SortSubtask *pThread){ int i; SorterRecord **aSlot; SorterRecord *p; @@ -974,7 +974,7 @@ static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SorterThread *pThread){ +static int vdbeSorterListToPMA(SortSubtask *pThread){ int rc = SQLITE_OK; /* Return code */ FileWriter writer; /* Object used to write to the file */ @@ -1025,7 +1025,7 @@ static int vdbeSorterListToPMA(SorterThread *pThread){ ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeSorterNext( - SorterThread *pThread, + SortSubtask *pThread, SorterMerger *pMerger, int *pbEof ){ @@ -1095,9 +1095,9 @@ static int vdbeSorterNext( /* ** The main routine for sorter-thread operations. */ -static void *vdbeSorterThreadMain(void *pCtx){ +static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; - SorterThread *pThread = (SorterThread*)pCtx; + SortSubtask *pThread = (SortSubtask*)pCtx; assert( pThread->eWork==SORTER_THREAD_SORT || pThread->eWork==SORTER_THREAD_TO_PMA @@ -1216,8 +1216,8 @@ static void *vdbeSorterThreadMain(void *pCtx){ ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ -static int vdbeSorterRunThread(SorterThread *pThread){ - int rc = SQLITE_PTR_TO_INT( vdbeSorterThreadMain((void*)pThread) ); +static int vdbeSorterRunThread(SortSubtask *pThread){ + int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pThread) ); assert( pThread->bDone ); pThread->bDone = 0; return rc; @@ -1233,7 +1233,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SorterThread *pThread = 0; /* Thread context used to create new PMA */ + SortSubtask *pThread = 0; /* Thread context used to create new PMA */ int nWorker = (pSorter->nThread-1); pSorter->bUsePMA = 1; @@ -1287,7 +1287,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); + rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { @@ -1422,7 +1422,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ if( pSorter->pRecord ){ - SorterThread *pThread = &pSorter->aThread[0]; + SortSubtask *pThread = &pSorter->aThread[0]; *pbEof = 0; pThread->pList = pSorter->pRecord; pThread->eWork = SORTER_THREAD_SORT; @@ -1451,7 +1451,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; for(i=0; rc==SQLITE_OK && inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; @@ -1459,7 +1459,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nThread-1) ){ void *pCtx = (void*)pThread; - rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); + rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSortSubtaskMain,pCtx); }else #endif { @@ -1492,7 +1492,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ for(iThread=0; iThreadnThread; iThread++){ int iPMA; i64 iReadOff = 0; - SorterThread *pThread = &pSorter->aThread[iThread]; + SortSubtask *pThread = &pSorter->aThread[iThread]; for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ i64 nDummy = 0; VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; From dd95d30f82ff926c7b79e0410788575c0e66cdfb Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Apr 2014 15:15:25 +0000 Subject: [PATCH 27/99] Fix some problems with OOM handling in vdbesort.c. FossilOrigin-Name: 47e702bd8392bc50c4edaf6a2c8c499af87b520e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 36 +++++++++++++++++------------------- test/mallocA.test | 23 ++++++++++++++++++++++- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 8bc41fb375..2cae57fb97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sSorterThread\sobject\sto\s"SortSubtask"\sto\savoid\sconfusion\nwith\sthe\sSQLiteThread\sobject. -D 2014-04-02T14:38:14.734 +C Fix\ssome\sproblems\swith\sOOM\shandling\sin\svdbesort.c. +D 2014-04-02T15:15:25.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0cb83fc36f8a56cc6e4ecdefc3c7d4155169cef2 +F src/vdbesort.c e830ea4a7333ff07177fc367918ede2b33fcfe10 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -671,7 +671,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e -F test/mallocA.test 1ba0367fb5434e7bc2fa4afcb30b14174d91b160 +F test/mallocA.test c049224adeb0244b8f6eb770c1fa6ac40f9b3518 F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 -R 2283517ddbfd0881db16b2410c51a15c -U drh -Z e6ea0be46d70fe73f5ea59fa8b577076 +P 4ee2d910fbbed8d4def15e4e99ee225839f3a739 +R 18420994a2ccade11be299f196096342 +U dan +Z 3f3c411cd945b7957dc8b5eeb10e94fa diff --git a/manifest.uuid b/manifest.uuid index 0fe5750cf3..4eba9e128c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ee2d910fbbed8d4def15e4e99ee225839f3a739 \ No newline at end of file +47e702bd8392bc50c4edaf6a2c8c499af87b520e \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ff315494e6..50680007b6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -443,10 +443,13 @@ static int vdbeSorterIterInit( pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); - - /* Try to xFetch() a mapping of the entire temp file. If this is possible, - ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + if( pIter->aAlloc ){ + /* Try to xFetch() a mapping of the entire temp file. If this is possible, + ** the PMA will be read via the mapping. Otherwise, use xRead(). */ + rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + }else{ + rc = SQLITE_NOMEM; + } if( rc==SQLITE_OK ){ if( pMap ){ @@ -698,23 +701,15 @@ static SorterMerger *vdbeSorterMergerNew(int nIter){ } /* -** Reset a merger +** Free the SorterMerger object passed as the only argument. */ -static void vdbeSorterMergerReset(SorterMerger *pMerger){ +static void vdbeSorterMergerFree(SorterMerger *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ vdbeSorterIterZero(&pMerger->aIter[i]); } } -} - - -/* -** Free the SorterMerger object passed as the only argument. -*/ -static void vdbeSorterMergerFree(SorterMerger *pMerger){ - vdbeSorterMergerReset(pMerger); sqlite3_free(pMerger); } @@ -724,6 +719,8 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); + vdbeSorterMergerFree(pSorter->pMerger); + pSorter->pMerger = 0; for(i=0; inThread; i++){ SortSubtask *pThread = &pSorter->aThread[i]; vdbeSortSubtaskCleanup(db, pThread); @@ -731,7 +728,6 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); } - vdbeSorterMergerReset(pSorter->pMerger); pSorter->pRecord = 0; pSorter->nInMemory = 0; pSorter->bUsePMA = 0; @@ -1292,11 +1288,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ #endif { /* Use the foreground thread for this operation */ - u8 *aMem; rc = vdbeSorterRunThread(pThread); - aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; + if( rc==SQLITE_OK ){ + u8 *aMem = pThread->aListMemory; + pThread->aListMemory = pSorter->aMemory; + pSorter->aMemory = aMem; + assert( pThread->pList==0 ); + } } } diff --git a/test/mallocA.test b/test/mallocA.test index 61e88a61e7..d6d6de8222 100644 --- a/test/mallocA.test +++ b/test/mallocA.test @@ -25,7 +25,6 @@ if {!$MEMDEBUG} { return } - # Construct a test database # forcedelete test.db.bu @@ -116,6 +115,28 @@ ifcapable stat3 { } } +do_execsql_test 7.0 { + PRAGMA cache_size = 5; +} +do_faultsim_test 7 -faults oom-trans* -prep { + if {$iFail < 500} { set iFail 2000 } + if {$iFail > 1215} { set iFail 2000 } +} -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } +} -test { + set res [list 200 100 200 100 200 100 200 100 200 100] + faultsim_test_result [list 0 $res] +} + + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From ac4f0039c0aeabb76f63085697371d9673cd7a70 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 2 Apr 2014 18:58:49 +0000 Subject: [PATCH 28/99] Add a big introductory comment to vdbesort.c explaining its operation at a high level. Also adjust some symbolic names and fix other comment issues in that file. FossilOrigin-Name: eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b --- manifest | 14 ++--- manifest.uuid | 2 +- src/vdbesort.c | 137 +++++++++++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 2cae57fb97..e1d3814d6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\sOOM\shandling\sin\svdbesort.c. -D 2014-04-02T15:15:25.762 +C Add\sa\sbig\sintroductory\scomment\sto\svdbesort.c\sexplaining\sits\soperation\sat\sa\nhigh\slevel.\s\sAlso\sadjust\ssome\ssymbolic\snames\sand\sfix\sother\scomment\sissues\sin\nthat\sfile. +D 2014-04-02T18:58:49.259 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c e830ea4a7333ff07177fc367918ede2b33fcfe10 +F src/vdbesort.c 523283d7c3f499444df97d700503f3c9ddd746b7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4ee2d910fbbed8d4def15e4e99ee225839f3a739 -R 18420994a2ccade11be299f196096342 -U dan -Z 3f3c411cd945b7957dc8b5eeb10e94fa +P 47e702bd8392bc50c4edaf6a2c8c499af87b520e +R a364509f4308441b3af07ec467078f09 +U drh +Z a6f0e24eb45caefc4a14c210e684429c diff --git a/manifest.uuid b/manifest.uuid index 4eba9e128c..30ed24d4f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47e702bd8392bc50c4edaf6a2c8c499af87b520e \ No newline at end of file +eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 50680007b6..c5d13c685c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,15 +10,87 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a multi-threaded external merge sort +** algorithm that is efficient even if the number of element being sorted +** exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). +** +** Algorithm: +** +** Records to be sorted are initially held in memory, in the order in +** which they arrive from Write(). When the amount of memory needed exceeds +** a threshold, all in-memory records are sorted and then appended to +** a temporary file as a "Packed-Memory-Array" or "PMA" and the memory is +** reset. There is a single temporary file used for all PMAs. The PMAs +** are packed one after another in the file. The VdbeSorter object keeps +** track of the number of PMAs written. +** +** When the Rewind() is seen, any records still held in memory are sorted. +** If no PMAs have been written (if all records are still held in memory) +** then subsequent Rowkey(), Next(), and Compare() operations work directly +** from memory. But if PMAs have been written things get a little more +** complicated. +** +** When Rewind() is seen after PMAs have been written, any records still +** in memory are sorted and written as a final PMA. Then all the PMAs +** are merged together into a single massive PMA that Next(), Rowkey(), +** and Compare() walk to extract the records in sorted order. +** +** If SQLITE_MAX_WORKER_THREADS is non-zero, various steps of the above +** algorithm might be performed in parallel by separate threads. Threads +** are only used when one or more PMA spill to disk. If the sort is small +** enough to fit entirely in memory, everything happens on the main thread. */ - #include "sqliteInt.h" #include "vdbeInt.h" - +/* +** Private objects used by the sorter +*/ typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SortSubtask SortSubtask; typedef struct SorterRecord SorterRecord; @@ -29,20 +101,18 @@ typedef struct FileWriter FileWriter; /* ** Candidate values for SortSubtask.eWork */ -#define SORTER_THREAD_SORT 1 /* Sort records on pList */ -#define SORTER_THREAD_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pFile */ -#define SORTER_THREAD_CONS 3 /* Consolidate multiple PMAs */ +#define SORT_SUBTASK_SORT 1 /* Sort records on pList */ +#define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ +#define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ /* -** Much of the work performed in this module to sort the list of records is -** broken down into smaller units that may be peformed in parallel. In order -** to perform such a unit of work, an instance of the following structure -** is configured and passed to vdbeSortSubtaskMain() - either directly by -** the main thread or via a background thread. +** Sorting is divided up into smaller subtasks. Each subtask is controlled +** by an instance of this object. Subtask might run in either the main thread +** or in a background thread. ** -** Exactly SortSubtask.nThread instances of this structure are allocated +** Exactly VdbeSorter.nThread instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. SortSubtask.nThread is set to the number of worker threads allowed +** way. VdbeSorter.nThread is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SortSubtask.bDone @@ -52,21 +122,21 @@ typedef struct FileWriter FileWriter; ** exits. SortSubtask.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SortSubtask.nThread-1]) +** One object (specifically, VdbeSorter.aThread[VdbeSorter.nThread-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SortSubtask.eWork, ** as follows: ** -** SORTER_THREAD_SORT: +** SORT_SUBTASK_SORT: ** Sort the linked list of records at SortSubtask.pList. ** -** SORTER_THREAD_TO_PMA: +** SORT_SUBTASK_TO_PMA: ** Sort the linked list of records at SortSubtask.pList, and write ** the results to a new PMA in temp file SortSubtask.pTemp1. Open ** the temp file if it is not already open. ** -** SORTER_THREAD_CONS: +** SORT_SUBTASK_CONS: ** Merge existing PMAs until SortSubtask.nConsolidate or fewer ** remain in temp file SortSubtask.pTemp1. */ @@ -79,8 +149,8 @@ struct SortSubtask { UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ - u8 eWork; /* One of the SORTER_THREAD_* constants */ - int nConsolidate; /* For THREAD_CONS, max final PMAs */ + u8 eWork; /* One of the SORT_SUBTASK_* constants */ + int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ SorterRecord *pList; /* List of records for pThread to sort */ int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ @@ -1095,9 +1165,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; SortSubtask *pThread = (SortSubtask*)pCtx; - assert( pThread->eWork==SORTER_THREAD_SORT - || pThread->eWork==SORTER_THREAD_TO_PMA - || pThread->eWork==SORTER_THREAD_CONS + assert( pThread->eWork==SORT_SUBTASK_SORT + || pThread->eWork==SORT_SUBTASK_TO_PMA + || pThread->eWork==SORT_SUBTASK_CONS ); assert( pThread->bDone==0 ); @@ -1115,7 +1185,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ pThread->pUnpacked->errCode = 0; } - if( pThread->eWork==SORTER_THREAD_CONS ){ + if( pThread->eWork==SORT_SUBTASK_CONS ){ assert( pThread->pList==0 ); while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); @@ -1188,7 +1258,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ rc = vdbeSorterSort(pThread); /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pThread->eWork==SORTER_THREAD_TO_PMA ){ + if( rc==SQLITE_OK && pThread->eWork==SORT_SUBTASK_TO_PMA ){ #ifdef SQLITE_DEBUG i64 nExpect = pThread->nInMemory + sqlite3VarintLen(pThread->nInMemory) @@ -1258,7 +1328,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); - pThread->eWork = SORTER_THREAD_TO_PMA; + pThread->eWork = SORT_SUBTASK_TO_PMA; pThread->pList = pSorter->pRecord; pThread->nInMemory = pSorter->nInMemory; pSorter->nInMemory = 0; @@ -1306,7 +1376,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; @@ -1423,7 +1493,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SortSubtask *pThread = &pSorter->aThread[0]; *pbEof = 0; pThread->pList = pSorter->pRecord; - pThread->eWork = SORTER_THREAD_SORT; + pThread->eWork = SORT_SUBTASK_SORT; assert( pThread->aListMemory==0 ); pThread->aListMemory = pSorter->aMemory; rc = vdbeSorterRunThread(pThread); @@ -1452,7 +1522,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; - pThread->eWork = SORTER_THREAD_CONS; + pThread->eWork = SORT_SUBTASK_CONS; #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nThread-1) ){ @@ -1576,12 +1646,15 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** the comparison, ignore the rowid field at the end of each record. ** ** If the sorter cursor key contains any NULL values, consider it to be -** less than pVal. Evn if pVal also contains NULL values. +** less than pVal. Even if pVal also contains NULL values. ** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter ** key. +** +** This routine forms the core of the OP_SorterCompare opcode, which in +** turn is used to verify uniqueness when constructing a UNIQUE INDEX. */ int sqlite3VdbeSorterCompare( const VdbeCursor *pCsr, /* Sorter cursor */ From a634fb15778e176b4c63ca5d50fac06fe6e16947 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Apr 2014 02:54:27 +0000 Subject: [PATCH 29/99] Refactor local object and method names in vdbesort.c so that their names more closely reflect their actual use. FossilOrigin-Name: d284e30eb1db144965fa85566e4234e30464350b --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 610 +++++++++++++++++++++++++------------------------ 3 files changed, 314 insertions(+), 310 deletions(-) diff --git a/manifest b/manifest index e1d3814d6f..a1d8fa4b08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sbig\sintroductory\scomment\sto\svdbesort.c\sexplaining\sits\soperation\sat\sa\nhigh\slevel.\s\sAlso\sadjust\ssome\ssymbolic\snames\sand\sfix\sother\scomment\sissues\sin\nthat\sfile. -D 2014-04-02T18:58:49.259 +C Refactor\slocal\sobject\sand\smethod\snames\sin\svdbesort.c\sso\sthat\stheir\snames\nmore\sclosely\sreflect\stheir\sactual\suse. +D 2014-04-03T02:54:27.677 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 523283d7c3f499444df97d700503f3c9ddd746b7 +F src/vdbesort.c a4e349ff62196dc3d98eacb0f9281e2468c44cab F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 47e702bd8392bc50c4edaf6a2c8c499af87b520e -R a364509f4308441b3af07ec467078f09 +P eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b +R 0d1e8a8cee97e3fb0928896d3dcac011 U drh -Z a6f0e24eb45caefc4a14c210e684429c +Z 523d07a773fda305bad8a184629c5049 diff --git a/manifest.uuid b/manifest.uuid index 30ed24d4f7..0af0f90890 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b \ No newline at end of file +d284e30eb1db144965fa85566e4234e30464350b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index c5d13c685c..6b94ddfebf 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -91,11 +91,11 @@ /* ** Private objects used by the sorter */ -typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SortSubtask SortSubtask; -typedef struct SorterRecord SorterRecord; -typedef struct SorterMerger SorterMerger; -typedef struct FileWriter FileWriter; +typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ +typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ +typedef struct SorterRecord SorterRecord; /* A record being sorted */ +typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ /* @@ -107,22 +107,22 @@ typedef struct FileWriter FileWriter; /* ** Sorting is divided up into smaller subtasks. Each subtask is controlled -** by an instance of this object. Subtask might run in either the main thread +** by an instance of this object. A Subtask might run in either the main thread ** or in a background thread. ** -** Exactly VdbeSorter.nThread instances of this object are allocated +** Exactly VdbeSorter.nTask instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. VdbeSorter.nThread is set to the number of worker threads allowed +** way. VdbeSorter.nTask is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SortSubtask.bDone -** is set to 0 and the SortSubtask.pThread variable set to point to the +** is set to 0 and the SortSubtask.pTask variable set to point to the ** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main -** thread that joining SortSubtask.pThread will not block) before the thread -** exits. SortSubtask.pThread and bDone are always cleared after the +** thread that joining SortSubtask.pTask will not block) before the thread +** exits. SortSubtask.pTask and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[VdbeSorter.nThread-1]) +** One object (specifically, VdbeSorter.aTask[VdbeSorter.nTask-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SortSubtask.eWork, @@ -142,7 +142,7 @@ typedef struct FileWriter FileWriter; */ struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ - int bDone; /* Set to true by pThread when finished */ + int bDone; /* Set to true by pTask when finished */ sqlite3_vfs *pVfs; /* VFS used to open temporary files */ KeyInfo *pKeyInfo; /* How to compare records */ @@ -151,7 +151,7 @@ struct SortSubtask { u8 eWork; /* One of the SORT_SUBTASK_* constants */ int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ - SorterRecord *pList; /* List of records for pThread to sort */ + SorterRecord *pList; /* List of records for pTask to sort */ int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ @@ -162,29 +162,23 @@ struct SortSubtask { /* -** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** The MergeEngine object is used to combine two or more smaller PMAs into +** one big PMA using a merge operation. Separate PMAs all need to be +** combined into one big PMA in order to be able to step through the sorted +** records in order. ** -** As keys are added to the sorter, they are written to disk in a series -** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly -** the same as the cache-size allowed for temporary databases. In order -** to allow the caller to extract keys from the sorter in sorted order, -** all PMAs currently stored on disk must be merged together. This comment -** describes the data structure used to do so. The structure supports -** merging any number of arrays in a single pass with no redundant comparison -** operations. -** -** The aIter[] array contains an iterator for each of the PMAs being merged. -** An aIter[] iterator either points to a valid key or else is at EOF. For -** the purposes of the paragraphs below, we assume that the array is actually -** N elements in size, where N is the smallest power of 2 greater to or equal -** to the number of iterators being merged. The extra aIter[] elements are -** treated as if they are empty (always at EOF). +** The aIter[] array contains a PmaReader object for each of the PMAs being +** merged. An aIter[] object either points to a valid key or else is at EOF. +** For the purposes of the paragraphs below, we assume that the array is +** actually N elements in size, where N is the smallest power of 2 greater +** to or equal to the number of PMAs being merged. The extra aIter[] elements +** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in -** the VdbeSorter.nTree variable. +** the MergeEngine.nTree variable. ** ** The final (N/2) elements of aTree[] contain the results of comparing -** pairs of iterator keys together. Element i contains the result of +** pairs of PMA keys together. Element i contains the result of ** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** @@ -230,10 +224,10 @@ struct SortSubtask { ** key comparison operations are required, where N is the number of segments ** being merged (rounded up to the next power of 2). */ -struct SorterMerger { - int nTree; /* Used size of aTree/aIter (power of 2) */ - int *aTree; /* Current state of incremental merge */ - VdbeSorterIter *aIter; /* Array of iterators to merge data from */ +struct MergeEngine { + int nTree; /* Used size of aTree/aIter (power of 2) */ + int *aTree; /* Current state of incremental merge */ + PmaReader *aIter; /* Array of iterators to merge data from */ }; /* @@ -246,20 +240,21 @@ struct VdbeSorter { int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - SorterMerger *pMerger; /* For final merge of PMAs (by caller) */ + MergeEngine *pMerger; /* For final merge of PMAs (by caller) */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ - int nThread; /* Size of aThread[] array */ - SortSubtask aThread[1]; + int nTask; /* Size of aTask[] array */ + SortSubtask aTask[1]; /* One or more subtasks */ }; /* -** The following type is an iterator for a PMA. It caches the current key in -** variables nKey/aKey. If the iterator is at EOF, pFile==0. +** An instance of the following object is used to read records out of a +** PMA, in sorted order. The next key to be read is cached in nKey/aKey. +** pFile==0 at EOF. */ -struct VdbeSorterIter { +struct PmaReader { i64 iReadOff; /* Current read offset */ i64 iEof; /* 1 byte past EOF for this iterator */ int nAlloc; /* Bytes of space at aAlloc */ @@ -273,12 +268,14 @@ struct VdbeSorterIter { }; /* -** An instance of this structure is used to organize the stream of records -** being written to files by the merge-sort code into aligned, page-sized -** blocks. Doing all I/O in aligned page-sized blocks helps I/O to go -** faster on many operating systems. +** An instance of this object is used for writing a PMA. +** +** The PMA is written one record at a time. Each record is of an arbitrary +** size. But I/O is more efficient if it occurs in page-sized blocks where +** each block is aligned on a page boundary. This object caches writes to +** the PMA so that aligned, page-size blocks are written. */ -struct FileWriter { +struct PmaWriter { int eFWErr; /* Non-zero if in an error state */ u8 *aBuffer; /* Pointer to write buffer */ int nBuffer; /* Size of write buffer in bytes */ @@ -289,8 +286,8 @@ struct FileWriter { }; /* -** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord. +** This object is the header on a single record while that record is being +** held in memory and prior to being written out as part of a PMA. ** ** How the linked list is connected depends on how memory is being managed ** by this module. If using a separate allocation for each in-memory record @@ -307,11 +304,12 @@ struct FileWriter { ** vdbeSorterSort() for details. */ struct SorterRecord { - int nVal; + int nVal; /* Size of the record in bytes */ union { SorterRecord *pNext; /* Pointer to next record in list */ int iNext; /* Offset within aMemory of next record */ } u; + /* The data for the record immediately follows this header */ }; /* Return a pointer to the buffer containing the record data for SorterRecord @@ -325,18 +323,18 @@ struct SorterRecord { ** page size in bytes. */ #define SORTER_MIN_WORKING 10 -/* Maximum number of segments to merge in a single pass. */ +/* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 /* -** Free all memory belonging to the VdbeSorterIter object passed as the second +** Free all memory belonging to the PmaReader object passed as the second ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(VdbeSorterIter *pIter){ +static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - memset(pIter, 0, sizeof(VdbeSorterIter)); + memset(pIter, 0, sizeof(PmaReader)); } /* @@ -348,8 +346,8 @@ static void vdbeSorterIterZero(VdbeSorterIter *pIter){ ** The buffer indicated by *ppOut may only be considered valid until the ** next call to this function. */ -static int vdbeSorterIterRead( - VdbeSorterIter *p, /* Iterator */ +static int vdbePmaReadBlob( + PmaReader *p, /* Iterator */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ @@ -419,13 +417,13 @@ static int vdbeSorterIterRead( /* The following loop copies up to p->nBuffer bytes per iteration into ** the p->aAlloc[] buffer. */ while( nRem>0 ){ - int rc; /* vdbeSorterIterRead() return code */ + int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ u8 *aNext; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(p, nCopy, &aNext); + rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -442,7 +440,7 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ +static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ int iBuf; if( p->aMap ){ @@ -455,7 +453,7 @@ static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ u8 aVarint[16], *a; int i = 0, rc; do{ - rc = vdbeSorterIterRead(p, 1, &a); + rc = vdbePmaReadBlob(p, 1, &a); if( rc ) return rc; aVarint[(i++)&0xf] = a[0]; }while( (a[0]&0x80)!=0 ); @@ -471,20 +469,20 @@ static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbeSorterIterNext(VdbeSorterIter *pIter){ +static int vdbePmaReaderNext(PmaReader *pIter){ int rc; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ - vdbeSorterIterZero(pIter); + vdbePmaReaderClear(pIter); return SQLITE_OK; } - rc = vdbeSorterIterVarint(pIter, &nRec); + rc = vdbePmaReadVarint(pIter, &nRec); if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(pIter, (int)nRec, &pIter->aKey); + rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); } return rc; @@ -496,27 +494,27 @@ static int vdbeSorterIterNext(VdbeSorterIter *pIter){ ** leaves the iterator pointing to the first key in the PMA (or EOF if the ** PMA is empty). */ -static int vdbeSorterIterInit( - SortSubtask *pThread, /* Thread context */ - i64 iStart, /* Start offset in pThread->pTemp1 */ - VdbeSorterIter *pIter, /* Iterator to populate */ +static int vdbePmaReaderInit( + SortSubtask *pTask, /* Thread context */ + i64 iStart, /* Start offset in pTask->pTemp1 */ + PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc = SQLITE_OK; - int nBuf = pThread->pgsz; + int nBuf = pTask->pgsz; void *pMap = 0; /* Mapping of temp file */ - assert( pThread->iTemp1Off>iStart ); + assert( pTask->iTemp1Off>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pThread->pTemp1; + pIter->pFile = pTask->pTemp1; pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); }else{ rc = SQLITE_NOMEM; } @@ -533,11 +531,11 @@ static int vdbeSorterIterInit( int iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pThread->iTemp1Off ){ - nRead = (int)(pThread->iTemp1Off - iStart); + if( (iStart + nRead) > pTask->iTemp1Off ){ + nRead = (int)(pTask->iTemp1Off - iStart); } rc = sqlite3OsRead( - pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + pTask->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } @@ -547,14 +545,14 @@ static int vdbeSorterIterInit( if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pThread->iTemp1Off; - rc = vdbeSorterIterVarint(pIter, &nByte); + pIter->iEof = pTask->iTemp1Off; + rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(pIter); + rc = vdbePmaReaderNext(pIter); } return rc; } @@ -562,25 +560,25 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences +** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** -** Before returning, object (pThread->pUnpacked) is populated with the +** Before returning, object (pTask->pUnpacked) is populated with the ** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it -** is assumed that the (pThread->pUnpacked) structure already contains the +** is assumed that the (pTask->pUnpacked) structure already contains the ** unpacked key to use as key2. ** -** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set +** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( - SortSubtask *pThread, /* Thread context (for pKeyInfo) */ + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ - UnpackedRecord *r2 = pThread->pUnpacked; + UnpackedRecord *r2 = pTask->pUnpacked; if( pKey2 ){ - sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pTask->pKeyInfo, nKey2, pKey2, r2); } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } @@ -591,15 +589,15 @@ static int vdbeSorterCompare( ** value to recalculate. */ static int vdbeSorterDoCompare( - SortSubtask *pThread, - SorterMerger *pMerger, + SortSubtask *pTask, + MergeEngine *pMerger, int iOut ){ int i1; int i2; int iRes; - VdbeSorterIter *p1; - VdbeSorterIter *p2; + PmaReader *p1; + PmaReader *p2; assert( iOutnTree && iOut>0 ); @@ -620,9 +618,9 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ + assert( pTask->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ res = vdbeSorterCompare( - pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey + pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -638,9 +636,13 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. */ -int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit( + sqlite3 *db, /* Database connection (for malloc()) */ + int nField, /* Number of key fields in each record */ + VdbeCursor *pCsr /* Cursor that holds the new sorter */ +){ int pgsz; /* Page size of main database */ - int i; /* Used to iterate through aThread[] */ + int i; /* Used to iterate through aTask[] */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ @@ -664,12 +666,12 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ if( nField && nWorker==0 ) pKeyInfo->nField = nField; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->nThread = nWorker + 1; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - pThread->pKeyInfo = pKeyInfo; - pThread->pVfs = db->pVfs; - pThread->pgsz = pgsz; + pSorter->nTask = nWorker + 1; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + pTask->pKeyInfo = pKeyInfo; + pTask->pVfs = db->pVfs; + pTask->pgsz = pgsz; } if( !sqlite3TempInMemory(db) ){ @@ -706,22 +708,22 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } /* -** Free all resources owned by the object indicated by argument pThread. All -** fields of *pThread are zeroed before returning. +** Free all resources owned by the object indicated by argument pTask. All +** fields of *pTask are zeroed before returning. */ -static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ - sqlite3DbFree(db, pThread->pUnpacked); - pThread->pUnpacked = 0; - if( pThread->aListMemory==0 ){ - vdbeSorterRecordFree(0, pThread->pList); +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ + sqlite3DbFree(db, pTask->pUnpacked); + pTask->pUnpacked = 0; + if( pTask->aListMemory==0 ){ + vdbeSorterRecordFree(0, pTask->pList); }else{ - sqlite3_free(pThread->aListMemory); - pThread->aListMemory = 0; + sqlite3_free(pTask->aListMemory); + pTask->aListMemory = 0; } - pThread->pList = 0; - if( pThread->pTemp1 ){ - sqlite3OsCloseFree(pThread->pTemp1); - pThread->pTemp1 = 0; + pTask->pList = 0; + if( pTask->pTemp1 ){ + sqlite3OsCloseFree(pTask->pTemp1); + pTask->pTemp1 = 0; } } @@ -732,13 +734,13 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - if( pThread->pThread ){ + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + if( pTask->pTask ){ void *pRet; - int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); - pThread->pThread = 0; - pThread->bDone = 0; + int rc2 = sqlite3ThreadJoin(pTask->pTask, &pRet); + pTask->pTask = 0; + pTask->bDone = 0; if( rc==SQLITE_OK ) rc = rc2; if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); } @@ -750,34 +752,34 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new SorterMerger object with space for nIter iterators. +** Allocate a new MergeEngine object with space for nIter iterators. */ -static SorterMerger *vdbeSorterMergerNew(int nIter){ +static MergeEngine *vdbeMergeEngineNew(int nIter){ int N = 2; /* Smallest power of two >= nIter */ int nByte; /* Total bytes of space to allocate */ - SorterMerger *pNew; /* Pointer to allocated object to return */ + MergeEngine *pNew; /* Pointer to allocated object to return */ assert( nIter<=SORTER_MAX_MERGE_COUNT ); while( NnTree = N; - pNew->aIter = (VdbeSorterIter*)&pNew[1]; + pNew->aIter = (PmaReader*)&pNew[1]; pNew->aTree = (int*)&pNew->aIter[N]; } return pNew; } /* -** Free the SorterMerger object passed as the only argument. +** Free the MergeEngine object passed as the only argument. */ -static void vdbeSorterMergerFree(SorterMerger *pMerger){ +static void vdbeMergeEngineFree(MergeEngine *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ - vdbeSorterIterZero(&pMerger->aIter[i]); + vdbePmaReaderClear(&pMerger->aIter[i]); } } sqlite3_free(pMerger); @@ -788,12 +790,12 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ */ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; - vdbeSorterJoinAll(pSorter, SQLITE_OK); - vdbeSorterMergerFree(pSorter->pMerger); + (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + vdbeMergeEngineFree(pSorter->pMerger); pSorter->pMerger = 0; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - vdbeSortSubtaskCleanup(db, pThread); + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + vdbeSortSubtaskCleanup(db, pTask); } if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); @@ -811,7 +813,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - vdbeSorterMergerFree(pSorter->pMerger); + vdbeMergeEngineFree(pSorter->pMerger); sqlite3_free(pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; @@ -842,7 +844,7 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - SortSubtask *pThread, /* Calling thread context */ + SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -853,7 +855,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal); + res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; @@ -872,11 +874,11 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pThread->pList. Return +** Sort the linked list of records headed at pTask->pList. Return ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SortSubtask *pThread){ +static int vdbeSorterSort(SortSubtask *pTask){ int i; SorterRecord **aSlot; SorterRecord *p; @@ -886,15 +888,15 @@ static int vdbeSorterSort(SortSubtask *pThread){ return SQLITE_NOMEM; } - p = pThread->pList; + p = pTask->pList; while( p ){ SorterRecord *pNext; - if( pThread->aListMemory ){ - if( (u8*)p==pThread->aListMemory ){ + if( pTask->aListMemory ){ + if( (u8*)p==pTask->aListMemory ){ pNext = 0; }else{ - assert( p->u.iNextaListMemory) ); - pNext = (SorterRecord*)&pThread->aListMemory[p->u.iNext]; + assert( p->u.iNextaListMemory) ); + pNext = (SorterRecord*)&pTask->aListMemory[p->u.iNext]; } }else{ pNext = p->u.pNext; @@ -902,7 +904,7 @@ static int vdbeSorterSort(SortSubtask *pThread){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pThread, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -911,24 +913,24 @@ static int vdbeSorterSort(SortSubtask *pThread){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pThread, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); } - pThread->pList = p; + pTask->pList = p; sqlite3_free(aSlot); return SQLITE_OK; } /* -** Initialize a file-writer object. +** Initialize a PMA-writer object. */ -static void fileWriterInit( +static void vdbePmaWriterInit( sqlite3_file *pFile, /* File to write to */ - FileWriter *p, /* Object to populate */ + PmaWriter *p, /* Object to populate */ int nBuf, /* Buffer size */ i64 iStart /* Offset of pFile to begin writing at */ ){ - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ p->eFWErr = SQLITE_NOMEM; @@ -941,10 +943,10 @@ static void fileWriterInit( } /* -** Write nData bytes of data to the file-write object. Return SQLITE_OK +** Write nData bytes of data to the PMA. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ +static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ int nRem = nData; while( nRem>0 && p->eFWErr==0 ){ int nCopy = nRem; @@ -969,15 +971,15 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ } /* -** Flush any buffered data to disk and clean up the file-writer object. -** The results of using the file-writer after this call are undefined. +** Flush any buffered data to disk and clean up the PMA-writer object. +** The results of using the PMA-writer after this call are undefined. ** Return SQLITE_OK if flushing the buffered data succeeds or is not ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(FileWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFile, @@ -988,19 +990,19 @@ static int fileWriterFinish(FileWriter *p, i64 *piEof){ *piEof = (p->iWriteOff + p->iBufEnd); sqlite3_free(p->aBuffer); rc = p->eFWErr; - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); return rc; } /* -** Write value iVal encoded as a varint to the file-write object. Return +** Write value iVal encoded as a varint to the PMA. Return ** SQLITE_OK if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ +static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); - fileWriterWrite(p, aByte, nByte); + vdbePmaWriteBlob(p, aByte, nByte); } #if SQLITE_MAX_MMAP_SIZE>0 @@ -1040,25 +1042,25 @@ static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SortSubtask *pThread){ +static int vdbeSorterListToPMA(SortSubtask *pTask){ int rc = SQLITE_OK; /* Return code */ - FileWriter writer; /* Object used to write to the file */ + PmaWriter writer; /* Object used to write to the file */ - memset(&writer, 0, sizeof(FileWriter)); - assert( pThread->nInMemory>0 ); + memset(&writer, 0, sizeof(PmaWriter)); + assert( pTask->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( pThread->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pThread->pVfs, &pThread->pTemp1); - assert( rc!=SQLITE_OK || pThread->pTemp1 ); - assert( pThread->iTemp1Off==0 ); - assert( pThread->nPMA==0 ); + if( pTask->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1); + assert( rc!=SQLITE_OK || pTask->pTemp1 ); + assert( pTask->iTemp1Off==0 ); + assert( pTask->nPMA==0 ); } /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ rc = vdbeSorterExtendFile( - pThread->pTemp1, pThread->iTemp1Off + pThread->nInMemory + 9 + pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1066,46 +1068,47 @@ static int vdbeSorterListToPMA(SortSubtask *pThread){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(pThread->pTemp1, &writer, pThread->pgsz, pThread->iTemp1Off); - pThread->nPMA++; - fileWriterWriteVarint(&writer, pThread->nInMemory); - for(p=pThread->pList; p; p=pNext){ + vdbePmaWriterInit(pTask->pTemp1, &writer, pTask->pgsz, + pTask->iTemp1Off); + pTask->nPMA++; + vdbePmaWriteVarint(&writer, pTask->nInMemory); + for(p=pTask->pList; p; p=pNext){ pNext = p->u.pNext; - fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, SRVAL(p), p->nVal); - if( pThread->aListMemory==0 ) sqlite3_free(p); + vdbePmaWriteVarint(&writer, p->nVal); + vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); + if( pTask->aListMemory==0 ) sqlite3_free(p); } - pThread->pList = p; - rc = fileWriterFinish(&writer, &pThread->iTemp1Off); + pTask->pList = p; + rc = vdbePmaWriterFinish(&writer, &pTask->iTemp1Off); } - assert( pThread->pList==0 || rc!=SQLITE_OK ); + assert( pTask->pList==0 || rc!=SQLITE_OK ); return rc; } /* -** Advance the SorterMerger iterator passed as the second argument to +** Advance the MergeEngine iterator passed as the second argument to ** the next entry. Set *pbEof to true if this means the iterator has ** reached EOF. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeSorterNext( - SortSubtask *pThread, - SorterMerger *pMerger, + SortSubtask *pTask, + MergeEngine *pMerger, int *pbEof ){ int rc; int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ /* Advance the current iterator */ - rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); + rc = vdbePmaReaderNext(&pMerger->aIter[iPrev]); /* Update contents of aTree[] */ if( rc==SQLITE_OK ){ int i; /* Index of aTree[] to recalculate */ - VdbeSorterIter *pIter1; /* First iterator to compare */ - VdbeSorterIter *pIter2; /* Second iterator to compare */ + PmaReader *pIter1; /* First iterator to compare */ + PmaReader *pIter2; /* Second iterator to compare */ u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ /* Find the first two iterators to compare. The one that was just @@ -1122,19 +1125,19 @@ static int vdbeSorterNext( }else if( pIter2->pFile==0 ){ iRes = -1; }else{ - iRes = vdbeSorterCompare(pThread, + iRes = vdbeSorterCompare(pTask, pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey ); } /* If pIter1 contained the smaller value, set aTree[i] to its index. ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pThread->pUnpacked, so set + ** case there is no cache of pIter2 in pTask->pUnpacked, so set ** pKey2 to point to the record belonging to pIter2. ** ** Alternatively, if pIter2 contains the smaller of the two values, ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() - ** was actually called above, then pThread->pUnpacked now contains + ** was actually called above, then pTask->pUnpacked now contains ** a value equivalent to pIter2. So set pKey2 to NULL to prevent ** vdbeSorterCompare() from decoding pIter2 again. ** @@ -1163,59 +1166,59 @@ static int vdbeSorterNext( */ static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; - SortSubtask *pThread = (SortSubtask*)pCtx; + SortSubtask *pTask = (SortSubtask*)pCtx; - assert( pThread->eWork==SORT_SUBTASK_SORT - || pThread->eWork==SORT_SUBTASK_TO_PMA - || pThread->eWork==SORT_SUBTASK_CONS + assert( pTask->eWork==SORT_SUBTASK_SORT + || pTask->eWork==SORT_SUBTASK_TO_PMA + || pTask->eWork==SORT_SUBTASK_CONS ); - assert( pThread->bDone==0 ); + assert( pTask->bDone==0 ); - if( pThread->pUnpacked==0 ){ + if( pTask->pUnpacked==0 ){ char *pFree; - pThread->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pThread->pKeyInfo, 0, 0, &pFree + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree ); - assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ){ rc = SQLITE_NOMEM; goto thread_out; } - pThread->pUnpacked->nField = pThread->pKeyInfo->nField; - pThread->pUnpacked->errCode = 0; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; } - if( pThread->eWork==SORT_SUBTASK_CONS ){ - assert( pThread->pList==0 ); - while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ - int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); + if( pTask->eWork==SORT_SUBTASK_CONS ){ + assert( pTask->pList==0 ); + while( pTask->nPMA>pTask->nConsolidate && rc==SQLITE_OK ){ + int nIter = MIN(pTask->nPMA, SORTER_MAX_MERGE_COUNT); sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - SorterMerger *pMerger; /* Object for reading/merging PMA data */ + MergeEngine *pMerger; /* Object for reading/merging PMA data */ i64 iReadOff = 0; /* Offset in pTemp1 to read from */ i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ int i; /* Allocate a merger object to merge PMAs together. */ - pMerger = vdbeSorterMergerNew(nIter); + pMerger = vdbeMergeEngineNew(nIter); if( pMerger==0 ){ rc = SQLITE_NOMEM; break; } /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pThread->pVfs, &pTemp2); + rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pThread->iTemp1Off); + rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off); } if( rc!=SQLITE_OK ){ - vdbeSorterMergerFree(pMerger); + vdbeMergeEngineFree(pMerger); break; } /* This loop runs once for each output PMA. Each output PMA is made ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ - FileWriter writer; /* Object for writing data to pTemp2 */ + for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + PmaWriter writer; /* Object for writing data to pTemp2 */ i64 nOut = 0; /* Bytes of data in output PMA */ int bEof = 0; int rc2; @@ -1225,54 +1228,54 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ** if that is fewer). */ int iIter; for(iIter=0; iIteraIter[iIter]; - rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nOut); + PmaReader *pIter = &pMerger->aIter[iIter]; + rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nOut); iReadOff = pIter->iEof; - if( iReadOff>=pThread->iTemp1Off || rc!=SQLITE_OK ) break; + if( iReadOff>=pTask->iTemp1Off || rc!=SQLITE_OK ) break; } for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ - rc = vdbeSorterDoCompare(pThread, pMerger, iIter); + rc = vdbeSorterDoCompare(pTask, pMerger, iIter); } - fileWriterInit(pTemp2, &writer, pThread->pgsz, iWriteOff); - fileWriterWriteVarint(&writer, nOut); + vdbePmaWriterInit(pTemp2, &writer, pTask->pgsz, iWriteOff); + vdbePmaWriteVarint(&writer, nOut); while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; + PmaReader *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; assert( pIter->pFile!=0 ); /* pIter is not at EOF */ - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = vdbeSorterNext(pThread, pMerger, &bEof); + vdbePmaWriteVarint(&writer, pIter->nKey); + vdbePmaWriteBlob(&writer, pIter->aKey, pIter->nKey); + rc = vdbeSorterNext(pTask, pMerger, &bEof); } - rc2 = fileWriterFinish(&writer, &iWriteOff); + rc2 = vdbePmaWriterFinish(&writer, &iWriteOff); if( rc==SQLITE_OK ) rc = rc2; } - vdbeSorterMergerFree(pMerger); - sqlite3OsCloseFree(pThread->pTemp1); - pThread->pTemp1 = pTemp2; - pThread->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pThread->iTemp1Off = iWriteOff; + vdbeMergeEngineFree(pMerger); + sqlite3OsCloseFree(pTask->pTemp1); + pTask->pTemp1 = pTemp2; + pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); + pTask->iTemp1Off = iWriteOff; } }else{ - /* Sort the pThread->pList list */ - rc = vdbeSorterSort(pThread); + /* Sort the pTask->pList list */ + rc = vdbeSorterSort(pTask); /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pThread->eWork==SORT_SUBTASK_TO_PMA ){ + if( rc==SQLITE_OK && pTask->eWork==SORT_SUBTASK_TO_PMA ){ #ifdef SQLITE_DEBUG - i64 nExpect = pThread->nInMemory - + sqlite3VarintLen(pThread->nInMemory) - + pThread->iTemp1Off; + i64 nExpect = pTask->nInMemory + + sqlite3VarintLen(pTask->nInMemory) + + pTask->iTemp1Off; #endif - rc = vdbeSorterListToPMA(pThread); - assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); + rc = vdbeSorterListToPMA(pTask); + assert( rc!=SQLITE_OK || (nExpect==pTask->iTemp1Off) ); } } thread_out: - pThread->bDone = 1; - if( rc==SQLITE_OK && pThread->pUnpacked->errCode ){ - assert( pThread->pUnpacked->errCode==SQLITE_NOMEM ); + pTask->bDone = 1; + if( rc==SQLITE_OK && pTask->pUnpacked->errCode ){ + assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); rc = SQLITE_NOMEM; } return SQLITE_INT_TO_PTR(rc); @@ -1282,10 +1285,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ -static int vdbeSorterRunThread(SortSubtask *pThread){ - int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pThread) ); - assert( pThread->bDone ); - pThread->bDone = 0; +static int vdbeSorterRunTask(SortSubtask *pTask){ + int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pTask) ); + assert( pTask->bDone ); + pTask->bDone = 0; return rc; } @@ -1299,53 +1302,53 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SortSubtask *pThread = 0; /* Thread context used to create new PMA */ - int nWorker = (pSorter->nThread-1); + SortSubtask *pTask = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nTask-1); pSorter->bUsePMA = 1; for(i=0; iiPrev + i + 1) % nWorker; - pThread = &pSorter->aThread[iTest]; + pTask = &pSorter->aTask[iTest]; #if SQLITE_MAX_WORKER_THREADS>0 - if( pThread->bDone ){ + if( pTask->bDone ){ void *pRet; - assert( pThread->pThread ); - rc = sqlite3ThreadJoin(pThread->pThread, &pRet); - pThread->pThread = 0; - pThread->bDone = 0; + assert( pTask->pTask ); + rc = sqlite3ThreadJoin(pTask->pTask, &pRet); + pTask->pTask = 0; + pTask->bDone = 0; if( rc==SQLITE_OK ){ rc = SQLITE_PTR_TO_INT(pRet); } } #endif - if( pThread->pThread==0 ) break; - pThread = 0; + if( pTask->pThread==0 ) break; + pTask = 0; } - if( pThread==0 ){ - pThread = &pSorter->aThread[nWorker]; + if( pTask==0 ){ + pTask = &pSorter->aTask[nWorker]; } - pSorter->iPrev = (pThread - pSorter->aThread); + pSorter->iPrev = (pTask - pSorter->aTask); if( rc==SQLITE_OK ){ - assert( pThread->pThread==0 && pThread->bDone==0 ); - pThread->eWork = SORT_SUBTASK_TO_PMA; - pThread->pList = pSorter->pRecord; - pThread->nInMemory = pSorter->nInMemory; + assert( pTask->pThread==0 && pTask->bDone==0 ); + pTask->eWork = SORT_SUBTASK_TO_PMA; + pTask->pList = pSorter->pRecord; + pTask->nInMemory = pSorter->nInMemory; pSorter->nInMemory = 0; pSorter->pRecord = 0; if( pSorter->aMemory ){ - u8 *aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; + u8 *aMem = pTask->aListMemory; + pTask->aListMemory = pSorter->aMemory; pSorter->aMemory = aMem; } #if SQLITE_MAX_WORKER_THREADS>0 - if( !bFg && pThread!=&pSorter->aThread[nWorker] ){ + if( !bFg && pTask!=&pSorter->aTask[nWorker] ){ /* Launch a background thread for this operation */ - void *pCtx = (void*)pThread; - assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); - if( pThread->aListMemory ){ + void *pCtx = (void*)pTask; + assert( pSorter->aMemory==0 || pTask->aListMemory!=0 ); + if( pTask->aListMemory ){ if( pSorter->aMemory==0 ){ pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); if( pSorter->aMemory==0 ) return SQLITE_NOMEM; @@ -1353,17 +1356,17 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSortSubtaskMain, pCtx); + rc = sqlite3ThreadCreate(&pTask->pTask, vdbeSortSubtaskMain, pCtx); }else #endif { /* Use the foreground thread for this operation */ - rc = vdbeSorterRunThread(pThread); + rc = vdbeSorterRunTask(pTask); if( rc==SQLITE_OK ){ - u8 *aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; + u8 *aMem = pTask->aListMemory; + pTask->aListMemory = pSorter->aMemory; pSorter->aMemory = aMem; - assert( pThread->pList==0 ); + assert( pTask->pList==0 ); } } } @@ -1469,15 +1472,16 @@ int sqlite3VdbeSorterWrite( static int vdbeSorterCountPMA(VdbeSorter *pSorter){ int nPMA = 0; int i; - for(i=0; inThread; i++){ - nPMA += pSorter->aThread[i].nPMA; + for(i=0; inTask; i++){ + nPMA += pSorter->aTask[i].nPMA; } return nPMA; } /* -** Once the sorter has been populated, this function is called to prepare -** for iterating through its contents in sorted order. +** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, +** this function is called to prepare for iterating through the records +** in sorted order. */ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; @@ -1490,16 +1494,16 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ if( pSorter->pRecord ){ - SortSubtask *pThread = &pSorter->aThread[0]; + SortSubtask *pTask = &pSorter->aTask[0]; *pbEof = 0; - pThread->pList = pSorter->pRecord; - pThread->eWork = SORT_SUBTASK_SORT; - assert( pThread->aListMemory==0 ); - pThread->aListMemory = pSorter->aMemory; - rc = vdbeSorterRunThread(pThread); - pThread->aListMemory = 0; - pSorter->pRecord = pThread->pList; - pThread->pList = 0; + pTask->pList = pSorter->pRecord; + pTask->eWork = SORT_SUBTASK_SORT; + assert( pTask->aListMemory==0 ); + pTask->aListMemory = pSorter->aMemory; + rc = vdbeSorterRunTask(pTask); + pTask->aListMemory = 0; + pSorter->pRecord = pTask->pList; + pTask->pList = 0; }else{ *pbEof = 1; } @@ -1518,20 +1522,20 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** some of them together so that this is no longer the case. */ if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; - for(i=0; rc==SQLITE_OK && inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; - pThread->eWork = SORT_SUBTASK_CONS; + for(i=0; rc==SQLITE_OK && inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + if( pTask->pTemp1 ){ + pTask->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nTask; + pTask->eWork = SORT_SUBTASK_CONS; #if SQLITE_MAX_WORKER_THREADS>0 - if( i<(pSorter->nThread-1) ){ - void *pCtx = (void*)pThread; - rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSortSubtaskMain,pCtx); + if( i<(pSorter->nTask-1) ){ + void *pCtx = (void*)pTask; + rc = sqlite3ThreadCreate(&pTask->pTask,vdbeSortSubtaskMain,pCtx); }else #endif { - rc = vdbeSorterRunThread(pThread); + rc = vdbeSorterRunTask(pTask); } } } @@ -1546,31 +1550,31 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( rc==SQLITE_OK ){ int nIter = 0; /* Number of iterators used */ int i; - SorterMerger *pMerger; - for(i=0; inThread; i++){ - nIter += pSorter->aThread[i].nPMA; + MergeEngine *pMerger; + for(i=0; inTask; i++){ + nIter += pSorter->aTask[i].nPMA; } - pSorter->pMerger = pMerger = vdbeSorterMergerNew(nIter); + pSorter->pMerger = pMerger = vdbeMergeEngineNew(nIter); if( pMerger==0 ){ rc = SQLITE_NOMEM; }else{ int iIter = 0; int iThread = 0; - for(iThread=0; iThreadnThread; iThread++){ + for(iThread=0; iThreadnTask; iThread++){ int iPMA; i64 iReadOff = 0; - SortSubtask *pThread = &pSorter->aThread[iThread]; - for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ + SortSubtask *pTask = &pSorter->aTask[iThread]; + for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ i64 nDummy = 0; - VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; - rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nDummy); + PmaReader *pIter = &pMerger->aIter[iIter++]; + rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nDummy); iReadOff = pIter->iEof; } } for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(&pSorter->aThread[0], pMerger, i); + rc = vdbeSorterDoCompare(&pSorter->aTask[0], pMerger, i); } } } @@ -1589,7 +1593,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ int rc; /* Return code */ if( pSorter->pMerger ){ - rc = vdbeSorterNext(&pSorter->aThread[0], pSorter->pMerger, pbEof); + rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->u.pNext; @@ -1611,7 +1615,7 @@ static void *vdbeSorterRowkey( ){ void *pKey; if( pSorter->pMerger ){ - VdbeSorterIter *pIter; + PmaReader *pIter; pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; *pnKey = pIter->nKey; pKey = pIter->aKey; @@ -1663,7 +1667,7 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked; + UnpackedRecord *r2 = pSorter->aTask[0].pUnpacked; KeyInfo *pKeyInfo = pCsr->pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ From 6e4cc55e1f8e5cce7d7be6dd4fd5fd576d787a33 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Apr 2014 14:29:08 +0000 Subject: [PATCH 30/99] Fix minor errors causing compilation to fail with SQLITE_MAX_WORKER_THREADS set to a value greater than zero. FossilOrigin-Name: 0561272abf357a2f4709f6c02866e570d19cd344 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index a1d8fa4b08..2e20875ba9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\slocal\sobject\sand\smethod\snames\sin\svdbesort.c\sso\sthat\stheir\snames\nmore\sclosely\sreflect\stheir\sactual\suse. -D 2014-04-03T02:54:27.677 +C Fix\sminor\serrors\scausing\scompilation\sto\sfail\swith\sSQLITE_MAX_WORKER_THREADS\sset\sto\sa\svalue\sgreater\sthan\szero. +D 2014-04-03T14:29:08.251 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c a4e349ff62196dc3d98eacb0f9281e2468c44cab +F src/vdbesort.c 5e7ed44bb4f2af809b6d229ae00f97825efab89a F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b -R 0d1e8a8cee97e3fb0928896d3dcac011 -U drh -Z 523d07a773fda305bad8a184629c5049 +P d284e30eb1db144965fa85566e4234e30464350b +R 45d899d78ea7a6cd4a92080d8bb33ecf +U dan +Z 013157fb51930f7eb005a94358375580 diff --git a/manifest.uuid b/manifest.uuid index 0af0f90890..9cf01bf6ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d284e30eb1db144965fa85566e4234e30464350b \ No newline at end of file +0561272abf357a2f4709f6c02866e570d19cd344 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6b94ddfebf..78f5183b5d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -736,10 +736,10 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pTask ){ + if( pTask->pThread ){ void *pRet; - int rc2 = sqlite3ThreadJoin(pTask->pTask, &pRet); - pTask->pTask = 0; + int rc2 = sqlite3ThreadJoin(pTask->pThread, &pRet); + pTask->pThread = 0; pTask->bDone = 0; if( rc==SQLITE_OK ) rc = rc2; if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); @@ -1312,9 +1312,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ #if SQLITE_MAX_WORKER_THREADS>0 if( pTask->bDone ){ void *pRet; - assert( pTask->pTask ); - rc = sqlite3ThreadJoin(pTask->pTask, &pRet); - pTask->pTask = 0; + assert( pTask->pThread ); + rc = sqlite3ThreadJoin(pTask->pThread, &pRet); + pTask->pThread = 0; pTask->bDone = 0; if( rc==SQLITE_OK ){ rc = SQLITE_PTR_TO_INT(pRet); @@ -1356,7 +1356,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pTask->pTask, vdbeSortSubtaskMain, pCtx); + rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { @@ -1531,7 +1531,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nTask-1) ){ void *pCtx = (void*)pTask; - rc = sqlite3ThreadCreate(&pTask->pTask,vdbeSortSubtaskMain,pCtx); + rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { From 8930c2ab0cf4f38a7aef3db0cf5965c4c36583e1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Apr 2014 16:25:29 +0000 Subject: [PATCH 31/99] Fix an integer overflow problem in the sorter. FossilOrigin-Name: 9d3351b8d713232133dad149c73fb2a27c72abb1 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/main.c | 8 ++++++++ src/sqlite.h.in | 3 ++- src/sqliteInt.h | 1 + src/test1.c | 14 ++++++++++++++ src/vdbesort.c | 33 +++++++++++++++++++-------------- test/permutations.test | 1 + 8 files changed, 57 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 2e20875ba9..1e7716c432 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\serrors\scausing\scompilation\sto\sfail\swith\sSQLITE_MAX_WORKER_THREADS\sset\sto\sa\svalue\sgreater\sthan\szero. -D 2014-04-03T14:29:08.251 +C Fix\san\sinteger\soverflow\sproblem\sin\sthe\ssorter. +D 2014-04-03T16:25:29.778 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c d3655832585baef4c2356529a5c6ca5ca3bd7c1f +F src/main.c fcceb01d74a79c2d7984f33545b35b06da3bb1e8 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -219,15 +219,15 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e -F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a +F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3ed0fedb5b64ece395a2114b7c73417678f3e420 +F src/sqliteInt.h 78c89401120b062660427c7b642de4de7673bc46 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 31596bf8a9c0629f88e514a4ec864847c8946c4e +F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 5e7ed44bb4f2af809b6d229ae00f97825efab89a +F src/vdbesort.c 252d7ab7620649945b53289510a172bc73133f17 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -738,7 +738,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 +F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d284e30eb1db144965fa85566e4234e30464350b -R 45d899d78ea7a6cd4a92080d8bb33ecf +P 0561272abf357a2f4709f6c02866e570d19cd344 +R 8288f2959bddd3667c4349a94ca23e0f U dan -Z 013157fb51930f7eb005a94358375580 +Z 4eb0e7377049f06d09d1ea7ce591ab92 diff --git a/manifest.uuid b/manifest.uuid index 9cf01bf6ac..cdc8213641 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0561272abf357a2f4709f6c02866e570d19cd344 \ No newline at end of file +9d3351b8d713232133dad149c73fb2a27c72abb1 \ No newline at end of file diff --git a/src/main.c b/src/main.c index c1eaa6849a..f18d1a6507 100644 --- a/src/main.c +++ b/src/main.c @@ -2504,6 +2504,7 @@ static int openDatabase( db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex @@ -3330,6 +3331,13 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 78aa9c36e1..469504b149 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6129,7 +6129,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 -#define SQLITE_TESTCTRL_LAST 21 +#define SQLITE_TESTCTRL_SORTER_MMAP 22 +#define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0763f085ab..b802d7aab7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -981,6 +981,7 @@ struct sqlite3 { int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ diff --git a/src/test1.c b/src/test1.c index 4b485ce741..44e96c2c1f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5884,6 +5884,7 @@ static int test_test_control( int i; } aVerb[] = { { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, + { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, }; int iVerb; int iFlag; @@ -5911,6 +5912,19 @@ static int test_test_control( sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val); break; } + + case SQLITE_TESTCTRL_SORTER_MMAP: { + int val; + sqlite3 *db; + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR; + sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val); + break; + } } Tcl_ResetResult(interp); diff --git a/src/vdbesort.c b/src/vdbesort.c index 78f5183b5d..ece66d4c11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -144,7 +144,7 @@ struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ int bDone; /* Set to true by pTask when finished */ - sqlite3_vfs *pVfs; /* VFS used to open temporary files */ + sqlite3 *db; /* Database connection */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ @@ -514,7 +514,9 @@ static int vdbePmaReaderInit( if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + } }else{ rc = SQLITE_NOMEM; } @@ -670,8 +672,8 @@ int sqlite3VdbeSorterInit( for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pKeyInfo = pKeyInfo; - pTask->pVfs = db->pVfs; pTask->pgsz = pgsz; + pTask->db = db; } if( !sqlite3TempInMemory(db) ){ @@ -1015,17 +1017,20 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ - int rc = sqlite3OsTruncate(pFile, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); +static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ + int rc = SQLITE_OK; + if( nByte<=(i64)(db->nMaxSorterMmap) ){ + rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } } return rc; } #else -# define vdbeSorterExtendFile(x,y) SQLITE_OK +# define vdbeSorterExtendFile(x,y,z) SQLITE_OK #endif @@ -1051,7 +1056,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1); + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1); assert( rc!=SQLITE_OK || pTask->pTemp1 ); assert( pTask->iTemp1Off==0 ); assert( pTask->nPMA==0 ); @@ -1059,7 +1064,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile( + rc = vdbeSorterExtendFile(pTask->db, pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1206,9 +1211,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ } /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2); + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off); + rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); } if( rc!=SQLITE_OK ){ vdbeMergeEngineFree(pMerger); diff --git a/test/permutations.test b/test/permutations.test index 7f1485f831..4487af055b 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,6 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + sort3.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] From 6b5d4759cea2278aacd2f06bdd150bbe40c8fbea Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 02:13:26 +0000 Subject: [PATCH 32/99] Change vdbeSorterExtendFile() so that it makes a best effort to create the PMA file of the desired size, but does not return an error if unable. FossilOrigin-Name: 217814bc4b53fab7bdad433e24e8aef8998c38fe --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 15 ++++++--------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b7536a4c10..51e948c44c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\schanges\sfrom\strunk. -D 2014-04-03T16:42:21.478 +C Change\svdbeSorterExtendFile()\sso\sthat\sit\smakes\sa\sbest\seffort\sto\screate\sthe\nPMA\sfile\sof\sthe\sdesired\ssize,\sbut\sdoes\snot\sreturn\san\serror\sif\sunable. +D 2014-04-04T02:13:26.507 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 252d7ab7620649945b53289510a172bc73133f17 +F src/vdbesort.c 8da916fc74e78edd5bc95653206942e01710ac09 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9d3351b8d713232133dad149c73fb2a27c72abb1 d5513dfa23baa0b0a095aaf17d19aacd30dcef61 -R 6137d1586a278eeecb535398f89aa582 +P a0910079adde95245680dee59b43613b60903f10 +R ae0d06309ee79fa9e2b8e2b495c941f2 U drh -Z 41afe239279763eb22b85c89d966d688 +Z 2dc7a665c4d0ee5a145d17c4c35dd575 diff --git a/manifest.uuid b/manifest.uuid index 701e5c70e7..c76da35986 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0910079adde95245680dee59b43613b60903f10 \ No newline at end of file +217814bc4b53fab7bdad433e24e8aef8998c38fe \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ece66d4c11..f59e8f51f5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1017,17 +1017,15 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ - int rc = SQLITE_OK; +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ - rc = sqlite3OsTruncate(pFile, nByte); + int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } } - return rc; } #else # define vdbeSorterExtendFile(x,y,z) SQLITE_OK @@ -1064,7 +1062,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTask->db, + vdbeSorterExtendFile(pTask->db, pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1213,16 +1211,15 @@ static void *vdbeSortSubtaskMain(void *pCtx){ /* Open a second temp file to write merged data to */ rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); - } - if( rc!=SQLITE_OK ){ + vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); + }else{ vdbeMergeEngineFree(pMerger); break; } /* This loop runs once for each output PMA. Each output PMA is made ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + for(i=0; rc==SQLITE_OK && inPMA; i+=SORTER_MAX_MERGE_COUNT){ PmaWriter writer; /* Object for writing data to pTemp2 */ i64 nOut = 0; /* Bytes of data in output PMA */ int bEof = 0; From 8daefc2af05742f0c36ca7a992981283408a0c36 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 4 Apr 2014 07:52:44 +0000 Subject: [PATCH 33/99] Add test file sort3.test, which should have been part of commit [9d3351b8d7]. FossilOrigin-Name: dceed2c803fca23c83c02c448d5ae7c4698efee1 --- manifest | 13 +++++----- manifest.uuid | 2 +- test/sort3.test | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 test/sort3.test diff --git a/manifest b/manifest index 51e948c44c..ba8d195a88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\svdbeSorterExtendFile()\sso\sthat\sit\smakes\sa\sbest\seffort\sto\screate\sthe\nPMA\sfile\sof\sthe\sdesired\ssize,\sbut\sdoes\snot\sreturn\san\serror\sif\sunable. -D 2014-04-04T02:13:26.507 +C Add\stest\sfile\ssort3.test,\swhich\sshould\shave\sbeen\spart\sof\scommit\s[9d3351b8d7]. +D 2014-04-04T07:52:44.563 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -819,6 +819,7 @@ F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 +F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a0910079adde95245680dee59b43613b60903f10 -R ae0d06309ee79fa9e2b8e2b495c941f2 -U drh -Z 2dc7a665c4d0ee5a145d17c4c35dd575 +P 217814bc4b53fab7bdad433e24e8aef8998c38fe +R 3decea650c26960f7b93e4d30e9eb4e6 +U dan +Z 1dd1e90c2d62e06b6853f5bd55d6e7b2 diff --git a/manifest.uuid b/manifest.uuid index c76da35986..b284b74278 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -217814bc4b53fab7bdad433e24e8aef8998c38fe \ No newline at end of file +dceed2c803fca23c83c02c448d5ae7c4698efee1 \ No newline at end of file diff --git a/test/sort3.test b/test/sort3.test new file mode 100644 index 0000000000..9963dcccd3 --- /dev/null +++ b/test/sort3.test @@ -0,0 +1,63 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort3 + +# Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without. +# +foreach {itest limit} { + 1 5000000 + 2 0x7FFFFFFF +} { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $limit + do_execsql_test 1.$itest { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 20000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); + } { + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + } +} + +# Sort more than 2GB of data. At one point this was causing a problem. +# This test might take one minute or more to run. +# +do_execsql_test 2 { + PRAGMA cache_size = 20000; + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 2200000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); +} { + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 +} + +finish_test + From f7da5555da88a8b4660909bf350725bb25fedae5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 4 Apr 2014 21:40:38 +0000 Subject: [PATCH 34/99] Fix typo in a Windows threading support routine. FossilOrigin-Name: 5e3dfa27c71a666e122e3cf64897038ff8424800 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4728b7994c..b330e04b4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-04T18:37:36.101 +C Fix\stypo\sin\sa\sWindows\sthreading\ssupport\sroutine. +D 2014-04-04T21:40:38.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 6992f70cab8d5d8451a6b5641a9256d1749af87b +F src/threads.c b8e7232f2b9c9d148d6886117160de394d172f85 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dceed2c803fca23c83c02c448d5ae7c4698efee1 683dd379a293b2f330e1e4cd746f190527fe48ee -R ded894407cec029874d1f7230b628469 -U drh -Z 4f065338769b0de2d9e8b418eaae9ff0 +P 39ac79cffe716f88af0871bdd206231b6a6511ff +R 0cd4553a2bbfab9bd06e3f865048f05f +U mistachkin +Z cb7697a0807ce9ae67c5a22136134623 diff --git a/manifest.uuid b/manifest.uuid index d9a62a86f8..89d1ae46cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39ac79cffe716f88af0871bdd206231b6a6511ff \ No newline at end of file +5e3dfa27c71a666e122e3cf64897038ff8424800 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 96cb17cef7..eeab5379ce 100644 --- a/src/threads.c +++ b/src/threads.c @@ -149,7 +149,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask==0 ){ - rc = WAIT_OBJECT_O; + rc = WAIT_OBJECT_0; }else{ rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); From bf20a35d5f692d0c5e850d043f155797094dd99e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 22:44:59 +0000 Subject: [PATCH 35/99] Fix harmless compiler warnings. FossilOrigin-Name: e54dded2012f0ab486ee138e9bd57c528af33980 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 2 +- src/sqliteInt.h | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b330e04b4c..2eb2b1da22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sa\sWindows\sthreading\ssupport\sroutine. -D 2014-04-04T21:40:38.243 +C Fix\sharmless\scompiler\swarnings. +D 2014-04-04T22:44:59.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1 +F src/build.c e120a3693ac78ab3e74048205f07a3a522d9b982 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -222,7 +222,7 @@ F src/shell.c afc0b1a5a646d287142ef0c9a2a6e3139d57cba2 F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 78c89401120b062660427c7b642de4de7673bc46 +F src/sqliteInt.h 533154bf9d8401ff3e6f8030f4373774c0a6ad41 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 39ac79cffe716f88af0871bdd206231b6a6511ff -R 0cd4553a2bbfab9bd06e3f865048f05f -U mistachkin -Z cb7697a0807ce9ae67c5a22136134623 +P 5e3dfa27c71a666e122e3cf64897038ff8424800 +R bbcde0d30a2bb025a3c15f4a10b2b404 +U drh +Z 9376e61a8443d421f4f6f69d3d5500ac diff --git a/manifest.uuid b/manifest.uuid index 89d1ae46cd..44109c7358 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e3dfa27c71a666e122e3cf64897038ff8424800 \ No newline at end of file +e54dded2012f0ab486ee138e9bd57c528af33980 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 26c2922ebd..4886ec1836 100644 --- a/src/build.c +++ b/src/build.c @@ -3018,7 +3018,7 @@ Index *sqlite3CreateIndex( pParse->checkSchema = 1; goto exit_create_index; } - assert( pTab->nCol<=0x7fff && j<=0x7fff ); + assert( j<=0x7fff ); pIndex->aiColumn[i] = (i16)j; if( pListItem->pExpr ){ int nColl; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b802d7aab7..092a14345f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3652,8 +3652,10 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); /* ** Threading interface */ +#if SQLITE_MAX_WORKER_THREADS>0 int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); +#endif /* ** Win32 interface From d30ab3d9dd17d37f152dfad1d7b9ffd7fd96bddc Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 9 Apr 2014 20:04:17 +0000 Subject: [PATCH 36/99] Experimental multi-threaded sorting changes to allow the sorter to begin returning items to the VDBE before all data is sorted. FossilOrigin-Name: f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f --- manifest | 19 ++- manifest.uuid | 2 +- src/vdbesort.c | 443 ++++++++++++++++++++++++++++++++++++------------ test/sort2.test | 6 + 4 files changed, 348 insertions(+), 122 deletions(-) diff --git a/manifest b/manifest index 2eb2b1da22..8831053880 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-04-04T22:44:59.018 +C Experimental\smulti-threaded\ssorting\schanges\sto\sallow\sthe\ssorter\sto\sbegin\sreturning\sitems\sto\sthe\sVDBE\sbefore\sall\sdata\sis\ssorted. +D 2014-04-09T20:04:17.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 8da916fc74e78edd5bc95653206942e01710ac09 +F src/vdbesort.c 26823b626c3231a52e45f5e78a18cb8681bb1b88 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 +F test/sort2.test bbc2eb244fb862141a900a851056d48705b5997b F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1163,7 +1163,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5e3dfa27c71a666e122e3cf64897038ff8424800 -R bbcde0d30a2bb025a3c15f4a10b2b404 -U drh -Z 9376e61a8443d421f4f6f69d3d5500ac +P e54dded2012f0ab486ee138e9bd57c528af33980 +R 803b4ddf4cddf7e21aeddc04109caaf0 +T *branch * threads-experimental +T *sym-threads-experimental * +T -sym-threads * +U dan +Z 3b5c615396ccbaaa23add5a8103bd906 diff --git a/manifest.uuid b/manifest.uuid index 44109c7358..6cf45357c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e54dded2012f0ab486ee138e9bd57c528af33980 \ No newline at end of file +f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index f59e8f51f5..e558c42f11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -96,6 +96,8 @@ typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ +typedef struct SorterFile SorterFile; +typedef struct IncrMerger IncrMerger; /* @@ -105,6 +107,11 @@ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ #define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ #define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ +struct SorterFile { + sqlite3_file *pFd; + i64 iEof; +}; + /* ** Sorting is divided up into smaller subtasks. Each subtask is controlled ** by an instance of this object. A Subtask might run in either the main thread @@ -145,6 +152,7 @@ struct SortSubtask { int bDone; /* Set to true by pTask when finished */ sqlite3 *db; /* Database connection */ + VdbeSorter *pSorter; /* Sorter */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ @@ -155,9 +163,8 @@ struct SortSubtask { int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ - int nPMA; /* Number of PMAs currently in pTemp1 */ - i64 iTemp1Off; /* Offset to write to in pTemp1 */ - sqlite3_file *pTemp1; /* File to write PMAs to, or NULL */ + int nPMA; /* Number of PMAs currently in file */ + SorterFile file; }; @@ -239,8 +246,10 @@ struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ + int bUseThreads; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - MergeEngine *pMerger; /* For final merge of PMAs (by caller) */ + PmaReader *pReader; /* Read data from here after Rewind() */ + UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ @@ -265,6 +274,16 @@ struct PmaReader { u8 *aBuffer; /* Current read buffer */ int nBuffer; /* Size of read buffer in bytes */ u8 *aMap; /* Pointer to mapping of entire file */ + IncrMerger *pIncr; /* Incremental merger */ +}; + +struct IncrMerger { + int mxSz; /* Maximum size of files */ + SortSubtask *pTask; /* Task that owns this merger */ + int bEof; /* Set to true when merge is finished */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + SQLiteThread *pThread; /* Thread currently populating aFile[1] */ }; /* @@ -326,6 +345,9 @@ struct SorterRecord { /* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 +static int vdbeIncrSwap(IncrMerger*); +static void vdbeIncrFree(IncrMerger*); + /* ** Free all memory belonging to the PmaReader object passed as the second ** argument. All structure fields are set to zero before returning. @@ -334,6 +356,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); + if( pIter->pIncr ) vdbeIncrFree(pIter->pIncr); memset(pIter, 0, sizeof(PmaReader)); } @@ -400,7 +423,7 @@ static int vdbePmaReadBlob( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; + int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM; @@ -464,22 +487,70 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ return SQLITE_OK; } +static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ + int rc = SQLITE_OK; + if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); + } + return rc; +} + +static int vdbePmaReaderReinit(PmaReader *pIter){ + IncrMerger *pIncr = pIter->pIncr; + SortSubtask *pTask = pIncr->pTask; + int rc = SQLITE_OK; + + assert( pIncr->bEof==0 ); + + if( pIter->aMap ){ + sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); + pIter->aMap = 0; + } + pIter->iReadOff = 0; + pIter->iEof = pIncr->aFile[0].iEof; + pIter->pFile = pIncr->aFile[0].pFd; + + rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); + if( rc==SQLITE_OK ){ + if( pIter->aMap==0 && pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pTask->pgsz; + } + } + + return rc; +} + /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ static int vdbePmaReaderNext(PmaReader *pIter){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ - /* This is an EOF condition */ - vdbePmaReaderClear(pIter); - return SQLITE_OK; + int bEof = 1; + if( pIter->pIncr ){ + rc = vdbeIncrSwap(pIter->pIncr); + if( rc==SQLITE_OK && pIter->pIncr->bEof==0 ){ + rc = vdbePmaReaderReinit(pIter); + bEof = 0; + } + } + + if( bEof ){ + /* This is an EOF condition */ + vdbePmaReaderClear(pIter); + return rc; + } } - rc = vdbePmaReadVarint(pIter, &nRec); + if( rc==SQLITE_OK ){ + rc = vdbePmaReadVarint(pIter, &nRec); + } if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); @@ -493,10 +564,14 @@ static int vdbePmaReaderNext(PmaReader *pIter){ ** starting at offset iStart and ending at offset iEof-1. This function ** leaves the iterator pointing to the first key in the PMA (or EOF if the ** PMA is empty). +** +** If the pnByte parameter is NULL, then it is assumed that the file +** contains a single PMA, and that that PMA omits the initial length varint. */ static int vdbePmaReaderInit( - SortSubtask *pTask, /* Thread context */ - i64 iStart, /* Start offset in pTask->pTemp1 */ + SortSubtask *pTask, /* Task context */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iStart, /* Start offset in pFile */ PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ @@ -504,18 +579,18 @@ static int vdbePmaReaderInit( int nBuf = pTask->pgsz; void *pMap = 0; /* Mapping of temp file */ - assert( pTask->iTemp1Off>iStart ); + assert( pFile->iEof>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pTask->pTemp1; + pIter->pFile = pFile->pFd; pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pIter->pFile, 0, pFile->iEof, &pMap); } }else{ rc = SQLITE_NOMEM; @@ -533,12 +608,12 @@ static int vdbePmaReaderInit( int iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pTask->iTemp1Off ){ - nRead = (int)(pTask->iTemp1Off - iStart); + if( (iStart + nRead) > pFile->iEof ){ + nRead = (int)(pFile->iEof - iStart); } rc = sqlite3OsRead( - pTask->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); + pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart + ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } } @@ -547,7 +622,7 @@ static int vdbePmaReaderInit( if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pTask->iTemp1Off; + pIter->iEof = pFile->iEof; rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; @@ -669,11 +744,13 @@ int sqlite3VdbeSorterInit( pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; + pSorter->bUseThreads = (pSorter->nTask>1); for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pKeyInfo = pKeyInfo; pTask->pgsz = pgsz; pTask->db = db; + pTask->pSorter = pSorter; } if( !sqlite3TempInMemory(db) ){ @@ -723,9 +800,10 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ pTask->aListMemory = 0; } pTask->pList = 0; - if( pTask->pTemp1 ){ - sqlite3OsCloseFree(pTask->pTemp1); - pTask->pTemp1 = 0; + if( pTask->file.pFd ){ + sqlite3OsCloseFree(pTask->file.pFd); + pTask->file.pFd = 0; + pTask->file.iEof = 0; } } @@ -761,7 +839,8 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - assert( nIter<=SORTER_MAX_MERGE_COUNT ); + /* assert( nIter<=SORTER_MAX_MERGE_COUNT ); */ + while( NpMerger); - pSorter->pMerger = 0; + if( pSorter->pReader ){ + vdbePmaReaderClear(pSorter->pReader); + sqlite3DbFree(db, pSorter->pReader); + pSorter->pReader = 0; + } for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); @@ -806,6 +888,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ pSorter->nInMemory = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; + sqlite3DbFree(db, pSorter->pUnpacked); + pSorter->pUnpacked = 0; } /* @@ -815,7 +899,6 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - vdbeMergeEngineFree(pSorter->pMerger); sqlite3_free(pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; @@ -1053,17 +1136,17 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ assert( pTask->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( pTask->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1); - assert( rc!=SQLITE_OK || pTask->pTemp1 ); - assert( pTask->iTemp1Off==0 ); + if( pTask->file.pFd==0 ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file.pFd); + assert( rc!=SQLITE_OK || pTask->file.pFd ); + assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); } /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ vdbeSorterExtendFile(pTask->db, - pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 + pTask->file.pFd, pTask->file.iEof + pTask->nInMemory + 9 ); } @@ -1071,8 +1154,8 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ SorterRecord *p; SorterRecord *pNext = 0; - vdbePmaWriterInit(pTask->pTemp1, &writer, pTask->pgsz, - pTask->iTemp1Off); + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, + pTask->file.iEof); pTask->nPMA++; vdbePmaWriteVarint(&writer, pTask->nInMemory); for(p=pTask->pList; p; p=pNext){ @@ -1082,7 +1165,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ if( pTask->aListMemory==0 ) sqlite3_free(p); } pTask->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->iTemp1Off); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); } assert( pTask->pList==0 || rc!=SQLITE_OK ); @@ -1164,6 +1247,23 @@ static int vdbeSorterNext( return rc; } +#if 0 +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(db->pVfs, &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(x,y) +#endif + /* ** The main routine for sorter-thread operations. */ @@ -1177,6 +1277,8 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ); assert( pTask->bDone==0 ); + vdbeSorterWorkDebug(pTask, "enter"); + if( pTask->pUnpacked==0 ){ char *pFree; pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( @@ -1211,7 +1313,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ /* Open a second temp file to write merged data to */ rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); + vdbeSorterExtendFile(pTask->db, pTemp2, pTask->file.iEof); }else{ vdbeMergeEngineFree(pMerger); break; @@ -1231,9 +1333,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ int iIter; for(iIter=0; iIteraIter[iIter]; - rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nOut); + rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nOut); iReadOff = pIter->iEof; - if( iReadOff>=pTask->iTemp1Off || rc!=SQLITE_OK ) break; + if( iReadOff>=pTask->file.iEof || rc!=SQLITE_OK ) break; } for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ rc = vdbeSorterDoCompare(pTask, pMerger, iIter); @@ -1253,10 +1355,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ } vdbeMergeEngineFree(pMerger); - sqlite3OsCloseFree(pTask->pTemp1); - pTask->pTemp1 = pTemp2; + sqlite3OsCloseFree(pTask->file.pFd); + pTask->file.pFd = pTemp2; pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pTask->iTemp1Off = iWriteOff; + pTask->file.iEof = iWriteOff; } }else{ /* Sort the pTask->pList list */ @@ -1267,10 +1369,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ #ifdef SQLITE_DEBUG i64 nExpect = pTask->nInMemory + sqlite3VarintLen(pTask->nInMemory) - + pTask->iTemp1Off; + + pTask->file.iEof; #endif rc = vdbeSorterListToPMA(pTask); - assert( rc!=SQLITE_OK || (nExpect==pTask->iTemp1Off) ); + assert( rc!=SQLITE_OK || (nExpect==pTask->file.iEof) ); } } @@ -1280,6 +1382,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); rc = SQLITE_NOMEM; } + vdbeSorterWorkDebug(pTask, "exit"); return SQLITE_INT_TO_PTR(rc); } @@ -1480,6 +1583,164 @@ static int vdbeSorterCountPMA(VdbeSorter *pSorter){ return nPMA; } +/* +** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format +** of the data stored in aFile[1] is the same as that used by regular PMAs, +** except that the number-of-bytes varint is omitted from the start. +*/ +static int vdbeIncrPopulate(IncrMerger *pIncr){ + int rc = SQLITE_OK; + int rc2; + SorterFile *pOut = &pIncr->aFile[1]; + MergeEngine *pMerger = pIncr->pMerger; + PmaWriter writer; + assert( pIncr->bEof==0 ); + + vdbePmaWriterInit(pIncr->aFile[1].pFd, &writer, pIncr->pTask->pgsz, 0); + while( rc==SQLITE_OK ){ + int dummy; + PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; + int nKey = pReader->nKey; + i64 iEof = writer.iWriteOff + writer.iBufEnd; + + /* Check if the output file is full or if the input has been exhausted. + ** In either case exit the loop. */ + if( pReader->pFile==0 ) break; + if( iEof && (iEof + nKey)>pIncr->mxSz ) break; + + /* Write the next key to the output. */ + vdbePmaWriteVarint(&writer, nKey); + vdbePmaWriteBlob(&writer, pReader->aKey, nKey); + rc = vdbeSorterNext(pIncr->pTask, pIncr->pMerger, &dummy); + } + + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +static void *vdbeIncrPopulateThreadMain(void *pCtx){ + IncrMerger *pIncr = (IncrMerger*)pCtx; + return SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); +} + +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + int rc; + assert( pIncr->pThread==0 ); + if( pIncr->pTask->pSorter->bUseThreads==0 ){ + rc = vdbeIncrPopulate(pIncr); + }else{ + void *pCtx = (void*)pIncr; + rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); + } + return rc; +} + +static int vdbeIncrSwap(IncrMerger *pIncr){ + int rc = SQLITE_OK; + + if( pIncr->pThread ){ + void *pRet; + rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + pIncr->pThread = 0; + } + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + + if( pIncr->aFile[0].iEof==0 ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + + return rc; +} + +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr->pThread ){ + void *pRet; + sqlite3ThreadJoin(pIncr->pThread, &pRet); + } + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); +} + +/* +** Populate iterator *pIter so that it may be used to iterate through all +** keys stored in subtask pTask using the incremental merge method. +*/ +static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ + SortSubtask *pTask0 = &pSorter->aTask[0]; + int rc = SQLITE_OK; + MergeEngine *pMerger = 0; + IncrMerger *pIncr = 0; + int i; + int nPMA = 0; + + for(i=0; inTask; i++){ + nPMA += pSorter->aTask[i].nPMA; + } + pMerger = vdbeMergeEngineNew(nPMA); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + }else{ + int iIter = 0; + int iPMA; + for(i=0; inTask; i++){ + i64 iReadOff = 0; + SortSubtask *pTask = &pSorter->aTask[i]; + for(iPMA=0; iPMAnPMA; iPMA++){ + i64 nDummy = 0; + PmaReader *pIter = &pMerger->aIter[iIter++]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nDummy); + iReadOff = pIter->iEof; + } + } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pTask0, pMerger, i); + } + } + + if( rc==SQLITE_OK ){ + pIncr = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); + if( pIncr==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pIncr, 0, sizeof(IncrMerger)); + pIncr->mxSz = (pSorter->mxPmaSize / 2); + pIncr->pMerger = pMerger; + pIncr->pTask = pTask0; + } + } + + /* Open the two temp files. */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[0].pFd); + } + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[1].pFd); + } + + /* Launch a background thread to populate aFile[1]. */ + if( rc==SQLITE_OK ){ + rc = vdbeIncrBgPopulate(pIncr); + } + + pIter->pIncr = pIncr; + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pIter); + } + return rc; +} + + /* ** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, ** this function is called to prepare for iterating through the records @@ -1520,70 +1781,21 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); - /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge - ** some of them together so that this is no longer the case. */ - if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ - int i; - for(i=0; rc==SQLITE_OK && inTask; i++){ - SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pTemp1 ){ - pTask->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nTask; - pTask->eWork = SORT_SUBTASK_CONS; + vdbeSorterRewindDebug(db, "rewind"); -#if SQLITE_MAX_WORKER_THREADS>0 - if( i<(pSorter->nTask-1) ){ - void *pCtx = (void*)pTask; - rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); - }else -#endif - { - rc = vdbeSorterRunTask(pTask); - } - } - } - } - - /* Join all threads */ - rc = vdbeSorterJoinAll(pSorter, rc); - - /* Assuming no errors have occurred, set up a merger structure to read - ** and merge all remaining PMAs. */ - assert( pSorter->pMerger==0 ); + /* Assuming no errors have occurred, set up a merger structure to + ** incrementally read and merge all remaining PMAs. */ + assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - int nIter = 0; /* Number of iterators used */ - int i; - MergeEngine *pMerger; - for(i=0; inTask; i++){ - nIter += pSorter->aTask[i].nPMA; - } - - pSorter->pMerger = pMerger = vdbeMergeEngineNew(nIter); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - }else{ - int iIter = 0; - int iThread = 0; - for(iThread=0; iThreadnTask; iThread++){ - int iPMA; - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[iThread]; - for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ - i64 nDummy = 0; - PmaReader *pIter = &pMerger->aIter[iIter++]; - rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nDummy); - iReadOff = pIter->iEof; - } - } - - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(&pSorter->aTask[0], pMerger, i); - } - } + PmaReader *pReader; + pReader = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReader; + rc = vdbePmaReaderIncrInit(pSorter, pReader); + assert( rc!=SQLITE_OK || pReader->pFile ); + *pbEof = 0; } - if( rc==SQLITE_OK ){ - *pbEof = (pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]].pFile==0); - } + vdbeSorterRewindDebug(db, "rewinddone"); return rc; } @@ -1594,8 +1806,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->pMerger ){ - rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + if( pSorter->pReader ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFile==0); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->u.pNext; @@ -1616,11 +1829,9 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->pMerger ){ - PmaReader *pIter; - pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; - *pnKey = pIter->nKey; - pKey = pIter->aKey; + if( pSorter->pReader ){ + *pnKey = pSorter->pReader->nKey; + pKey = pSorter->pReader->aKey; }else{ *pnKey = pSorter->pRecord->nVal; pKey = SRVAL(pSorter->pRecord); @@ -1669,13 +1880,19 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->aTask[0].pUnpacked; + UnpackedRecord *r2 = pSorter->pUnpacked; KeyInfo *pKeyInfo = pCsr->pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + if( r2==0 ){ + char *p; + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); + assert( pSorter->pUnpacked==(UnpackedRecord*)p ); + if( r2==0 ) return SQLITE_NOMEM; + r2->nField = pKeyInfo->nField-nIgnore; + } assert( r2->nField>=pKeyInfo->nField-nIgnore ); - r2->nField = pKeyInfo->nField-nIgnore; pKey = vdbeSorterRowkey(pSorter, &nKey); sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); diff --git a/test/sort2.test b/test/sort2.test index f8bfb0fe51..626630050c 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -47,6 +47,12 @@ do_execsql_test 2.2 { CREATE UNIQUE INDEX i1 ON t1(b, a); } +do_execsql_test 2.3 { + CREATE UNIQUE INDEX i2 ON t1(a); +} + +do_execsql_test 2.4 { PRAGMA integrity_check } {ok} + db close sqlite3_shutdown sqlite3_config_worker_threads 0 From 4be4c406faf605d78a9c737795c3abc6bb62621e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 11 Apr 2014 19:43:07 +0000 Subject: [PATCH 37/99] Avoid having the sorter merge too many PMAs at a time when incrementally merging data following a SorterRewind(). FossilOrigin-Name: 98bf0307b121b0776a7170108cc8d3f948a7ebfe --- manifest | 19 +- manifest.uuid | 2 +- src/shell.c | 2 +- src/vdbesort.c | 498 +++++++++++++++++++++++++++++++++++++----------- test/sort2.test | 102 ++++++---- 5 files changed, 455 insertions(+), 168 deletions(-) diff --git a/manifest b/manifest index 8831053880..f48df9c72d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\smulti-threaded\ssorting\schanges\sto\sallow\sthe\ssorter\sto\sbegin\sreturning\sitems\sto\sthe\sVDBE\sbefore\sall\sdata\sis\ssorted. -D 2014-04-09T20:04:17.324 +C Avoid\shaving\sthe\ssorter\smerge\stoo\smany\sPMAs\sat\sa\stime\swhen\sincrementally\smerging\sdata\sfollowing\sa\sSorterRewind(). +D 2014-04-11T19:43:07.755 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa -F src/shell.c afc0b1a5a646d287142ef0c9a2a6e3139d57cba2 +F src/shell.c b44c3f17f0bf41b3431e9cc171706251156ae85f F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 26823b626c3231a52e45f5e78a18cb8681bb1b88 +F src/vdbesort.c 2984e3624383adf9c762558b8f85a17a626c11a7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test bbc2eb244fb862141a900a851056d48705b5997b +F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1163,10 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e54dded2012f0ab486ee138e9bd57c528af33980 -R 803b4ddf4cddf7e21aeddc04109caaf0 -T *branch * threads-experimental -T *sym-threads-experimental * -T -sym-threads * +P f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f +R f6c598c1c558c5930404cda096730209 U dan -Z 3b5c615396ccbaaa23add5a8103bd906 +Z 3e9d4ee1a6e7b343cf831d1b18651067 diff --git a/manifest.uuid b/manifest.uuid index 6cf45357c4..b1943056c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f \ No newline at end of file +98bf0307b121b0776a7170108cc8d3f948a7ebfe \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e032bd36d2..40ac24093a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3535,7 +3535,7 @@ static void main_init(struct callback_data *data) { sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 4); } /* diff --git a/src/vdbesort.c b/src/vdbesort.c index e558c42f11..16f6c618c6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -164,7 +164,8 @@ struct SortSubtask { u8 *aListMemory; /* Records memory (or NULL) */ int nPMA; /* Number of PMAs currently in file */ - SorterFile file; + SorterFile file; /* Temp file for level-0 PMAs */ + SorterFile file2; /* Space for other PMAs */ }; @@ -240,6 +241,11 @@ struct MergeEngine { /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. +** +** mxKeysize: +** As records are added to the sorter by calls to sqlite3VdbeSorterWrite(), +** this variable is updated so as to be set to the size on disk of the +** largest record in the sorter. */ struct VdbeSorter { int nInMemory; /* Current size of pRecord list as PMA */ @@ -249,6 +255,7 @@ struct VdbeSorter { int bUseThreads; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ PmaReader *pReader; /* Read data from here after Rewind() */ + int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ @@ -277,13 +284,21 @@ struct PmaReader { IncrMerger *pIncr; /* Incremental merger */ }; +/* +** Normally, a PmaReader object iterates through an existing PMA stored +** within a temp file. However, if the PmaReader.pIncr variable points to +** an object of the following type, it may be used to iterate/merge through +** multiple PMAs simultaneously. +*/ struct IncrMerger { - int mxSz; /* Maximum size of files */ SortSubtask *pTask; /* Task that owns this merger */ - int bEof; /* Set to true when merge is finished */ - SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ - MergeEngine *pMerger; /* Merge engine thread reads data from */ SQLiteThread *pThread; /* Thread currently populating aFile[1] */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + i64 iStartOff; /* Offset to start writing file at */ + int mxSz; /* Maximum bytes of data to store */ + int bEof; /* Set to true when merge is finished */ + int bUseThread; /* True to use a bg thread for this object */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ }; /* @@ -506,16 +521,30 @@ static int vdbePmaReaderReinit(PmaReader *pIter){ sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); pIter->aMap = 0; } - pIter->iReadOff = 0; + pIter->iReadOff = pIncr->iStartOff; pIter->iEof = pIncr->aFile[0].iEof; pIter->pFile = pIncr->aFile[0].pFd; rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); if( rc==SQLITE_OK ){ - if( pIter->aMap==0 && pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pTask->pgsz; + if( pIter->aMap==0 ){ + /* TODO: Combine this code with similar code in vdbePmaReaderInit() */ + int iBuf = pIter->iReadOff % pTask->pgsz; + if( pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pTask->pgsz; + } + if( iBuf ){ + int nRead = pTask->pgsz - iBuf; + if( (pIter->iReadOff + nRead) > pIter->iEof ){ + nRead = (int)(pIter->iEof - pIter->iReadOff); + } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + } } } @@ -577,7 +606,6 @@ static int vdbePmaReaderInit( ){ int rc = SQLITE_OK; int nBuf = pTask->pgsz; - void *pMap = 0; /* Mapping of temp file */ assert( pFile->iEof>iStart ); assert( pIter->aAlloc==0 ); @@ -589,33 +617,27 @@ static int vdbePmaReaderInit( if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pIter->pFile, 0, pFile->iEof, &pMap); - } + rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); }else{ rc = SQLITE_NOMEM; } - if( rc==SQLITE_OK ){ - if( pMap ){ - pIter->aMap = (u8*)pMap; + if( rc==SQLITE_OK && pIter->aMap==0 ){ + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; }else{ - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pFile->iEof ){ - nRead = (int)(pFile->iEof - iStart); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); + int iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pFile->iEof ){ + nRead = (int)(pFile->iEof - iStart); } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } } } @@ -805,6 +827,11 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ pTask->file.pFd = 0; pTask->file.iEof = 0; } + if( pTask->file2.pFd ){ + sqlite3OsCloseFree(pTask->file2.pFd); + pTask->file2.pFd = 0; + pTask->file2.iEof = 0; + } } /* @@ -839,7 +866,7 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - /* assert( nIter<=SORTER_MAX_MERGE_COUNT ); */ + assert( nIter<=SORTER_MAX_MERGE_COUNT ); while( NnInMemory = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; + pSorter->mxKeysize = 0; sqlite3DbFree(db, pSorter->pUnpacked); pSorter->pUnpacked = 0; } @@ -1259,11 +1287,35 @@ static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ sqlite3OsCurrentTimeInt64(db->pVfs, &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} #else # define vdbeSorterWorkDebug(x,y) # define vdbeSorterRewindDebug(x,y) +# define vdbeSorterPopulateDebug(x,y) #endif +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + /* ** The main routine for sorter-thread operations. */ @@ -1279,19 +1331,8 @@ static void *vdbeSortSubtaskMain(void *pCtx){ vdbeSorterWorkDebug(pTask, "enter"); - if( pTask->pUnpacked==0 ){ - char *pFree; - pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree - ); - assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ){ - rc = SQLITE_NOMEM; - goto thread_out; - } - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; - pTask->pUnpacked->errCode = 0; - } + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) goto thread_out; if( pTask->eWork==SORT_SUBTASK_CONS ){ assert( pTask->pList==0 ); @@ -1533,6 +1574,9 @@ int sqlite3VdbeSorterWrite( } pSorter->nInMemory += nPMA; + if( nPMA>pSorter->mxKeysize ){ + pSorter->mxKeysize = nPMA; + } if( pSorter->aMemory ){ int nMin = pSorter->iMemory + nReq; @@ -1591,12 +1635,15 @@ static int vdbeSorterCountPMA(VdbeSorter *pSorter){ static int vdbeIncrPopulate(IncrMerger *pIncr){ int rc = SQLITE_OK; int rc2; + i64 iStart = pIncr->iStartOff; SorterFile *pOut = &pIncr->aFile[1]; MergeEngine *pMerger = pIncr->pMerger; PmaWriter writer; assert( pIncr->bEof==0 ); - vdbePmaWriterInit(pIncr->aFile[1].pFd, &writer, pIncr->pTask->pgsz, 0); + vdbeSorterPopulateDebug(pIncr->pTask, "enter"); + + vdbePmaWriterInit(pOut->pFd, &writer, pIncr->pTask->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; @@ -1606,7 +1653,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Check if the output file is full or if the input has been exhausted. ** In either case exit the loop. */ if( pReader->pFile==0 ) break; - if( iEof && (iEof + nKey)>pIncr->mxSz ) break; + if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); @@ -1616,6 +1663,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); if( rc==SQLITE_OK ) rc = rc2; + vdbeSorterPopulateDebug(pIncr->pTask, "exit"); return rc; } @@ -1627,34 +1675,50 @@ static void *vdbeIncrPopulateThreadMain(void *pCtx){ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ int rc; assert( pIncr->pThread==0 ); - if( pIncr->pTask->pSorter->bUseThreads==0 ){ + if( pIncr->bUseThread==0 ){ rc = vdbeIncrPopulate(pIncr); - }else{ + } +#if SQLITE_MAX_WORKER_THREADS>0 + else{ void *pCtx = (void*)pIncr; rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); } +#endif return rc; } static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; - - if( pIncr->pThread ){ - void *pRet; - rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - pIncr->pThread = 0; - } - if( rc==SQLITE_OK ){ - SorterFile f0 = pIncr->aFile[0]; + if( pIncr->bUseThread ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->pThread ){ + void *pRet; + assert( pIncr->bUseThread ); + rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + pIncr->pThread = 0; + } +#endif + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + } + + if( rc==SQLITE_OK ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + }else{ + rc = vdbeIncrPopulate(pIncr); pIncr->aFile[0] = pIncr->aFile[1]; - pIncr->aFile[1] = f0; - - if( pIncr->aFile[0].iEof==0 ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ pIncr->bEof = 1; - }else{ - rc = vdbeIncrBgPopulate(pIncr); } } @@ -1662,81 +1726,283 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ } static void vdbeIncrFree(IncrMerger *pIncr){ - if( pIncr->pThread ){ - void *pRet; - sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->pThread ){ + void *pRet; + sqlite3ThreadJoin(pIncr->pThread, &pRet); + } + if( pIncr->bUseThread ){ + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); } - if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); - if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); - vdbeMergeEngineFree(pIncr->pMerger); - sqlite3_free(pIncr); +} + +static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ + IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); + if( pIncr ){ + memset(pIncr, 0, sizeof(IncrMerger)); + pIncr->pMerger = pMerger; + pIncr->pTask = pTask; + pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); + pTask->file2.iEof += pIncr->mxSz; + +#if 0 + /* Open the two temp files. */ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + } + if( rc!=SQLITE_OK ){ + vdbeIncrFree(pIncr); + pIncr = 0; + } +#endif + } + return pIncr; +} + +static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ + if( bUseThread ){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; + } +} + +static int vdbeIncrInit2(PmaReader *pIter){ + int rc = SQLITE_OK; + IncrMerger *pIncr = pIter->pIncr; + if( pIncr ){ + SortSubtask *pTask = pIncr->pTask; + int i; + MergeEngine *pMerger = pIncr->pMerger; + + for(i=0; rc==SQLITE_OK && inTree; i++){ + rc = vdbeIncrInit2(&pMerger->aIter[i]); + } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); + } + + /* Set up the required files for pIncr */ + if( rc==SQLITE_OK ){ + if( pIncr->bUseThread==0 ){ + if( pTask->file2.pFd==0 ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file2.pFd); + assert( pTask->file2.iEof>0 ); + if( rc==SQLITE_OK ){ + vdbeSorterExtendFile(pTask->db,pTask->file2.pFd,pTask->file2.iEof); + pTask->file2.iEof = 0; + } + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += pIncr->mxSz; + } + }else{ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + } + } + } + + if( rc==SQLITE_OK && pIncr->bUseThread ){ + rc = vdbeIncrBgPopulate(pIncr); + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pIter); + } + } + return rc; +} + +/* +** Allocate a new MergeEngine object to merge the contents of nPMA level-0 +** PMAs from pTask->file. If no error occurs, set *ppOut to point to +** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut +** to NULL and return an SQLite error code. +** +** When this function is called, *piOffset is set to the offset of the +** first PMA to read from pTask->file. Assuming no error occurs, it is +** set to the offset immediately following the last byte of the last +** PMA before returning. If an error does occur, then the final value of +** *piOffset is undefined. +*/ +static int vdbeMergeEngineLevel0( + SortSubtask *pTask, /* Sorter task to read from */ + int nPMA, /* Number of PMAs to read */ + i64 *piOffset, /* IN/OUT: Read offset in pTask->file */ + MergeEngine **ppOut /* OUT: New merge-engine */ +){ + MergeEngine *pNew; /* Merge engine to return */ + i64 iOff = *piOffset; + int i; + int rc = SQLITE_OK; + + *ppOut = pNew = vdbeMergeEngineNew(nPMA); + if( pNew==0 ) rc = SQLITE_NOMEM; + + for(i=0; iaIter[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pIter, &nDummy); + iOff = pIter->iEof; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pNew); + *ppOut = 0; + } + *piOffset = iOff; + return rc; +} + +typedef struct IncrBuilder IncrBuilder; +struct IncrBuilder { + int nPMA; /* Number of iterators used so far */ + MergeEngine *pMerger; /* Merge engine to populate. */ +}; + +static int vdbeAddToBuilder( + SortSubtask *pTask, + IncrBuilder *pBuilder, + MergeEngine *pMerger +){ + int rc = SQLITE_OK; + IncrMerger *pIncr; + + assert( pMerger ); + if( pBuilder->nPMA==SORTER_MAX_MERGE_COUNT ){ + rc = vdbeAddToBuilder(pTask, &pBuilder[1], pBuilder->pMerger); + pBuilder->pMerger = 0; + pBuilder->nPMA = 0; + } + + if( rc==SQLITE_OK && pBuilder->pMerger==0 ){ + pBuilder->pMerger = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pBuilder->pMerger==0 ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pIncr = vdbeIncrNew(pTask, pMerger); + if( pIncr==0 ) rc = SQLITE_NOMEM; + pBuilder->pMerger->aIter[pBuilder->nPMA++].pIncr = pIncr; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMerger); + } + + return rc; } /* ** Populate iterator *pIter so that it may be used to iterate through all -** keys stored in subtask pTask using the incremental merge method. +** keys stored in all PMAs created by this sorter. */ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ SortSubtask *pTask0 = &pSorter->aTask[0]; + MergeEngine *pMain = 0; + sqlite3 *db = pTask0->db; int rc = SQLITE_OK; - MergeEngine *pMerger = 0; - IncrMerger *pIncr = 0; - int i; - int nPMA = 0; + int iTask; - for(i=0; inTask; i++){ - nPMA += pSorter->aTask[i].nPMA; + IncrBuilder *aMerge; + const int nMerge = 32; + aMerge = sqlite3DbMallocZero(db, sizeof(aMerge[0])*nMerge); + if( aMerge==0 ) return SQLITE_NOMEM; + + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM; } - pMerger = vdbeMergeEngineNew(nPMA); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - }else{ - int iIter = 0; + + for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + MergeEngine *pRoot = 0; int iPMA; - for(i=0; inTask; i++){ - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[i]; - for(iPMA=0; iPMAnPMA; iPMA++){ - i64 nDummy = 0; - PmaReader *pIter = &pMerger->aIter[iIter++]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nDummy); - iReadOff = pIter->iEof; + i64 iReadOff = 0; + SortSubtask *pTask = &pSorter->aTask[iTask]; + if( pTask->nPMA==0 ) continue; + for(iPMA=0; iPMAnPMA; iPMA += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; + int nReader = MIN(pTask->nPMA - iPMA, SORTER_MAX_MERGE_COUNT); + + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc!=SQLITE_OK ) break; + + if( iPMA==0 ){ + pRoot = pMerger; + }else{ + if( pRoot ){ + rc = vdbeAddToBuilder(pTask, &aMerge[0], pRoot); + pRoot = 0; + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMerger); + break; + } + } + rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); } } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pTask0, pMerger, i); + + if( pRoot==0 ){ + int i; + for(i=0; rc==SQLITE_OK && iaIter[iTask].pIncr = pNew; + if( pNew==0 ) rc = SQLITE_NOMEM; + } + memset(aMerge, 0, nMerge*sizeof(aMerge[0])); } } if( rc==SQLITE_OK ){ - pIncr = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); - if( pIncr==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pIncr, 0, sizeof(IncrMerger)); - pIncr->mxSz = (pSorter->mxPmaSize / 2); - pIncr->pMerger = pMerger; - pIncr->pTask = pTask0; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pIter->pIncr = vdbeIncrNew(pLast, pMain); + if( pIter->pIncr==0 ){ + rc = SQLITE_NOMEM; + }else{ + vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aIter[iTask].pIncr) ){ + vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + assert( pIncr->pTask!=pLast ); + } + } + } } } - - /* Open the two temp files. */ if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[0].pFd); - } - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeIncrInit2(pIter); } - /* Launch a background thread to populate aFile[1]. */ - if( rc==SQLITE_OK ){ - rc = vdbeIncrBgPopulate(pIncr); - } - - pIter->pIncr = pIncr; - if( rc==SQLITE_OK ){ - rc = vdbePmaReaderNext(pIter); - } + sqlite3_free(aMerge); return rc; } diff --git a/test/sort2.test b/test/sort2.test index 626630050c..4fb6a9462b 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -15,47 +15,71 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sort2 -db close -sqlite3_shutdown -sqlite3_config_worker_threads 7 -reset_db - -do_execsql_test 1 { - PRAGMA cache_size = 5; - WITH r(x,y) AS ( - SELECT 1, randomblob(100) - UNION ALL - SELECT x+1, randomblob(100) FROM r - LIMIT 100000 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) +foreach {tn script} { + 1 { } + 2 { + catch { db close } + sqlite3_shutdown + sqlite3_config_worker_threads 7 + reset_db + } } { - 20000 100 20000 100 20000 100 20000 100 20000 100 + + eval $script + + do_execsql_test $tn.1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 20000 100 20000 100 20000 100 20000 100 20000 100 + } + + do_execsql_test $tn.2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; + } + + do_execsql_test $tn.2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); + } + + do_execsql_test $tn.2.3 { + CREATE UNIQUE INDEX i2 ON t1(a); + } + + do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} + + breakpoint + do_execsql_test $tn.3 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 200000 100 200000 100 200000 100 200000 100 200000 100 + } + + db close + sqlite3_shutdown + sqlite3_config_worker_threads 0 + sqlite3_initialize + } -do_execsql_test 2.1 { - CREATE TABLE t1(a, b); - WITH r(x,y) AS ( - SELECT 1, randomblob(100) - UNION ALL - SELECT x+1, randomblob(100) FROM r - LIMIT 10000 - ) INSERT INTO t1 SELECT * FROM r; -} - -do_execsql_test 2.2 { - CREATE UNIQUE INDEX i1 ON t1(b, a); -} - -do_execsql_test 2.3 { - CREATE UNIQUE INDEX i2 ON t1(a); -} - -do_execsql_test 2.4 { PRAGMA integrity_check } {ok} - -db close -sqlite3_shutdown -sqlite3_config_worker_threads 0 -sqlite3_initialize finish_test From 82a8a9f1204ac4fe3ba6b658199884f1852a2b4a Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 12 Apr 2014 19:34:44 +0000 Subject: [PATCH 38/99] Fix many issues with new code. FossilOrigin-Name: 62c406a042d7246f6df6b943421182a88483b2e3 --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 660 ++++++++++++++++++++++--------------------------- 3 files changed, 301 insertions(+), 373 deletions(-) diff --git a/manifest b/manifest index f48df9c72d..bca9ab362e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\shaving\sthe\ssorter\smerge\stoo\smany\sPMAs\sat\sa\stime\swhen\sincrementally\smerging\sdata\sfollowing\sa\sSorterRewind(). -D 2014-04-11T19:43:07.755 +C Fix\smany\sissues\swith\snew\scode. +D 2014-04-12T19:34:44.467 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2984e3624383adf9c762558b8f85a17a626c11a7 +F src/vdbesort.c bc0d90e00abcc88997f463d4d41b7ba4a10cfd88 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f -R f6c598c1c558c5930404cda096730209 +P 98bf0307b121b0776a7170108cc8d3f948a7ebfe +R f3107fdb117ba86f9bee4609a5b08bfd U dan -Z 3e9d4ee1a6e7b343cf831d1b18651067 +Z 5a1b16a83fca264558f06f5fb6536949 diff --git a/manifest.uuid b/manifest.uuid index b1943056c1..e7ab0df5c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98bf0307b121b0776a7170108cc8d3f948a7ebfe \ No newline at end of file +62c406a042d7246f6df6b943421182a88483b2e3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 16f6c618c6..1889c8fde1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -88,6 +88,15 @@ #include "sqliteInt.h" #include "vdbeInt.h" +/* +** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various +** messages to stderr that may be helpful in understanding the performance +** characteristics of the sorter in multi-threaded mode. +*/ +#if 0 +# define SQLITE_DEBUG_SORTER_THREADS 1 +#endif + /* ** Private objects used by the sorter */ @@ -97,19 +106,48 @@ typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ typedef struct SorterFile SorterFile; +typedef struct SorterThread SorterThread; +typedef struct SorterList SorterList; typedef struct IncrMerger IncrMerger; +/* +** A container for a temp file handle and the current amount of data +** stored in the file. +*/ +struct SorterFile { + sqlite3_file *pFd; /* File handle */ + i64 iEof; /* Bytes of data stored in pFd */ +}; /* -** Candidate values for SortSubtask.eWork +** An object of this type is used to store the thread handle for each +** background thread launched by the sorter. Before the thread is launched, +** variable bDone is set to 0. Then, right before it exits, the thread +** itself sets bDone to 1. +** +** This is then used for two purposes: +** +** 1. When flushing the contents of memory to a level-0 PMA on disk, to +** attempt to select a SortSubtask for which there is not already an +** active background thread (since doing so causes the main thread +** to block until it finishes). +** +** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call +** to sqlite3ThreadJoin() is likely to block. +** +** In both cases, the effects of the main thread seeing (bDone==0) even +** after the thread has finished are not dire. So we don't worry about +** memory barriers and such here. */ -#define SORT_SUBTASK_SORT 1 /* Sort records on pList */ -#define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ -#define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ +struct SorterThread { + SQLiteThread *pThread; + int bDone; +}; -struct SorterFile { - sqlite3_file *pFd; - i64 iEof; +struct SorterList { + SorterRecord *pList; /* Linked list of records */ + u8 *aMemory; /* If non-NULL, blob of memory for pList */ + int szPMA; /* Size of pList as PMA in bytes */ }; /* @@ -148,21 +186,13 @@ struct SorterFile { ** remain in temp file SortSubtask.pTemp1. */ struct SortSubtask { - SQLiteThread *pThread; /* Thread handle, or NULL */ - int bDone; /* Set to true by pTask when finished */ - + SorterThread thread; sqlite3 *db; /* Database connection */ VdbeSorter *pSorter; /* Sorter */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ - - u8 eWork; /* One of the SORT_SUBTASK_* constants */ - int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ - SorterRecord *pList; /* List of records for pTask to sort */ - int nInMemory; /* Expected size of PMA based on pList */ - u8 *aListMemory; /* Records memory (or NULL) */ - + SorterList list; /* List for thread to write to a PMA */ int nPMA; /* Number of PMAs currently in file */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ @@ -248,16 +278,19 @@ struct MergeEngine { ** largest record in the sorter. */ struct VdbeSorter { - int nInMemory; /* Current size of pRecord list as PMA */ int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ int bUseThreads; /* True if one or more PMAs created */ - SorterRecord *pRecord; /* Head of in-memory record list */ PmaReader *pReader; /* Read data from here after Rewind() */ int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ +#if 0 + int nInMemory; /* Current size of pRecord list as PMA */ + SorterRecord *pRecord; /* Head of in-memory record list */ u8 *aMemory; /* Block of memory to alloc records from */ +#endif + SorterList list; int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ @@ -292,7 +325,7 @@ struct PmaReader { */ struct IncrMerger { SortSubtask *pTask; /* Task that owns this merger */ - SQLiteThread *pThread; /* Thread currently populating aFile[1] */ + SorterThread thread; /* Thread for populating aFile[1] */ MergeEngine *pMerger; /* Merge engine thread reads data from */ i64 iStartOff; /* Offset to start writing file at */ int mxSz; /* Maximum bytes of data to store */ @@ -787,8 +820,8 @@ int sqlite3VdbeSorterInit( if( sqlite3GlobalConfig.pHeap==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; - pSorter->aMemory = (u8*)sqlite3Malloc(pgsz); - if( !pSorter->aMemory ) rc = SQLITE_NOMEM; + pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; } } } @@ -815,13 +848,13 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); pTask->pUnpacked = 0; - if( pTask->aListMemory==0 ){ - vdbeSorterRecordFree(0, pTask->pList); + if( pTask->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pTask->list.pList); }else{ - sqlite3_free(pTask->aListMemory); - pTask->aListMemory = 0; + sqlite3_free(pTask->list.aMemory); + pTask->list.aMemory = 0; } - pTask->pList = 0; + pTask->list.pList = 0; if( pTask->file.pFd ){ sqlite3OsCloseFree(pTask->file.pFd); pTask->file.pFd = 0; @@ -834,28 +867,96 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ } } -/* -** Join all threads. -*/ +#ifdef SQLITE_DEBUG_SORTER_THREADS +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(db->pVfs, &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterBlockDebug( + SortSubtask *pTask, + int bBlocked, + const char *zEvent +){ + if( bBlocked ){ + i64 t; + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:main %s\n", t, zEvent); + } +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(x,y) +# define vdbeSorterPopulateDebug(x,y) +# define vdbeSorterBlockDebug(x,y,z) +#endif + #if SQLITE_MAX_WORKER_THREADS>0 +/* +** Join thread p. +*/ +static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ + int rc = SQLITE_OK; + if( p->pThread ){ +#ifdef SQLITE_DEBUG_SORTER_THREADS + int bDone = p->bDone; +#endif + void *pRet; + vdbeSorterBlockDebug(pTask, !bDone, "enter"); + rc = sqlite3ThreadJoin(p->pThread, &pRet); + vdbeSorterBlockDebug(pTask, !bDone, "exit"); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + assert( p->bDone==1 ); + p->bDone = 0; + p->pThread = 0; + } + return rc; +} + +/* +** Launch a background thread to run xTask(pIn). +*/ +static int vdbeSorterCreateThread( + SorterThread *p, /* Thread object to populate */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + assert( p->pThread==0 && p->bDone==0 ); + return sqlite3ThreadCreate(&p->pThread, xTask, pIn); +} + +/* +** Join all outstanding threads launched by SorterWrite() to create +** level-0 PMAs. +*/ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pThread ){ - void *pRet; - int rc2 = sqlite3ThreadJoin(pTask->pThread, &pRet); - pTask->pThread = 0; - pTask->bDone = 0; - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - } + int rc2 = vdbeSorterJoinThread(pTask, &pTask->thread); + if( rc==SQLITE_OK ) rc = rc2; } return rc; } #else # define vdbeSorterJoinAll(x,rcin) (rcin) +# define vdbeSorterJoinThread(pTask,p) SQLITE_OK #endif /* @@ -908,11 +1009,11 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); } - if( pSorter->aMemory==0 ){ - vdbeSorterRecordFree(0, pSorter->pRecord); + if( pSorter->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pSorter->list.pList); } - pSorter->pRecord = 0; - pSorter->nInMemory = 0; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; pSorter->mxKeysize = 0; @@ -927,7 +1028,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - sqlite3_free(pSorter->aMemory); + sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -952,6 +1053,21 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ return rc; } +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + + /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. @@ -991,25 +1107,29 @@ static void vdbeSorterMerge( ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SortSubtask *pTask){ +static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; SorterRecord **aSlot; SorterRecord *p; + int rc; + + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) return rc; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pTask->pList; + p = pList->pList; while( p ){ SorterRecord *pNext; - if( pTask->aListMemory ){ - if( (u8*)p==pTask->aListMemory ){ + if( pList->aMemory ){ + if( (u8*)p==pList->aMemory ){ pNext = 0; }else{ - assert( p->u.iNextaListMemory) ); - pNext = (SorterRecord*)&pTask->aListMemory[p->u.iNext]; + assert( p->u.iNextaMemory) ); + pNext = (SorterRecord*)&pList->aMemory[p->u.iNext]; } }else{ pNext = p->u.pNext; @@ -1028,9 +1148,13 @@ static int vdbeSorterSort(SortSubtask *pTask){ for(i=0; i<64; i++){ vdbeSorterMerge(pTask, p, aSlot[i], &p); } - pTask->pList = p; + pList->pList = p; sqlite3_free(aSlot); + if( pTask->pUnpacked->errCode ){ + assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); + return SQLITE_NOMEM; + } return SQLITE_OK; } @@ -1144,8 +1268,9 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ /* -** Write the current contents of the in-memory linked-list to a PMA. Return -** SQLITE_OK if successful, or an SQLite error code otherwise. +** Write the current contents of in-memory linked-list pList to a level-0 +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -1156,12 +1281,19 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SortSubtask *pTask){ +static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ int rc = SQLITE_OK; /* Return code */ PmaWriter writer; /* Object used to write to the file */ +#ifdef SQLITE_DEBUG + /* Set iSz to the expected size of file pTask->file after writing the PMA. + ** This is used by an assert() statement at the end of this function. */ + i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; +#endif + + vdbeSorterWorkDebug(pTask, "enter"); memset(&writer, 0, sizeof(PmaWriter)); - assert( pTask->nInMemory>0 ); + assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ @@ -1174,10 +1306,15 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ vdbeSorterExtendFile(pTask->db, - pTask->file.pFd, pTask->file.iEof + pTask->nInMemory + 9 + pTask->file.pFd, pTask->file.iEof + pList->szPMA + 9 ); } + /* Sort the list */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterSort(pTask, pList); + } + if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; @@ -1185,18 +1322,20 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, pTask->file.iEof); pTask->nPMA++; - vdbePmaWriteVarint(&writer, pTask->nInMemory); - for(p=pTask->pList; p; p=pNext){ + vdbePmaWriteVarint(&writer, pList->szPMA); + for(p=pList->pList; p; p=pNext){ pNext = p->u.pNext; vdbePmaWriteVarint(&writer, p->nVal); vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); - if( pTask->aListMemory==0 ) sqlite3_free(p); + if( pList->aMemory==0 ) sqlite3_free(p); } - pTask->pList = p; + pList->pList = p; rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); } - assert( pTask->pList==0 || rc!=SQLITE_OK ); + vdbeSorterWorkDebug(pTask, "exit"); + assert( rc!=SQLITE_OK || pList->pList==0 ); + assert( rc!=SQLITE_OK || pTask->file.iEof==iSz ); return rc; } @@ -1275,249 +1414,84 @@ static int vdbeSorterNext( return rc; } -#if 0 -static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ - i64 t; - int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); - fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); -} -static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ - i64 t; - sqlite3OsCurrentTimeInt64(db->pVfs, &t); - fprintf(stderr, "%lld:X %s\n", t, zEvent); -} -static void vdbeSorterPopulateDebug( - SortSubtask *pTask, - const char *zEvent -){ - i64 t; - int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); - fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); -} -#else -# define vdbeSorterWorkDebug(x,y) -# define vdbeSorterRewindDebug(x,y) -# define vdbeSorterPopulateDebug(x,y) -#endif - -static int vdbeSortAllocUnpacked(SortSubtask *pTask){ - if( pTask->pUnpacked==0 ){ - char *pFree; - pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree - ); - assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ) return SQLITE_NOMEM; - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; - pTask->pUnpacked->errCode = 0; - } - return SQLITE_OK; -} - /* ** The main routine for sorter-thread operations. */ -static void *vdbeSortSubtaskMain(void *pCtx){ - int rc = SQLITE_OK; +static void *vdbeSorterFlushThread(void *pCtx){ SortSubtask *pTask = (SortSubtask*)pCtx; - - assert( pTask->eWork==SORT_SUBTASK_SORT - || pTask->eWork==SORT_SUBTASK_TO_PMA - || pTask->eWork==SORT_SUBTASK_CONS - ); - assert( pTask->bDone==0 ); - - vdbeSorterWorkDebug(pTask, "enter"); - - rc = vdbeSortAllocUnpacked(pTask); - if( rc!=SQLITE_OK ) goto thread_out; - - if( pTask->eWork==SORT_SUBTASK_CONS ){ - assert( pTask->pList==0 ); - while( pTask->nPMA>pTask->nConsolidate && rc==SQLITE_OK ){ - int nIter = MIN(pTask->nPMA, SORTER_MAX_MERGE_COUNT); - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - MergeEngine *pMerger; /* Object for reading/merging PMA data */ - i64 iReadOff = 0; /* Offset in pTemp1 to read from */ - i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ - int i; - - /* Allocate a merger object to merge PMAs together. */ - pMerger = vdbeMergeEngineNew(nIter); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - break; - } - - /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); - if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, pTemp2, pTask->file.iEof); - }else{ - vdbeMergeEngineFree(pMerger); - break; - } - - /* This loop runs once for each output PMA. Each output PMA is made - ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; rc==SQLITE_OK && inPMA; i+=SORTER_MAX_MERGE_COUNT){ - PmaWriter writer; /* Object for writing data to pTemp2 */ - i64 nOut = 0; /* Bytes of data in output PMA */ - int bEof = 0; - int rc2; - - /* Configure the merger object to read and merge data from the next - ** SORTER_MAX_MERGE_COUNT PMAs in pTemp1 (or from all remaining PMAs, - ** if that is fewer). */ - int iIter; - for(iIter=0; iIteraIter[iIter]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nOut); - iReadOff = pIter->iEof; - if( iReadOff>=pTask->file.iEof || rc!=SQLITE_OK ) break; - } - for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ - rc = vdbeSorterDoCompare(pTask, pMerger, iIter); - } - - vdbePmaWriterInit(pTemp2, &writer, pTask->pgsz, iWriteOff); - vdbePmaWriteVarint(&writer, nOut); - while( rc==SQLITE_OK && bEof==0 ){ - PmaReader *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; - assert( pIter->pFile!=0 ); /* pIter is not at EOF */ - vdbePmaWriteVarint(&writer, pIter->nKey); - vdbePmaWriteBlob(&writer, pIter->aKey, pIter->nKey); - rc = vdbeSorterNext(pTask, pMerger, &bEof); - } - rc2 = vdbePmaWriterFinish(&writer, &iWriteOff); - if( rc==SQLITE_OK ) rc = rc2; - } - - vdbeMergeEngineFree(pMerger); - sqlite3OsCloseFree(pTask->file.pFd); - pTask->file.pFd = pTemp2; - pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pTask->file.iEof = iWriteOff; - } - }else{ - /* Sort the pTask->pList list */ - rc = vdbeSorterSort(pTask); - - /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pTask->eWork==SORT_SUBTASK_TO_PMA ){ -#ifdef SQLITE_DEBUG - i64 nExpect = pTask->nInMemory - + sqlite3VarintLen(pTask->nInMemory) - + pTask->file.iEof; -#endif - rc = vdbeSorterListToPMA(pTask); - assert( rc!=SQLITE_OK || (nExpect==pTask->file.iEof) ); - } - } - - thread_out: - pTask->bDone = 1; - if( rc==SQLITE_OK && pTask->pUnpacked->errCode ){ - assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); - rc = SQLITE_NOMEM; - } - vdbeSorterWorkDebug(pTask, "exit"); + int rc; /* Return code */ + assert( pTask->thread.bDone==0 ); + rc = vdbeSorterListToPMA(pTask, &pTask->list); + pTask->thread.bDone = 1; return SQLITE_INT_TO_PTR(rc); } /* -** Run the activity scheduled by the object passed as the only argument -** in the current thread. -*/ -static int vdbeSorterRunTask(SortSubtask *pTask){ - int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pTask) ); - assert( pTask->bDone ); - pTask->bDone = 0; - return rc; -} - -/* -** Flush the current contents of VdbeSorter.pRecord to a new PMA, possibly +** Flush the current contents of VdbeSorter.list to a new PMA, possibly ** using a background thread. -** -** If argument bFg is non-zero, the operation always uses the calling thread. */ -static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ - VdbeSorter *pSorter = pCsr->pSorter; +static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ +#if SQLITE_MAX_WORKER_THREADS==0 + pSorter->bUsePMA = 1; + return vdbeSorterListToPMA(&pSorter->aTask[0], &pSorter->list); +#else int rc = SQLITE_OK; int i; SortSubtask *pTask = 0; /* Thread context used to create new PMA */ int nWorker = (pSorter->nTask-1); + /* Set the flag to indicate that at least one PMA has been written. + ** Or will be, anyhow. */ pSorter->bUsePMA = 1; + + /* Select a sub-task to sort and flush the current list of in-memory + ** records to disk. If the sorter is running in multi-threaded mode, + ** round-robin between the first (pSorter->nTask-1) tasks. Except, if + ** the background thread from a sub-tasks previous turn is still running, + ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, + ** fall back to using the final sub-task. The first (pSorter->nTask-1) + ** sub-tasks are prefered as they use background threads - the final + ** sub-task uses the main thread. */ for(i=0; iiPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; -#if SQLITE_MAX_WORKER_THREADS>0 - if( pTask->bDone ){ - void *pRet; - assert( pTask->pThread ); - rc = sqlite3ThreadJoin(pTask->pThread, &pRet); - pTask->pThread = 0; - pTask->bDone = 0; - if( rc==SQLITE_OK ){ - rc = SQLITE_PTR_TO_INT(pRet); - } + if( pTask->thread.bDone ){ + rc = vdbeSorterJoinThread(pTask, &pTask->thread); } -#endif - if( pTask->pThread==0 ) break; - pTask = 0; + if( pTask->thread.pThread==0 || rc!=SQLITE_OK ) break; } - if( pTask==0 ){ - pTask = &pSorter->aTask[nWorker]; - } - pSorter->iPrev = (pTask - pSorter->aTask); if( rc==SQLITE_OK ){ - assert( pTask->pThread==0 && pTask->bDone==0 ); - pTask->eWork = SORT_SUBTASK_TO_PMA; - pTask->pList = pSorter->pRecord; - pTask->nInMemory = pSorter->nInMemory; - pSorter->nInMemory = 0; - pSorter->pRecord = 0; - - if( pSorter->aMemory ){ - u8 *aMem = pTask->aListMemory; - pTask->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; - } - -#if SQLITE_MAX_WORKER_THREADS>0 - if( !bFg && pTask!=&pSorter->aTask[nWorker] ){ - /* Launch a background thread for this operation */ - void *pCtx = (void*)pTask; - assert( pSorter->aMemory==0 || pTask->aListMemory!=0 ); - if( pTask->aListMemory ){ - if( pSorter->aMemory==0 ){ - pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); - if( pSorter->aMemory==0 ) return SQLITE_NOMEM; - }else{ - pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); - } - } - rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); - }else -#endif - { + if( i==nWorker ){ /* Use the foreground thread for this operation */ - rc = vdbeSorterRunTask(pTask); - if( rc==SQLITE_OK ){ - u8 *aMem = pTask->aListMemory; - pTask->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; - assert( pTask->pList==0 ); + rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); + }else{ + /* Launch a background thread for this operation */ + u8 *aMem = pTask->list.aMemory; + void *pCtx = (void*)pTask; + + assert( pTask->thread.pThread==0 && pTask->thread.bDone==0 ); + assert( pTask->list.pList==0 ); + assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + + pSorter->iPrev = (pTask - pSorter->aTask); + pTask->list = pSorter->list; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + if( aMem ){ + pSorter->list.aMemory = aMem; + pSorter->nMemory = sqlite3MallocSize(aMem); + }else{ + pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); + if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } + + rc = vdbeSorterCreateThread(&pTask->thread, vdbeSorterFlushThread, pCtx); } } return rc; +#endif } /* @@ -1557,28 +1531,28 @@ int sqlite3VdbeSorterWrite( nReq = pVal->n + sizeof(SorterRecord); nPMA = pVal->n + sqlite3VarintLen(pVal->n); if( pSorter->mxPmaSize ){ - if( pSorter->aMemory ){ + if( pSorter->list.aMemory ){ bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; }else{ bFlush = ( - (pSorter->nInMemory > pSorter->mxPmaSize) - || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + (pSorter->list.szPMA > pSorter->mxPmaSize) + || (pSorter->list.szPMA > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) ); } if( bFlush ){ - rc = vdbeSorterFlushPMA(db, pCsr, 0); - pSorter->nInMemory = 0; + rc = vdbeSorterFlushPMA(pSorter); + pSorter->list.szPMA = 0; pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + assert( rc!=SQLITE_OK || pSorter->list.pList==0 ); } } - pSorter->nInMemory += nPMA; + pSorter->list.szPMA += nPMA; if( nPMA>pSorter->mxKeysize ){ pSorter->mxKeysize = nPMA; } - if( pSorter->aMemory ){ + if( pSorter->list.aMemory ){ int nMin = pSorter->iMemory + nReq; if( nMin>pSorter->nMemory ){ @@ -1588,45 +1562,33 @@ int sqlite3VdbeSorterWrite( if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; - aNew = sqlite3Realloc(pSorter->aMemory, nNew); + aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = (SorterRecord*)( - aNew + ((u8*)pSorter->pRecord - pSorter->aMemory) + pSorter->list.pList = (SorterRecord*)( + aNew + ((u8*)pSorter->list.pList - pSorter->list.aMemory) ); - pSorter->aMemory = aNew; + pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } - pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory]; + pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; + pNew->u.iNext = (u8*)(pSorter->list.pList) - pSorter->list.aMemory; }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ return SQLITE_NOMEM; } - pNew->u.pNext = pSorter->pRecord; + pNew->u.pNext = pSorter->list.pList; } memcpy(SRVAL(pNew), pVal->z, pVal->n); pNew->nVal = pVal->n; - pSorter->pRecord = pNew; + pSorter->list.pList = pNew; return rc; } -/* -** Return the total number of PMAs in all temporary files. -*/ -static int vdbeSorterCountPMA(VdbeSorter *pSorter){ - int nPMA = 0; - int i; - for(i=0; inTask; i++){ - nPMA += pSorter->aTask[i].nPMA; - } - return nPMA; -} - /* ** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format ** of the data stored in aFile[1] is the same as that used by regular PMAs, @@ -1667,39 +1629,27 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ return rc; } -static void *vdbeIncrPopulateThreadMain(void *pCtx){ +static void *vdbeIncrPopulateThread(void *pCtx){ IncrMerger *pIncr = (IncrMerger*)pCtx; - return SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + pIncr->thread.bDone = 1; + return pRet; } -static int vdbeIncrBgPopulate(IncrMerger *pIncr){ - int rc; - assert( pIncr->pThread==0 ); - if( pIncr->bUseThread==0 ){ - rc = vdbeIncrPopulate(pIncr); - } #if SQLITE_MAX_WORKER_THREADS>0 - else{ - void *pCtx = (void*)pIncr; - rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); - } -#endif - return rc; +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + void *pCtx = (void*)pIncr; + assert( pIncr->bUseThread ); + return vdbeSorterCreateThread(&pIncr->thread, vdbeIncrPopulateThread, pCtx); } +#endif static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; - if( pIncr->bUseThread ){ #if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->pThread ){ - void *pRet; - assert( pIncr->bUseThread ); - rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - pIncr->pThread = 0; - } -#endif + if( pIncr->bUseThread ){ + rc = vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); if( rc==SQLITE_OK ){ SorterFile f0 = pIncr->aFile[0]; @@ -1714,7 +1664,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ rc = vdbeIncrBgPopulate(pIncr); } } - }else{ + }else +#endif + { rc = vdbeIncrPopulate(pIncr); pIncr->aFile[0] = pIncr->aFile[1]; if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ @@ -1728,10 +1680,7 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ static void vdbeIncrFree(IncrMerger *pIncr){ if( pIncr ){ #if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->pThread ){ - void *pRet; - sqlite3ThreadJoin(pIncr->pThread, &pRet); - } + vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); if( pIncr->bUseThread ){ if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); @@ -1750,18 +1699,6 @@ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); pTask->file2.iEof += pIncr->mxSz; - -#if 0 - /* Open the two temp files. */ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); - } - if( rc!=SQLITE_OK ){ - vdbeIncrFree(pIncr); - pIncr = 0; - } -#endif } return pIncr; } @@ -1784,9 +1721,6 @@ static int vdbeIncrInit2(PmaReader *pIter){ for(i=0; rc==SQLITE_OK && inTree; i++){ rc = vdbeIncrInit2(&pMerger->aIter[i]); } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); - } /* Set up the required files for pIncr */ if( rc==SQLITE_OK ){ @@ -1812,6 +1746,10 @@ static int vdbeIncrInit2(PmaReader *pIter){ } } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); + } + if( rc==SQLITE_OK && pIncr->bUseThread ){ rc = vdbeIncrBgPopulate(pIncr); } @@ -1998,9 +1936,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } } } - if( rc==SQLITE_OK ){ - rc = vdbeIncrInit2(pIter); - } + if( rc==SQLITE_OK ) rc = vdbeIncrInit2(pIter); sqlite3_free(aMerge); return rc; @@ -2022,17 +1958,9 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ - if( pSorter->pRecord ){ - SortSubtask *pTask = &pSorter->aTask[0]; + if( pSorter->list.pList ){ *pbEof = 0; - pTask->pList = pSorter->pRecord; - pTask->eWork = SORT_SUBTASK_SORT; - assert( pTask->aListMemory==0 ); - pTask->aListMemory = pSorter->aMemory; - rc = vdbeSorterRunTask(pTask); - pTask->aListMemory = 0; - pSorter->pRecord = pTask->pList; - pTask->pList = 0; + rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ *pbEof = 1; } @@ -2040,8 +1968,8 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ } /* Write the current in-memory list to a PMA. */ - if( pSorter->pRecord ){ - rc = vdbeSorterFlushPMA(db, pCsr, 1); + if( pSorter->list.pList ){ + rc = vdbeSorterFlushPMA(pSorter); } /* Join all threads */ @@ -2076,11 +2004,11 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ rc = vdbePmaReaderNext(pSorter->pReader); *pbEof = (pSorter->pReader->pFile==0); }else{ - SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->u.pNext; + SorterRecord *pFree = pSorter->list.pList; + pSorter->list.pList = pFree->u.pNext; pFree->u.pNext = 0; - if( pSorter->aMemory==0 ) vdbeSorterRecordFree(db, pFree); - *pbEof = !pSorter->pRecord; + if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->list.pList; rc = SQLITE_OK; } return rc; @@ -2099,8 +2027,8 @@ static void *vdbeSorterRowkey( *pnKey = pSorter->pReader->nKey; pKey = pSorter->pReader->aKey; }else{ - *pnKey = pSorter->pRecord->nVal; - pKey = SRVAL(pSorter->pRecord); + *pnKey = pSorter->list.pList->nVal; + pKey = SRVAL(pSorter->list.pList); } return pKey; } From be3018c18f69c5cfbd689e3772a502fcc2edd0b4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 07:30:39 +0000 Subject: [PATCH 39/99] Improve use of multiple threads in sqlite3VdbeSorterRewind(). FossilOrigin-Name: e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 44 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index bca9ab362e..bb8ef3522e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smany\sissues\swith\snew\scode. -D 2014-04-12T19:34:44.467 +C Improve\suse\sof\smultiple\sthreads\sin\ssqlite3VdbeSorterRewind(). +D 2014-04-14T07:30:39.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c bc0d90e00abcc88997f463d4d41b7ba4a10cfd88 +F src/vdbesort.c b047de6a9c89b122ad8649b083c848b6336b91cb F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 98bf0307b121b0776a7170108cc8d3f948a7ebfe -R f3107fdb117ba86f9bee4609a5b08bfd +P 62c406a042d7246f6df6b943421182a88483b2e3 +R c3e4d1b3a81e551661cfec7b71fa4391 U dan -Z 5a1b16a83fca264558f06f5fb6536949 +Z 2f53ddd699d6387555ef072104853e10 diff --git a/manifest.uuid b/manifest.uuid index e7ab0df5c2..1fc2ca4f8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62c406a042d7246f6df6b943421182a88483b2e3 \ No newline at end of file +e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 1889c8fde1..32305ed792 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1710,7 +1710,10 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ } } -static int vdbeIncrInit2(PmaReader *pIter){ +#define INCRINIT2_NORMAL 0 +#define INCRINIT2_TASK 1 +#define INCRINIT2_ROOT 2 +static int vdbeIncrInit2(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ @@ -1719,7 +1722,12 @@ static int vdbeIncrInit2(PmaReader *pIter){ MergeEngine *pMerger = pIncr->pMerger; for(i=0; rc==SQLITE_OK && inTree; i++){ - rc = vdbeIncrInit2(&pMerger->aIter[i]); + IncrMerger *p; + if( eMode==INCRINIT2_ROOT ){ + rc = vdbePmaReaderNext(&pMerger->aIter[i]); + }else{ + rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + } } /* Set up the required files for pIncr */ @@ -1751,16 +1759,32 @@ static int vdbeIncrInit2(PmaReader *pIter){ } if( rc==SQLITE_OK && pIncr->bUseThread ){ - rc = vdbeIncrBgPopulate(pIncr); + /* Use the current thread */ + assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); + rc = vdbeIncrPopulate(pIncr); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eMode!=INCRINIT2_TASK ){ rc = vdbePmaReaderNext(pIter); } } return rc; } +static void *vdbeIncrInit2Thread(void *pCtx){ + PmaReader *pReader = (PmaReader*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); + pReader->pIncr->thread.bDone = 1; + return pRet; +} + +static int vdbeIncrBgInit2(PmaReader *pIter){ + void *pCtx = (void*)pIter; + return vdbeSorterCreateThread( + &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx + ); +} + /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 ** PMAs from pTask->file. If no error occurs, set *ppOut to point to @@ -1933,10 +1957,20 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ assert( pIncr->pTask!=pLast ); } } + if( pSorter->nTask>1 ){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + } + } } } } - if( rc==SQLITE_OK ) rc = vdbeIncrInit2(pIter); + if( rc==SQLITE_OK ){ + int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + rc = vdbeIncrInit2(pIter, eMode); + } sqlite3_free(aMerge); return rc; From 92a20dde1f91f5485216dd10b51cc6de99fedd44 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 08:45:32 +0000 Subject: [PATCH 40/99] Minor fixes so that builds with SQLITE_MAX_WORKER_THREADS=0 work. FossilOrigin-Name: e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bb8ef3522e..eb0882e7ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\suse\sof\smultiple\sthreads\sin\ssqlite3VdbeSorterRewind(). -D 2014-04-14T07:30:39.899 +C Minor\sfixes\sso\sthat\sbuilds\swith\sSQLITE_MAX_WORKER_THREADS=0\swork. +D 2014-04-14T08:45:32.394 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b047de6a9c89b122ad8649b083c848b6336b91cb +F src/vdbesort.c 364cb94cf1eaefa1f79d86b7b1e4778c58192e1c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 62c406a042d7246f6df6b943421182a88483b2e3 -R c3e4d1b3a81e551661cfec7b71fa4391 +P e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 +R 15b762d2cb0315f25703d6fa43b514ac U dan -Z 2f53ddd699d6387555ef072104853e10 +Z 8e0f4aca92a8e556e8d49ca399252be9 diff --git a/manifest.uuid b/manifest.uuid index 1fc2ca4f8c..bd230a5f99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 \ No newline at end of file +e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 32305ed792..870e4e2e1c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1771,6 +1771,7 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); @@ -1784,6 +1785,7 @@ static int vdbeIncrBgInit2(PmaReader *pIter){ &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx ); } +#endif /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 @@ -1948,7 +1950,9 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ pIter->pIncr = vdbeIncrNew(pLast, pMain); if( pIter->pIncr==0 ){ rc = SQLITE_NOMEM; - }else{ + } +#if SQLITE_MAX_WORKER_THREADS>0 + else{ vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; @@ -1965,6 +1969,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } } } +#endif } } if( rc==SQLITE_OK ){ From f77ceba59467de3149dab507ff58e356a6d26024 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 18:41:21 +0000 Subject: [PATCH 41/99] Improve performance in single-threaded mode by having the final merge pass keys directly to the VDBE, instead of going via a final PMA. FossilOrigin-Name: 02610cd9b77caa2c181210056088beb3ad6ce30f --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 154 +++++++++++++++++++++++++++++-------------------- 3 files changed, 98 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index eb0882e7ed..08cf12e621 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sso\sthat\sbuilds\swith\sSQLITE_MAX_WORKER_THREADS=0\swork. -D 2014-04-14T08:45:32.394 +C Improve\sperformance\sin\ssingle-threaded\smode\sby\shaving\sthe\sfinal\smerge\spass\skeys\sdirectly\sto\sthe\sVDBE,\sinstead\sof\sgoing\svia\sa\sfinal\sPMA. +D 2014-04-14T18:41:21.894 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 364cb94cf1eaefa1f79d86b7b1e4778c58192e1c +F src/vdbesort.c 15f59dc56c7c4509b4783ad20fb25479ac63d267 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 -R 15b762d2cb0315f25703d6fa43b514ac +P e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f +R b74f7a073d8c77e32a2dce332520aa10 U dan -Z 8e0f4aca92a8e556e8d49ca399252be9 +Z cd43207b847b4088208c6c3dd9c99bf8 diff --git a/manifest.uuid b/manifest.uuid index bd230a5f99..0cd782fad3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f \ No newline at end of file +02610cd9b77caa2c181210056088beb3ad6ce30f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 870e4e2e1c..857b7ba690 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -280,21 +280,17 @@ struct MergeEngine { struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ - int bUsePMA; /* True if one or more PMAs created */ - int bUseThreads; /* True if one or more PMAs created */ PmaReader *pReader; /* Read data from here after Rewind() */ + MergeEngine *pMerger; /* Or here, if bUseThreads==0 */ int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ -#if 0 - int nInMemory; /* Current size of pRecord list as PMA */ - SorterRecord *pRecord; /* Head of in-memory record list */ - u8 *aMemory; /* Block of memory to alloc records from */ -#endif - SorterList list; - int iMemory; /* Offset of first free byte in aMemory */ - int nMemory; /* Size of aMemory allocation in bytes */ - int iPrev; /* Previous thread used to flush PMA */ - int nTask; /* Size of aTask[] array */ + SorterList list; /* List of in-memory records */ + int iMemory; /* Offset of free space in list.aMemory */ + int nMemory; /* Size of list.aMemory allocation in bytes */ + u8 bUsePMA; /* True if one or more PMAs created */ + u8 bUseThreads; /* True to use background threads */ + u8 iPrev; /* Previous thread used to flush PMA */ + u8 nTask; /* Size of aTask[] array */ SortSubtask aTask[1]; /* One or more subtasks */ }; @@ -1005,6 +1001,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ sqlite3DbFree(db, pSorter->pReader); pSorter->pReader = 0; } + vdbeMergeEngineFree(pSorter->pMerger); + pSorter->pMerger = 0; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); @@ -1713,22 +1711,39 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 + +static int vdbeIncrInit2(PmaReader *pIter, int eMode); + +static int vdbeIncrInitMerger( + SortSubtask *pTask, + MergeEngine *pMerger, + int eMode +){ + int i; + int rc = SQLITE_OK; + + for(i=0; rc==SQLITE_OK && inTree; i++){ + if( eMode==INCRINIT2_ROOT ){ + rc = vdbePmaReaderNext(&pMerger->aIter[i]); + }else{ + rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + } + } + + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pTask, pMerger, i); + } + + return rc; +} + static int vdbeIncrInit2(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; - int i; - MergeEngine *pMerger = pIncr->pMerger; - for(i=0; rc==SQLITE_OK && inTree; i++){ - IncrMerger *p; - if( eMode==INCRINIT2_ROOT ){ - rc = vdbePmaReaderNext(&pMerger->aIter[i]); - }else{ - rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); - } - } + rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr */ if( rc==SQLITE_OK ){ @@ -1754,10 +1769,6 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ } } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); - } - if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread */ assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); @@ -1871,7 +1882,7 @@ static int vdbeAddToBuilder( ** Populate iterator *pIter so that it may be used to iterate through all ** keys stored in all PMAs created by this sorter. */ -static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ +static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; sqlite3 *db = pTask0->db; @@ -1943,39 +1954,49 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } if( rc==SQLITE_OK ){ - SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; - - rc = vdbeSortAllocUnpacked(pLast); - if( rc==SQLITE_OK ){ - pIter->pIncr = vdbeIncrNew(pLast, pMain); - if( pIter->pIncr==0 ){ - rc = SQLITE_NOMEM; +#if SQLITE_MAX_WORKER_THREADS + if( pSorter->bUseThreads ){ + PmaReader *pIter; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pIter; } -#if SQLITE_MAX_WORKER_THREADS>0 - else{ - vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); - for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ - IncrMerger *pIncr; - if( (pIncr = pMain->aIter[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); - assert( pIncr->pTask!=pLast ); - } + if( rc==SQLITE_OK ){ + pIter->pIncr = vdbeIncrNew(pLast, pMain); + if( pIter->pIncr==0 ){ + rc = SQLITE_NOMEM; } - if( pSorter->nTask>1 ){ - for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + else{ + vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aIter[iTask].pIncr) ){ + vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + assert( pIncr->pTask!=pLast ); + } + } + if( pSorter->nTask>1 ){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + } } } } + if( rc==SQLITE_OK ){ + int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + rc = vdbeIncrInit2(pIter, eMode); + } + }else #endif + { + pSorter->pMerger = pMain; + rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); } } - if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); - rc = vdbeIncrInit2(pIter, eMode); - } sqlite3_free(aMerge); return rc; @@ -2020,11 +2041,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** incrementally read and merge all remaining PMAs. */ assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - PmaReader *pReader; - pReader = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); - pSorter->pReader = pReader; - rc = vdbePmaReaderIncrInit(pSorter, pReader); - assert( rc!=SQLITE_OK || pReader->pFile ); + rc = vdbePmaReaderIncrInit(pSorter); *pbEof = 0; } @@ -2039,9 +2056,17 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->pReader ){ - rc = vdbePmaReaderNext(pSorter->pReader); - *pbEof = (pSorter->pReader->pFile==0); + assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); + if( pSorter->bUsePMA ){ + assert( pSorter->pReader==0 || pSorter->pMerger==0 ); + assert( pSorter->bUseThreads==0 || pSorter->pReader ); + assert( pSorter->bUseThreads==1 || pSorter->pMerger ); + if( pSorter->bUseThreads ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFile==0); + }else{ + rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + } }else{ SorterRecord *pFree = pSorter->list.pList; pSorter->list.pList = pFree->u.pNext; @@ -2062,9 +2087,12 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->pReader ){ - *pnKey = pSorter->pReader->nKey; - pKey = pSorter->pReader->aKey; + if( pSorter->bUsePMA ){ + PmaReader *pReader = (pSorter->bUseThreads ? + pSorter->pReader : &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]] + ); + *pnKey = pReader->nKey; + pKey = pReader->aKey; }else{ *pnKey = pSorter->list.pList->nVal; pKey = SRVAL(pSorter->list.pList); From 1a088a8ef58bb012e5ecb2e2e807530797db18c3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 Apr 2014 19:52:34 +0000 Subject: [PATCH 42/99] Fix further code and documentation issues in vdbesort.c. FossilOrigin-Name: d03f5b8622d304f029f73c7cd0bee3182a81d081 --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 540 +++++++++++++++++++++++++++---------------------- 3 files changed, 310 insertions(+), 244 deletions(-) diff --git a/manifest b/manifest index f04882a0e8..0ca4cbc345 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\ssorter\sto\sbegin\sreturning\sdata\sto\sthe\sVDBE\sas\ssoon\sas\sit\sis\savailable,\sinstead\sof\swaiting\suntil\sall\skeys\shave\sbeen\ssorted. -D 2014-04-14T19:23:18.139 +C Fix\sfurther\scode\sand\sdocumentation\sissues\sin\svdbesort.c. +D 2014-04-15T19:52:34.797 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 15f59dc56c7c4509b4783ad20fb25479ac63d267 +F src/vdbesort.c ceb8e16055327d0c52bdd2087fcdd7d132fe314f F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e54dded2012f0ab486ee138e9bd57c528af33980 02610cd9b77caa2c181210056088beb3ad6ce30f -R b74f7a073d8c77e32a2dce332520aa10 +P cb0ab20c48962cdee03115efa93d7d501780ac73 +R 44c59b06ba4f012b63c2d0e26d30a7e6 U dan -Z ebd968da8bc859eb7bfdd863767ac91c +Z 2cc8bdff8b0228a6efe530348d9564c8 diff --git a/manifest.uuid b/manifest.uuid index 1714edb86b..b7e5cbc645 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb0ab20c48962cdee03115efa93d7d501780ac73 \ No newline at end of file +d03f5b8622d304f029f73c7cd0bee3182a81d081 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 857b7ba690..8b6cce6259 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** a VdbeCursor to sort large numbers of keys for CREATE INDEX statements ** or by SELECT statements with ORDER BY clauses that cannot be satisfied ** using indexes and without LIMIT clauses. ** @@ -57,33 +57,84 @@ ** ** The interfaces above must be called in a particular order. Write() can ** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and -** Compare() can only occur in between Rewind() and Close()/Reset(). +** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. +** +** Init() +** for each record: Write() +** Rewind() +** Rowkey()/Compare() +** Next() +** Close() ** ** Algorithm: ** -** Records to be sorted are initially held in memory, in the order in -** which they arrive from Write(). When the amount of memory needed exceeds -** a threshold, all in-memory records are sorted and then appended to -** a temporary file as a "Packed-Memory-Array" or "PMA" and the memory is -** reset. There is a single temporary file used for all PMAs. The PMAs -** are packed one after another in the file. The VdbeSorter object keeps -** track of the number of PMAs written. +** Records passed to the sorter via calls to Write() are initially held +** unsorted in main memory. Assuming the amount of memory used never exceeds +** a threshold, when Rewind() is called the set of records is sorted using +** an in-memory merge sort. In this case, no temporary files are required +** and subsequent calls to Rowkey(), Next() and Compare() read records +** directly from main memory. ** -** When the Rewind() is seen, any records still held in memory are sorted. -** If no PMAs have been written (if all records are still held in memory) -** then subsequent Rowkey(), Next(), and Compare() operations work directly -** from memory. But if PMAs have been written things get a little more -** complicated. +** If the amount of space used to store records in main memory exceeds the +** threshold, then the set of records currently in memory are sorted and +** written to a temporary file in "Packed Memory Array" (PMA) format. +** A PMA created at this point is known as a "level-0 PMA". Higher levels +** of PMAs may be created by merging existing PMAs together - for example +** merging two or more level-0 PMAs together creates a level-1 PMA. ** -** When Rewind() is seen after PMAs have been written, any records still -** in memory are sorted and written as a final PMA. Then all the PMAs -** are merged together into a single massive PMA that Next(), Rowkey(), -** and Compare() walk to extract the records in sorted order. +** The threshold for the amount of main memory to use before flushing +** records to a PMA is roughly the same as the limit configured for the +** page-cache of the main database. Specifically, the threshold is set to +** the value returned multiplied by "PRAGMA main.page_size" multipled by +** that returned by "PRAGMA main.cache_size", in bytes. ** -** If SQLITE_MAX_WORKER_THREADS is non-zero, various steps of the above -** algorithm might be performed in parallel by separate threads. Threads -** are only used when one or more PMA spill to disk. If the sort is small -** enough to fit entirely in memory, everything happens on the main thread. +** If the sorter is running in single-threaded mode, then all PMAs generated +** are appended to a single temporary file. Or, if the sorter is running in +** multi-threaded mode then up to (N+1) temporary files may be opened, where +** N is the configured number of worker threads. In this case, instead of +** sorting the records and writing the PMA to a temporary file itself, the +** calling thread usually launches a worker thread to do so. Except, if +** there are already N worker threads running, the main thread does the work +** itself. +** +** The sorter is running in multi-threaded mode if (a) the library was built +** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater +** than zero, and (b) worker threads have been enabled at runtime by calling +** sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, ...). +** +** When Rewind() is called, any data remaining in memory is flushed to a +** final PMA. So at this point the data is stored in some number of sorted +** PMAs within temporary files on disk. Within a single file sorter is +** running in single threaded mode, or distributed between one or more files +** for multi-threaded sorters. +** +** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the +** sorter is running in single-threaded mode, then these PMAs are merged +** incrementally as keys are retreived from the sorter by the VDBE. See +** comments above object MergeEngine below for details. +** +** Or, if running in multi-threaded mode, then a background thread is +** launched to merge the existing PMAs. Once the background thread has +** merged T bytes of data into a single sorted PMA, the main thread +** begins reading keys from that PMA while the background thread proceeds +** with merging the next T bytes of data. And so on. +** +** Parameter T is set to half the value of the memory threshold used +** by Write() above to determine when to create a new PMA. +** +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** disk are merged together. Then T bytes of data from the second set, and +** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT +** PMAs at a time. This done is to improve locality. +** +** If running in multi-threaded mode and there are more than +** SORTER_MAX_MERGE_COUNT PMAs on disk when Rewind() is called, then more +** than one background thread may be created. Specifically, there may be +** one background thread for each temporary file on disk, and one background +** thread to merge the output of each of the others to a single PMA for +** the main thread to read from. */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -102,12 +153,11 @@ */ typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ -typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write one PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ -typedef struct SorterFile SorterFile; -typedef struct SorterThread SorterThread; -typedef struct SorterList SorterList; +typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ +typedef struct SorterList SorterList; /* In-memory list of records */ typedef struct IncrMerger IncrMerger; /* @@ -120,85 +170,14 @@ struct SorterFile { }; /* -** An object of this type is used to store the thread handle for each -** background thread launched by the sorter. Before the thread is launched, -** variable bDone is set to 0. Then, right before it exits, the thread -** itself sets bDone to 1. -** -** This is then used for two purposes: -** -** 1. When flushing the contents of memory to a level-0 PMA on disk, to -** attempt to select a SortSubtask for which there is not already an -** active background thread (since doing so causes the main thread -** to block until it finishes). -** -** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call -** to sqlite3ThreadJoin() is likely to block. -** -** In both cases, the effects of the main thread seeing (bDone==0) even -** after the thread has finished are not dire. So we don't worry about -** memory barriers and such here. +** In memory linked list of records. */ -struct SorterThread { - SQLiteThread *pThread; - int bDone; -}; - struct SorterList { SorterRecord *pList; /* Linked list of records */ u8 *aMemory; /* If non-NULL, blob of memory for pList */ int szPMA; /* Size of pList as PMA in bytes */ }; -/* -** Sorting is divided up into smaller subtasks. Each subtask is controlled -** by an instance of this object. A Subtask might run in either the main thread -** or in a background thread. -** -** Exactly VdbeSorter.nTask instances of this object are allocated -** as part of each VdbeSorter object. Instances are never allocated any other -** way. VdbeSorter.nTask is set to the number of worker threads allowed -** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). -** -** When a background thread is launched to perform work, SortSubtask.bDone -** is set to 0 and the SortSubtask.pTask variable set to point to the -** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main -** thread that joining SortSubtask.pTask will not block) before the thread -** exits. SortSubtask.pTask and bDone are always cleared after the -** background thread has been joined. -** -** One object (specifically, VdbeSorter.aTask[VdbeSorter.nTask-1]) -** is reserved for the foreground thread. -** -** The nature of the work performed is determined by SortSubtask.eWork, -** as follows: -** -** SORT_SUBTASK_SORT: -** Sort the linked list of records at SortSubtask.pList. -** -** SORT_SUBTASK_TO_PMA: -** Sort the linked list of records at SortSubtask.pList, and write -** the results to a new PMA in temp file SortSubtask.pTemp1. Open -** the temp file if it is not already open. -** -** SORT_SUBTASK_CONS: -** Merge existing PMAs until SortSubtask.nConsolidate or fewer -** remain in temp file SortSubtask.pTemp1. -*/ -struct SortSubtask { - SorterThread thread; - sqlite3 *db; /* Database connection */ - VdbeSorter *pSorter; /* Sorter */ - KeyInfo *pKeyInfo; /* How to compare records */ - UnpackedRecord *pUnpacked; /* Space to unpack a record */ - int pgsz; /* Main database page size */ - SorterList list; /* List for thread to write to a PMA */ - int nPMA; /* Number of PMAs currently in file */ - SorterFile file; /* Temp file for level-0 PMAs */ - SorterFile file2; /* Space for other PMAs */ -}; - - /* ** The MergeEngine object is used to combine two or more smaller PMAs into ** one big PMA using a merge operation. Separate PMAs all need to be @@ -268,6 +247,45 @@ struct MergeEngine { PmaReader *aIter; /* Array of iterators to merge data from */ }; +/* +** Exactly VdbeSorter.nTask instances of this object are allocated +** as part of each VdbeSorter object. Instances are never allocated any +** other way. VdbeSorter.nTask is set to the number of worker threads allowed +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). +** +** Essentially, this structure contains all those fields of the VdbeSorter +** structure for which each thread requires a separate instance. For example, +** each thread requries its own UnpackedRecord object to unpack records in +** as part of comparison operations. +** +** Before a background thread is launched, variable bDone is set to 0. Then, +** right before it exits, the thread itself sets bDone to 1. This is used for +** two purposes: +** +** 1. When flushing the contents of memory to a level-0 PMA on disk, to +** attempt to select a SortSubtask for which there is not already an +** active background thread (since doing so causes the main thread +** to block until it finishes). +** +** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call +** to sqlite3ThreadJoin() is likely to block. Cases that are likely to +** block provoke debugging output. +** +** In both cases, the effects of the main thread seeing (bDone==0) even +** after the thread has finished are not dire. So we don't worry about +** memory barriers and such here. +*/ +struct SortSubtask { + SQLiteThread *pThread; /* Background thread, if any */ + int bDone; /* Set if thread is finished but not joined */ + VdbeSorter *pSorter; /* Sorter that owns this sub-task */ + UnpackedRecord *pUnpacked; /* Space to unpack a record */ + SorterList list; /* List for thread to write to a PMA */ + int nPMA; /* Number of PMAs currently in file */ + SorterFile file; /* Temp file for level-0 PMAs */ + SorterFile file2; /* Space for other PMAs */ +}; + /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. @@ -280,9 +298,12 @@ struct MergeEngine { struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ + int mxKeysize; /* Largest serialized key seen so far */ + int pgsz; /* Main database page size */ PmaReader *pReader; /* Read data from here after Rewind() */ MergeEngine *pMerger; /* Or here, if bUseThreads==0 */ - int mxKeysize; /* Largest serialized key seen so far */ + sqlite3 *db; /* Database connection */ + KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ SorterList list; /* List of in-memory records */ int iMemory; /* Offset of free space in list.aMemory */ @@ -318,10 +339,35 @@ struct PmaReader { ** within a temp file. However, if the PmaReader.pIncr variable points to ** an object of the following type, it may be used to iterate/merge through ** multiple PMAs simultaneously. +** +** There are two types of IncrMerger object - single (bUseThread==0) and +** multi-threaded (bUseThread==1). +** +** A multi-threaded IncrMerger object uses two temporary files - aFile[0] +** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in +** size. When the IncrMerger is initialized, it reads enough data from +** pMerger to populate aFile[0]. It then sets variables within the +** corresponding PmaReader object to read from that file and kicks off +** a background thread to populate aFile[1] with the next mxSz bytes of +** sorted record data from pMerger. +** +** When the PmaReader reaches the end of aFile[0], it blocks until the +** background thread has finished populating aFile[1]. It then exchanges +** the contents of the aFile[0] and aFile[1] variables within this structure, +** sets the PmaReader fields to read from the new aFile[0] and kicks off +** another background thread to populate the new aFile[1]. And so on, until +** the contents of pMerger are exhausted. +** +** A single-threaded IncrMerger does not open any temporary files of its +** own. Instead, it has exclusive access to mxSz bytes of space beginning +** at offset iStartOff of file pTask->file2. And instead of using a +** background thread to prepare data for the PmaReader, with a single +** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with +** keys from pMerger by the calling thread whenever the PmaReader runs out +** of data. */ struct IncrMerger { SortSubtask *pTask; /* Task that owns this merger */ - SorterThread thread; /* Thread for populating aFile[1] */ MergeEngine *pMerger; /* Merge engine thread reads data from */ i64 iStartOff; /* Offset to start writing file at */ int mxSz; /* Maximum bytes of data to store */ @@ -354,10 +400,10 @@ struct PmaWriter { ** ** How the linked list is connected depends on how memory is being managed ** by this module. If using a separate allocation for each in-memory record -** (VdbeSorter.aMemory==0), then the list is always connected using the +** (VdbeSorter.list.aMemory==0), then the list is always connected using the ** SorterRecord.u.pNext pointers. ** -** Or, if using the single large allocation method (VdbeSorter.aMemory!=0), +** Or, if using the single large allocation method (VdbeSorter.list.aMemory!=0), ** then while records are being accumulated the list is linked using the ** SorterRecord.u.iNext offset. This is because the aMemory[] array may ** be sqlite3Realloc()ed while records are being accumulated. Once the VM @@ -390,7 +436,7 @@ struct SorterRecord { #define SORTER_MAX_MERGE_COUNT 16 static int vdbeIncrSwap(IncrMerger*); -static void vdbeIncrFree(IncrMerger*); +static void vdbeIncrFree(IncrMerger *); /* ** Free all memory belonging to the PmaReader object passed as the second @@ -531,56 +577,69 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ return SQLITE_OK; } +/* +** Attempt to memory map file pFile. If successful, set *pp to point to the +** new mapping and return SQLITE_OK. If the mapping is not attempted +** (because the file is too large or the VFS layer is configured not to use +** mmap), return SQLITE_OK and set *pp to NULL. +** +** Or, if an error occurs, return an SQLite error code. The final value of +** *pp is undefined in this case. +*/ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; - if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); } return rc; } -static int vdbePmaReaderReinit(PmaReader *pIter){ - IncrMerger *pIncr = pIter->pIncr; - SortSubtask *pTask = pIncr->pTask; +/* +** Seek iterator pIter to offset iOff within file pFile. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static int vdbePmaReaderSeek( + SortSubtask *pTask, /* Task context */ + PmaReader *pIter, /* Iterate to populate */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iOff /* Offset in pFile */ +){ int rc = SQLITE_OK; - assert( pIncr->bEof==0 ); + assert( pIter->pIncr==0 || pIter->pIncr->bEof==0 ); if( pIter->aMap ){ sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); pIter->aMap = 0; } - pIter->iReadOff = pIncr->iStartOff; - pIter->iEof = pIncr->aFile[0].iEof; - pIter->pFile = pIncr->aFile[0].pFd; + pIter->iReadOff = iOff; + pIter->iEof = pFile->iEof; + pIter->pFile = pFile->pFd; - rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); - if( rc==SQLITE_OK ){ - if( pIter->aMap==0 ){ - /* TODO: Combine this code with similar code in vdbePmaReaderInit() */ - int iBuf = pIter->iReadOff % pTask->pgsz; - if( pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pTask->pgsz; - } - if( iBuf ){ - int nRead = pTask->pgsz - iBuf; - if( (pIter->iReadOff + nRead) > pIter->iEof ){ - nRead = (int)(pIter->iEof - pIter->iReadOff); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); + rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); + if( rc==SQLITE_OK && pIter->aMap==0 ){ + int pgsz = pTask->pSorter->pgsz; + int iBuf = pIter->iReadOff % pgsz; + if( pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pgsz; + } + if( iBuf ){ + int nRead = pgsz - iBuf; + if( (pIter->iReadOff + nRead) > pIter->iEof ){ + nRead = (int)(pIter->iEof - pIter->iReadOff); } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } } return rc; } - /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. @@ -590,11 +649,14 @@ static int vdbePmaReaderNext(PmaReader *pIter){ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ + IncrMerger *pIncr = pIter->pIncr; int bEof = 1; - if( pIter->pIncr ){ - rc = vdbeIncrSwap(pIter->pIncr); - if( rc==SQLITE_OK && pIter->pIncr->bEof==0 ){ - rc = vdbePmaReaderReinit(pIter); + if( pIncr ){ + rc = vdbeIncrSwap(pIncr); + if( rc==SQLITE_OK && pIncr->bEof==0 ){ + rc = vdbePmaReaderSeek( + pIncr->pTask, pIter, &pIncr->aFile[0], pIncr->iStartOff + ); bEof = 0; } } @@ -633,47 +695,16 @@ static int vdbePmaReaderInit( PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ - int rc = SQLITE_OK; - int nBuf = pTask->pgsz; + int rc; assert( pFile->iEof>iStart ); - assert( pIter->aAlloc==0 ); + assert( pIter->aAlloc==0 && pIter->nAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pFile->pFd; - pIter->iReadOff = iStart; - pIter->nAlloc = 128; - pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); - if( pIter->aAlloc ){ - /* Try to xFetch() a mapping of the entire temp file. If this is possible, - ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); - }else{ - rc = SQLITE_NOMEM; - } - - if( rc==SQLITE_OK && pIter->aMap==0 ){ - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pFile->iEof ){ - nRead = (int)(pFile->iEof - iStart); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); - } - } - } + assert( pIter->aMap==0 ); + rc = vdbePmaReaderSeek(pTask, pIter, pFile, iStart); if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pFile->iEof; rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; @@ -706,7 +737,7 @@ static int vdbeSorterCompare( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( pKey2 ){ - sqlite3VdbeRecordUnpack(pTask->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } @@ -788,19 +819,16 @@ int sqlite3VdbeSorterInit( if( pSorter==0 ){ rc = SQLITE_NOMEM; }else{ - pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); + pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; if( nField && nWorker==0 ) pKeyInfo->nField = nField; - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; pSorter->bUseThreads = (pSorter->nTask>1); + pSorter->db = db; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - pTask->pKeyInfo = pKeyInfo; - pTask->pgsz = pgsz; - pTask->db = db; pTask->pSorter = pSorter; } @@ -904,22 +932,22 @@ static void vdbeSorterBlockDebug( #if SQLITE_MAX_WORKER_THREADS>0 /* -** Join thread p. +** Join thread pTask->thread. */ -static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ +static int vdbeSorterJoinThread(SortSubtask *pTask){ int rc = SQLITE_OK; - if( p->pThread ){ + if( pTask->pThread ){ #ifdef SQLITE_DEBUG_SORTER_THREADS - int bDone = p->bDone; + int bDone = pTask->bDone; #endif void *pRet; vdbeSorterBlockDebug(pTask, !bDone, "enter"); - rc = sqlite3ThreadJoin(p->pThread, &pRet); + rc = sqlite3ThreadJoin(pTask->pThread, &pRet); vdbeSorterBlockDebug(pTask, !bDone, "exit"); if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - assert( p->bDone==1 ); - p->bDone = 0; - p->pThread = 0; + assert( pTask->bDone==1 ); + pTask->bDone = 0; + pTask->pThread = 0; } return rc; } @@ -928,12 +956,12 @@ static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ ** Launch a background thread to run xTask(pIn). */ static int vdbeSorterCreateThread( - SorterThread *p, /* Thread object to populate */ + SortSubtask *pTask, /* Thread will use this task object */ void *(*xTask)(void*), /* Routine to run in a separate thread */ void *pIn /* Argument passed into xTask() */ ){ - assert( p->pThread==0 && p->bDone==0 ); - return sqlite3ThreadCreate(&p->pThread, xTask, pIn); + assert( pTask->pThread==0 && pTask->bDone==0 ); + return sqlite3ThreadCreate(&pTask->pThread, xTask, pIn); } /* @@ -945,14 +973,14 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - int rc2 = vdbeSorterJoinThread(pTask, &pTask->thread); + int rc2 = vdbeSorterJoinThread(pTask); if( rc==SQLITE_OK ) rc = rc2; } return rc; } #else # define vdbeSorterJoinAll(x,rcin) (rcin) -# define vdbeSorterJoinThread(pTask,p) SQLITE_OK +# define vdbeSorterJoinThread(pTask) SQLITE_OK #endif /* @@ -990,6 +1018,24 @@ static void vdbeMergeEngineFree(MergeEngine *pMerger){ sqlite3_free(pMerger); } +/* +** Free all resources associated with the IncrMerger object indicated by +** the first argument. +*/ +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + vdbeSorterJoinThread(pIncr->pTask); + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); + } +} + /* ** Reset a sorting cursor back to its original empty state. */ @@ -1051,15 +1097,20 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ return rc; } +/* +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** if no allocation was required), or SQLITE_NOMEM otherwise. +*/ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ if( pTask->pUnpacked==0 ){ char *pFree; pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree + pTask->pSorter->pKeyInfo, 0, 0, &pFree ); assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ) return SQLITE_NOMEM; - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; pTask->pUnpacked->errCode = 0; } return SQLITE_OK; @@ -1280,6 +1331,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ ** key). The varint is the number of bytes in the blob of data. */ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ + sqlite3 *db = pTask->pSorter->db; int rc = SQLITE_OK; /* Return code */ PmaWriter writer; /* Object used to write to the file */ @@ -1295,7 +1347,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file.pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file.pFd); assert( rc!=SQLITE_OK || pTask->file.pFd ); assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); @@ -1303,9 +1355,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, - pTask->file.pFd, pTask->file.iEof + pList->szPMA + 9 - ); + vdbeSorterExtendFile(db, pTask->file.pFd, pTask->file.iEof+pList->szPMA+9); } /* Sort the list */ @@ -1317,7 +1367,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ SorterRecord *p; SorterRecord *pNext = 0; - vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pSorter->pgsz, pTask->file.iEof); pTask->nPMA++; vdbePmaWriteVarint(&writer, pList->szPMA); @@ -1413,14 +1463,14 @@ static int vdbeSorterNext( } /* -** The main routine for sorter-thread operations. +** The main routine for background threads that write level-0 PMAs. */ static void *vdbeSorterFlushThread(void *pCtx){ SortSubtask *pTask = (SortSubtask*)pCtx; int rc; /* Return code */ - assert( pTask->thread.bDone==0 ); + assert( pTask->bDone==0 ); rc = vdbeSorterListToPMA(pTask, &pTask->list); - pTask->thread.bDone = 1; + pTask->bDone = 1; return SQLITE_INT_TO_PTR(rc); } @@ -1453,10 +1503,10 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ for(i=0; iiPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; - if( pTask->thread.bDone ){ - rc = vdbeSorterJoinThread(pTask, &pTask->thread); + if( pTask->bDone ){ + rc = vdbeSorterJoinThread(pTask); } - if( pTask->thread.pThread==0 || rc!=SQLITE_OK ) break; + if( pTask->pThread==0 || rc!=SQLITE_OK ) break; } if( rc==SQLITE_OK ){ @@ -1468,7 +1518,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ u8 *aMem = pTask->list.aMemory; void *pCtx = (void*)pTask; - assert( pTask->thread.pThread==0 && pTask->thread.bDone==0 ); + assert( pTask->pThread==0 && pTask->bDone==0 ); assert( pTask->list.pList==0 ); assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); @@ -1484,7 +1534,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } - rc = vdbeSorterCreateThread(&pTask->thread, vdbeSorterFlushThread, pCtx); + rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); } } @@ -1597,13 +1647,14 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ int rc2; i64 iStart = pIncr->iStartOff; SorterFile *pOut = &pIncr->aFile[1]; + SortSubtask *pTask = pIncr->pTask; MergeEngine *pMerger = pIncr->pMerger; PmaWriter writer; assert( pIncr->bEof==0 ); - vdbeSorterPopulateDebug(pIncr->pTask, "enter"); + vdbeSorterPopulateDebug(pTask, "enter"); - vdbePmaWriterInit(pOut->pFd, &writer, pIncr->pTask->pgsz, iStart); + vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; @@ -1618,36 +1669,60 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeSorterNext(pIncr->pTask, pIncr->pMerger, &dummy); + rc = vdbeSorterNext(pTask, pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); if( rc==SQLITE_OK ) rc = rc2; - vdbeSorterPopulateDebug(pIncr->pTask, "exit"); + vdbeSorterPopulateDebug(pTask, "exit"); return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that populate aFile[1] of +** multi-threaded IncrMerger objects. +*/ static void *vdbeIncrPopulateThread(void *pCtx){ IncrMerger *pIncr = (IncrMerger*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); - pIncr->thread.bDone = 1; + pIncr->pTask->bDone = 1; return pRet; } -#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Launch a background thread to populate aFile[1] of pIncr. +*/ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ - void *pCtx = (void*)pIncr; + void *p = (void*)pIncr; assert( pIncr->bUseThread ); - return vdbeSorterCreateThread(&pIncr->thread, vdbeIncrPopulateThread, pCtx); + return vdbeSorterCreateThread(pIncr->pTask, vdbeIncrPopulateThread, p); } #endif +/* +** This function is called when the PmaReader corresponding to pIncr has +** finished reading the contents of aFile[0]. Its purpose is to "refill" +** aFile[0] such that the iterator should start rereading it from the +** beginning. +** +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. +** +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap +** aFile[0] and aFile[1] in place. If the contents of pMerger have not +** been exhausted, this function also launches a new background thread +** to populate the new aFile[1]. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. +*/ static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS>0 if( pIncr->bUseThread ){ - rc = vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); + rc = vdbeSorterJoinThread(pIncr->pTask); if( rc==SQLITE_OK ){ SorterFile f0 = pIncr->aFile[0]; @@ -1675,20 +1750,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ return rc; } -static void vdbeIncrFree(IncrMerger *pIncr){ - if( pIncr ){ -#if SQLITE_MAX_WORKER_THREADS>0 - vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); - if( pIncr->bUseThread ){ - if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); - if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); - } -#endif - vdbeMergeEngineFree(pIncr->pMerger); - sqlite3_free(pIncr); - } -} - +/* +** Allocate and return a new IncrMerger object to read data from pMerger. +*/ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); if( pIncr ){ @@ -1701,6 +1765,9 @@ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ return pIncr; } +/* +** Set the "use-threads" flag on object pIncr. +*/ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ if( bUseThread ){ pIncr->bUseThread = 1; @@ -1742,6 +1809,7 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); @@ -1749,10 +1817,10 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ if( rc==SQLITE_OK ){ if( pIncr->bUseThread==0 ){ if( pTask->file2.pFd==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file2.pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file2.pFd); assert( pTask->file2.iEof>0 ); if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db,pTask->file2.pFd,pTask->file2.iEof); + vdbeSorterExtendFile(db, pTask->file2.pFd, pTask->file2.iEof); pTask->file2.iEof = 0; } } @@ -1762,9 +1830,9 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ pTask->file2.iEof += pIncr->mxSz; } }else{ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[0].pFd); if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[1].pFd); } } } @@ -1786,15 +1854,13 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); - pReader->pIncr->thread.bDone = 1; + pReader->pIncr->pTask->bDone = 1; return pRet; } static int vdbeIncrBgInit2(PmaReader *pIter){ void *pCtx = (void*)pIter; - return vdbeSorterCreateThread( - &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx - ); + return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); } #endif @@ -1885,7 +1951,7 @@ static int vdbeAddToBuilder( static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; - sqlite3 *db = pTask0->db; + sqlite3 *db = pTask0->pSorter->db; int rc = SQLITE_OK; int iTask; From 22ace891799e7c7ff4159acfc7c3c4481cc9a2a3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 Apr 2014 20:52:27 +0000 Subject: [PATCH 43/99] Fix some problems to do with OOM conditions in vdbesort.c. Some problems remain. FossilOrigin-Name: 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/vdbesort.c | 16 +++++++++++++++- test/sortfault.test | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 test/sortfault.test diff --git a/manifest b/manifest index 0ca4cbc345..234641dfb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfurther\scode\sand\sdocumentation\sissues\sin\svdbesort.c. -D 2014-04-15T19:52:34.797 +C Fix\ssome\sproblems\sto\sdo\swith\sOOM\sconditions\sin\svdbesort.c.\sSome\sproblems\sremain. +D 2014-04-15T20:52:27.270 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c ceb8e16055327d0c52bdd2087fcdd7d132fe314f +F src/vdbesort.c 56c663772aaa7cd790ba2eb8f3bd3b4a4ac1fa4d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -820,6 +820,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af +F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1163,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cb0ab20c48962cdee03115efa93d7d501780ac73 -R 44c59b06ba4f012b63c2d0e26d30a7e6 +P d03f5b8622d304f029f73c7cd0bee3182a81d081 +R a54791a581239e379f5a047014cbfa51 U dan -Z 2cc8bdff8b0228a6efe530348d9564c8 +Z b9d6664d305a1dc63db907b778150c9a diff --git a/manifest.uuid b/manifest.uuid index b7e5cbc645..aa04dbee2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d03f5b8622d304f029f73c7cd0bee3182a81d081 \ No newline at end of file +2f94f9ce9bf11f1599bbc640b3fc8c15da588416 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8b6cce6259..0922ea88b0 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -625,7 +625,7 @@ static int vdbePmaReaderSeek( if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; pIter->nBuffer = pgsz; } - if( iBuf ){ + if( rc==SQLITE_OK && iBuf ){ int nRead = pgsz - iBuf; if( (pIter->iReadOff + nRead) > pIter->iEof ){ nRead = (int)(pIter->iEof - pIter->iReadOff); @@ -1990,6 +1990,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } } rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); + if( rc!=SQLITE_OK ) break; } } @@ -2017,6 +2018,10 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } memset(aMerge, 0, nMerge*sizeof(aMerge[0])); } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pRoot); + } } if( rc==SQLITE_OK ){ @@ -2050,6 +2055,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } } } + pMain = 0; } } if( rc==SQLITE_OK ){ @@ -2061,9 +2067,17 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ { pSorter->pMerger = pMain; rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + pMain = 0; } } + if( rc!=SQLITE_OK ){ + int i; + for(i=0; rc==SQLITE_OK && i Date: Wed, 16 Apr 2014 16:43:05 +0000 Subject: [PATCH 44/99] Rework the way trees of MergeEngine objects are built in vdbesort.c to make it easier to follow. Fix memory leaks that could follow an OOM or IO error. Add various comments to explain functions in vdbesort.c. FossilOrigin-Name: 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 --- manifest | 12 +-- manifest.uuid | 2 +- src/vdbesort.c | 288 ++++++++++++++++++++++++++++++------------------- 3 files changed, 184 insertions(+), 118 deletions(-) diff --git a/manifest b/manifest index 234641dfb4..0981a1907c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\sto\sdo\swith\sOOM\sconditions\sin\svdbesort.c.\sSome\sproblems\sremain. -D 2014-04-15T20:52:27.270 +C Rework\sthe\sway\strees\sof\sMergeEngine\sobjects\sare\sbuilt\sin\svdbesort.c\sto\smake\sit\seasier\sto\sfollow.\sFix\smemory\sleaks\sthat\scould\sfollow\san\sOOM\sor\sIO\serror.\sAdd\svarious\scomments\sto\sexplain\sfunctions\sin\svdbesort.c. +D 2014-04-16T16:43:05.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 56c663772aaa7cd790ba2eb8f3bd3b4a4ac1fa4d +F src/vdbesort.c 30505a846dd55e7ff62bccad455d465342487887 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d03f5b8622d304f029f73c7cd0bee3182a81d081 -R a54791a581239e379f5a047014cbfa51 +P 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 +R 826dd10e19d54d131d0af65a48be33dc U dan -Z b9d6664d305a1dc63db907b778150c9a +Z b3fd9ee2f5beb1e7affcfabe59de18f9 diff --git a/manifest.uuid b/manifest.uuid index aa04dbee2d..ef89cf487e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f94f9ce9bf11f1599bbc640b3fc8c15da588416 \ No newline at end of file +69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 0922ea88b0..4f03643007 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1752,17 +1752,28 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ /* ** Allocate and return a new IncrMerger object to read data from pMerger. +** +** If an OOM condition is encountered, return NULL. In this case free the +** pMerger argument before returning. */ -static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ - IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); +static int vdbeIncrNew( + SortSubtask *pTask, + MergeEngine *pMerger, + IncrMerger **ppOut +){ + int rc = SQLITE_OK; + IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); if( pIncr ){ memset(pIncr, 0, sizeof(IncrMerger)); pIncr->pMerger = pMerger; pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); pTask->file2.iEof += pIncr->mxSz; + }else{ + vdbeMergeEngineFree(pMerger); + rc = SQLITE_NOMEM; } - return pIncr; + return rc; } /* @@ -1778,16 +1789,32 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 - static int vdbeIncrInit2(PmaReader *pIter, int eMode); +/* +** Initialize the merger argument passed as the second argument. Once this +** function returns, the first key of merged data may be read from the merger +** object in the usual fashion. +** +** If argument eMode is INCRINIT2_ROOT, then it is assumed that any IncrMerge +** objects attached to the PmaReader objects that the merger reads from have +** already been populated, but that they have not yet populated aFile[0] and +** set the PmaReader objects up to read from it. In this case all that is +** required is to call vdbePmaReaderNext() on each iterator to point it at +** its first key. +** +** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use +** vdbeIncrInit2() to initialize each PmaReader that feeds data to pMerger. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ static int vdbeIncrInitMerger( SortSubtask *pTask, MergeEngine *pMerger, - int eMode + int eMode /* One of the INCRINIT2_XXX constants */ ){ - int i; - int rc = SQLITE_OK; + int rc = SQLITE_OK; /* Return code */ + int i; /* For iterating through PmaReader objects */ for(i=0; rc==SQLITE_OK && inTree; i++){ if( eMode==INCRINIT2_ROOT ){ @@ -1851,6 +1878,9 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ } #if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for vdbeIncrInit2() operations run in background threads. +*/ static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); @@ -1858,6 +1888,14 @@ static void *vdbeIncrInit2Thread(void *pCtx){ return pRet; } +/* +** Use a background thread to invoke vdbeIncrInit2(INCRINIT2_TASK) on the +** the PmaReader object passed as the first argument. +** +** This call will initialize the various fields of the pIter->pIncr +** structure and, if it is a multi-threaded IncrMerger, launch a +** background thread to populate aFile[1]. +*/ static int vdbeIncrBgInit2(PmaReader *pIter){ void *pCtx = (void*)pIter; return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); @@ -1905,42 +1943,146 @@ static int vdbeMergeEngineLevel0( return rc; } -typedef struct IncrBuilder IncrBuilder; -struct IncrBuilder { - int nPMA; /* Number of iterators used so far */ - MergeEngine *pMerger; /* Merge engine to populate. */ -}; +/* +** Return the depth of a tree comprising nPMA PMAs, assuming a fanout of +** SORTER_MAX_MERGE_COUNT. The returned value does not include leaf nodes. +** +** i.e. +** +** nPMA<=16 -> TreeDepth() == 0 +** nPMA<=256 -> TreeDepth() == 1 +** nPMA<=65536 -> TreeDepth() == 2 +*/ +static int vdbeSorterTreeDepth(int nPMA){ + int nDepth = 0; + i64 nDiv = SORTER_MAX_MERGE_COUNT; + while( nDiv < (i64)nPMA ){ + nDiv = nDiv * SORTER_MAX_MERGE_COUNT; + nDepth++; + } + return nDepth; +} -static int vdbeAddToBuilder( - SortSubtask *pTask, - IncrBuilder *pBuilder, - MergeEngine *pMerger +/* +** pRoot is the root of an incremental merge-tree with depth nDepth (according +** to vdbeSorterTreeDepth()). pLeaf is the iSeq'th leaf to be added to the +** tree, counting from zero. This function adds pLeaf to the tree. +** +** If successful, SQLITE_OK is returned. If an error occurs, an SQLite error +** code is returned and pLeaf is freed. +*/ +static int vdbeSorterAddToTree( + SortSubtask *pTask, /* Task context */ + int nDepth, /* Depth of tree according to TreeDepth() */ + int iSeq, /* Sequence number of leaf within tree */ + MergeEngine *pRoot, /* Root of tree */ + MergeEngine *pLeaf /* Leaf to add to tree */ ){ int rc = SQLITE_OK; + int nDiv = 1; + int i; + MergeEngine *p = pRoot; IncrMerger *pIncr; - assert( pMerger ); - if( pBuilder->nPMA==SORTER_MAX_MERGE_COUNT ){ - rc = vdbeAddToBuilder(pTask, &pBuilder[1], pBuilder->pMerger); - pBuilder->pMerger = 0; - pBuilder->nPMA = 0; + rc = vdbeIncrNew(pTask, pLeaf, &pIncr); + + for(i=1; ipMerger==0 ){ - pBuilder->pMerger = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); - if( pBuilder->pMerger==0 ) rc = SQLITE_NOMEM; + for(i=1; iaIter[iIter]; + + if( pIter->pIncr==0 ){ + MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); + } + } + + p = pIter->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; } if( rc==SQLITE_OK ){ - pIncr = vdbeIncrNew(pTask, pMerger); - if( pIncr==0 ) rc = SQLITE_NOMEM; - pBuilder->pMerger->aIter[pBuilder->nPMA++].pIncr = pIncr; + p->aIter[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + }else{ + vdbeIncrFree(pIncr); + } + return rc; +} + +/* +** This function is called as part of a SorterRewind() operation on a sorter +** that has already written two or more level-0 PMAs to one or more temp +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** can be used to incrementally merge all PMAs on disk. +** +** If successful, SQLITE_OK is returned and *ppOut set to point to the +** MergeEngine object at the root of the tree before returning. Or, if an +** error occurs, an SQLite error code is returned and the final value +** of *ppOut is undefined. +*/ +static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ + MergeEngine *pMain = 0; + int rc = SQLITE_OK; + int iTask; + + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly + ** one PmaReader per sub-task. */ + assert( pSorter->bUseThreads || pSorter->nTask==1 ); + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM; + } + + for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + SortSubtask *pTask = &pSorter->aTask[iTask]; + if( pTask->nPMA ){ + MergeEngine *pRoot = 0; /* Root node of tree for this task */ + int nDepth = vdbeSorterTreeDepth(pTask->nPMA); + i64 iReadOff = 0; + + if( pTask->nPMA<=SORTER_MAX_MERGE_COUNT ){ + rc = vdbeMergeEngineLevel0(pTask, pTask->nPMA, &iReadOff, &pRoot); + }else{ + int i; + int iSeq = 0; + pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pRoot==0 ) rc = SQLITE_NOMEM; + for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; /* New level-0 PMA merger */ + int nReader; /* Number of level-0 PMAs to merge */ + + nReader = MIN(pTask->nPMA - i, SORTER_MAX_MERGE_COUNT); + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc==SQLITE_OK ){ + rc = vdbeSorterAddToTree(pTask, nDepth, iSeq++, pRoot, pMerger); + } + } + } + + if( rc==SQLITE_OK ){ + if( pMain==0 ){ + pMain = pRoot; + }else{ + rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + } + }else{ + vdbeMergeEngineFree(pRoot); + } + } } if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pMerger); + vdbeMergeEngineFree(pMain); + pMain = 0; } - + *ppOut = pMain; return rc; } @@ -1949,81 +2091,13 @@ static int vdbeAddToBuilder( ** keys stored in all PMAs created by this sorter. */ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ + int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; sqlite3 *db = pTask0->pSorter->db; - int rc = SQLITE_OK; int iTask; - IncrBuilder *aMerge; - const int nMerge = 32; - aMerge = sqlite3DbMallocZero(db, sizeof(aMerge[0])*nMerge); - if( aMerge==0 ) return SQLITE_NOMEM; - - if( pSorter->nTask>1 ){ - pMain = vdbeMergeEngineNew(pSorter->nTask); - if( pMain==0 ) rc = SQLITE_NOMEM; - } - - for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ - MergeEngine *pRoot = 0; - int iPMA; - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[iTask]; - if( pTask->nPMA==0 ) continue; - for(iPMA=0; iPMAnPMA; iPMA += SORTER_MAX_MERGE_COUNT){ - MergeEngine *pMerger = 0; - int nReader = MIN(pTask->nPMA - iPMA, SORTER_MAX_MERGE_COUNT); - - rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); - if( rc!=SQLITE_OK ) break; - - if( iPMA==0 ){ - pRoot = pMerger; - }else{ - if( pRoot ){ - rc = vdbeAddToBuilder(pTask, &aMerge[0], pRoot); - pRoot = 0; - if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pMerger); - break; - } - } - rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); - if( rc!=SQLITE_OK ) break; - } - } - - if( pRoot==0 ){ - int i; - for(i=0; rc==SQLITE_OK && iaIter[iTask].pIncr = pNew; - if( pNew==0 ) rc = SQLITE_NOMEM; - } - memset(aMerge, 0, nMerge*sizeof(aMerge[0])); - } - - if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pRoot); - } - } - + rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS if( pSorter->bUseThreads ){ @@ -2035,11 +2109,8 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ pSorter->pReader = pIter; } if( rc==SQLITE_OK ){ - pIter->pIncr = vdbeIncrNew(pLast, pMain); - if( pIter->pIncr==0 ){ - rc = SQLITE_NOMEM; - } - else{ + rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); + if( rc==SQLITE_OK ){ vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; @@ -2055,8 +2126,8 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } } } - pMain = 0; } + pMain = 0; } if( rc==SQLITE_OK ){ int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); @@ -2065,20 +2136,15 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ }else #endif { - pSorter->pMerger = pMain; rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + pSorter->pMerger = pMain; pMain = 0; } } if( rc!=SQLITE_OK ){ - int i; - for(i=0; rc==SQLITE_OK && i Date: Wed, 16 Apr 2014 17:41:22 +0000 Subject: [PATCH 45/99] Change the name of vdbeIncrInit2 to vdbePmaReaderIncrInit. Add a header comment to the same function. FossilOrigin-Name: 6622d87675c1d7992b1e6d52ae7da1007a1568a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 41 ++++++++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 0981a1907c..b476a4c7f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sway\strees\sof\sMergeEngine\sobjects\sare\sbuilt\sin\svdbesort.c\sto\smake\sit\seasier\sto\sfollow.\sFix\smemory\sleaks\sthat\scould\sfollow\san\sOOM\sor\sIO\serror.\sAdd\svarious\scomments\sto\sexplain\sfunctions\sin\svdbesort.c. -D 2014-04-16T16:43:05.014 +C Change\sthe\sname\sof\svdbeIncrInit2\sto\svdbePmaReaderIncrInit.\sAdd\sa\sheader\scomment\sto\sthe\ssame\sfunction. +D 2014-04-16T17:41:22.128 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 30505a846dd55e7ff62bccad455d465342487887 +F src/vdbesort.c 70e42f93ebc6c4b08fff55f3f3bc6b25916cfeda F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 -R 826dd10e19d54d131d0af65a48be33dc +P 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 +R ed75157562bc112af42ea8e3070ab11f U dan -Z b3fd9ee2f5beb1e7affcfabe59de18f9 +Z ff87db58800fcd075476469fa2fb9f2f diff --git a/manifest.uuid b/manifest.uuid index ef89cf487e..52fd8bbfbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 \ No newline at end of file +6622d87675c1d7992b1e6d52ae7da1007a1568a4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 4f03643007..348eb447fd 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1789,7 +1789,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 -static int vdbeIncrInit2(PmaReader *pIter, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); /* ** Initialize the merger argument passed as the second argument. Once this @@ -1804,7 +1804,8 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode); ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use -** vdbeIncrInit2() to initialize each PmaReader that feeds data to pMerger. +** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data +** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ @@ -1820,7 +1821,7 @@ static int vdbeIncrInitMerger( if( eMode==INCRINIT2_ROOT ){ rc = vdbePmaReaderNext(&pMerger->aIter[i]); }else{ - rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT2_NORMAL); } } @@ -1831,7 +1832,7 @@ static int vdbeIncrInitMerger( return rc; } -static int vdbeIncrInit2(PmaReader *pIter, int eMode){ +static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ @@ -1879,26 +1880,27 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 /* -** The main routine for vdbeIncrInit2() operations run in background threads. +** The main routine for vdbePmaReaderIncrInit() operations run in +** background threads. */ -static void *vdbeIncrInit2Thread(void *pCtx){ +static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); + void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT2_TASK)); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbeIncrInit2(INCRINIT2_TASK) on the -** the PmaReader object passed as the first argument. +** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT2_TASK) +** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pIter->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a ** background thread to populate aFile[1]. */ -static int vdbeIncrBgInit2(PmaReader *pIter){ +static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ void *pCtx = (void*)pIter; - return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); + return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbePmaReaderBgInit, pCtx); } #endif @@ -2087,10 +2089,15 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } /* -** Populate iterator *pIter so that it may be used to iterate through all -** keys stored in all PMAs created by this sorter. +** This function is called as part of an sqlite3VdbeSorterRewind() operation +** on a sorter that has written two or more PMAs to temporary files. It sets +** up either VdbeSorter.pMerger (for single threaded sorters) or pReader +** (for multi-threaded sorters) so that it can be used to iterate through +** all records stored in the sorter. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ +static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; @@ -2123,7 +2130,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ PmaReader *p = &pMain->aIter[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } } } } @@ -2131,7 +2138,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } if( rc==SQLITE_OK ){ int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); - rc = vdbeIncrInit2(pIter, eMode); + rc = vdbePmaReaderIncrInit(pIter, eMode); } }else #endif @@ -2187,7 +2194,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** incrementally read and merge all remaining PMAs. */ assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pSorter); + rc = vdbeSorterSetupMerge(pSorter); *pbEof = 0; } From 31a0bfde9b6994c3040a7f87b29487532d094319 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 16 Apr 2014 19:04:23 +0000 Subject: [PATCH 46/99] Clarify the purpose of the nField argument passed to sqlite3VdbeSorterInit(). FossilOrigin-Name: c0c8cff17b7311bbc4fd081313a5552f927c3833 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b476a4c7f2..df788673d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\svdbeIncrInit2\sto\svdbePmaReaderIncrInit.\sAdd\sa\sheader\scomment\sto\sthe\ssame\sfunction. -D 2014-04-16T17:41:22.128 +C Clarify\sthe\spurpose\sof\sthe\snField\sargument\spassed\sto\ssqlite3VdbeSorterInit(). +D 2014-04-16T19:04:23.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 70e42f93ebc6c4b08fff55f3f3bc6b25916cfeda +F src/vdbesort.c fcd15eb438a3462f4dbeebf506ee9050947236bf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 -R ed75157562bc112af42ea8e3070ab11f +P 6622d87675c1d7992b1e6d52ae7da1007a1568a4 +R dd88eb57ef2cb3488d23de10e0a80225 U dan -Z ff87db58800fcd075476469fa2fb9f2f +Z 28b7dc83a03f9f7e8d724df0f79c29e1 diff --git a/manifest.uuid b/manifest.uuid index 52fd8bbfbc..8d63094336 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6622d87675c1d7992b1e6d52ae7da1007a1568a4 \ No newline at end of file +c0c8cff17b7311bbc4fd081313a5552f927c3833 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 348eb447fd..63f17e4e77 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -794,6 +794,22 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. +** +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** to determine the number of fields that should be compared from the +** records being sorted. However, if the value passed as argument nField +** is non-zero and the sorter is able to guarantee a stable sort, nField +** is used instead. This is used when sorting records for a CREATE INDEX +** statement. In this case, keys are always delivered to the sorter in +** order of the primary key, which happens to be make up the final part +** of the records being sorted. So if the sort is stable, there is never +** any reason to compare PK fields and they can be ignored for a small +** performance boost. +** +** The sorter can guarantee a stable sort when running in single-threaded +** mode, but not in multi-threaded mode. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ int sqlite3VdbeSorterInit( sqlite3 *db, /* Database connection (for malloc()) */ From a9f43d735b52276133151180a292891827158bda Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 17 Apr 2014 08:57:17 +0000 Subject: [PATCH 47/99] Fix build problems in vdbesort.c. Add further comments and changes to make things easier to understand. FossilOrigin-Name: 12b190db1d20c34390c768614b40ff58a9d3b74c --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 154 +++++++++++++++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index df788673d3..5fdab138c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\spurpose\sof\sthe\snField\sargument\spassed\sto\ssqlite3VdbeSorterInit(). -D 2014-04-16T19:04:23.830 +C Fix\sbuild\sproblems\sin\svdbesort.c.\sAdd\sfurther\scomments\sand\schanges\sto\smake\sthings\seasier\sto\sunderstand. +D 2014-04-17T08:57:17.925 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c fcd15eb438a3462f4dbeebf506ee9050947236bf +F src/vdbesort.c a41721a8e97735597afb80c930f04f195d86a817 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 6622d87675c1d7992b1e6d52ae7da1007a1568a4 -R dd88eb57ef2cb3488d23de10e0a80225 +P c0c8cff17b7311bbc4fd081313a5552f927c3833 +R 5c2193354b4ddf9cc33c16483e4c137f U dan -Z 28b7dc83a03f9f7e8d724df0f79c29e1 +Z c671751bbf58b5cb738173e8c3e25105 diff --git a/manifest.uuid b/manifest.uuid index 8d63094336..b5687b30fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0c8cff17b7311bbc4fd081313a5552f927c3833 \ No newline at end of file +12b190db1d20c34390c768614b40ff58a9d3b74c \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 63f17e4e77..ec9849bf16 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -911,7 +911,7 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ @@ -925,7 +925,7 @@ static void vdbeSorterPopulateDebug( ){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); } static void vdbeSorterBlockDebug( @@ -935,7 +935,7 @@ static void vdbeSorterBlockDebug( ){ if( bBlocked ){ i64 t; - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:main %s\n", t, zEvent); } } @@ -1094,21 +1094,52 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ } } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ + if( nByte<=(i64)(db->nMaxSorterMmap) ){ + int rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } + } +} +#else +# define vdbeSorterExtendFile(x,y,z) SQLITE_OK +#endif + /* ** Allocate space for a file-handle and open a temporary file. If successful, ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ +static int vdbeSorterOpenTempFile( + sqlite3 *db, /* Database handle doing sort */ + i64 nExtend, /* Attempt to extend file to this size */ + sqlite3_file **ppFile +){ int rc; - rc = sqlite3OsOpenMalloc(pVfs, 0, ppFile, + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); if( rc==SQLITE_OK ){ i64 max = SQLITE_MAX_MMAP_SIZE; - sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + sqlite3OsFileControlHint(*ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + if( nExtend>0 ){ + vdbeSorterExtendFile(db, *ppFile, nExtend); + } } return rc; } @@ -1307,31 +1338,6 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ vdbePmaWriteBlob(p, aByte, nByte); } -#if SQLITE_MAX_MMAP_SIZE>0 -/* -** The first argument is a file-handle open on a temporary file. The file -** is guaranteed to be nByte bytes or smaller in size. This function -** attempts to extend the file to nByte bytes in size and to ensure that -** the VFS has memory mapped it. -** -** Whether or not the file does end up memory mapped of course depends on -** the specific VFS implementation. -*/ -static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ - if( nByte<=(i64)(db->nMaxSorterMmap) ){ - int rc = sqlite3OsTruncate(pFile, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); - } - } -} -#else -# define vdbeSorterExtendFile(x,y,z) SQLITE_OK -#endif - - /* ** Write the current contents of in-memory linked-list pList to a level-0 ** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if @@ -1363,7 +1369,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file.pFd); + rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); assert( rc!=SQLITE_OK || pTask->file.pFd ); assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); @@ -1802,9 +1808,9 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ } } -#define INCRINIT2_NORMAL 0 -#define INCRINIT2_TASK 1 -#define INCRINIT2_ROOT 2 +#define INCRINIT_NORMAL 0 +#define INCRINIT_TASK 1 +#define INCRINIT_ROOT 2 static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); /* @@ -1812,14 +1818,14 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); ** function returns, the first key of merged data may be read from the merger ** object in the usual fashion. ** -** If argument eMode is INCRINIT2_ROOT, then it is assumed that any IncrMerge +** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge ** objects attached to the PmaReader objects that the merger reads from have ** already been populated, but that they have not yet populated aFile[0] and ** set the PmaReader objects up to read from it. In this case all that is ** required is to call vdbePmaReaderNext() on each iterator to point it at ** its first key. ** -** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use ** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data ** to pMerger. ** @@ -1828,16 +1834,16 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); static int vdbeIncrInitMerger( SortSubtask *pTask, MergeEngine *pMerger, - int eMode /* One of the INCRINIT2_XXX constants */ + int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ int i; /* For iterating through PmaReader objects */ for(i=0; rc==SQLITE_OK && inTree; i++){ - if( eMode==INCRINIT2_ROOT ){ + if( eMode==INCRINIT_ROOT ){ rc = vdbePmaReaderNext(&pMerger->aIter[i]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT2_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); } } @@ -1848,6 +1854,39 @@ static int vdbeIncrInitMerger( return rc; } +/* +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves +** to open and/or initialize the temp file related fields of the IncrMerge +** object at (pIter->pIncr). +** +** If argument eMode is set to INCRINIT_NORMAL, then PmaReader iterators +** in the sub-tree headed by pIter are also initialized. Data is then loaded +** into the buffers belonging to this iterator, pIter, and it is set to +** point to the first key in its range. +** +** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed +** to be a multi-threaded iterator and this function is being called in a +** background thread. In this case all iterators in the sub-tree are +** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to +** pIter is populated. However, the iterator itself is not set up to point +** to its first key. A call to vdbePmaReaderNext() is still required to do +** that. +** +** The reason this function does not call vdbePmaReaderNext() immediately +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that has +** to block on thread (pTask->thread) before accessing aFile[1]. But, since +** this entire function is being run by thread (pTask->thread), that will +** lead to the current background thread attempting to join itself. +** +** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed +** that pIter->pIncr is a multi-threaded IncrMerge objects, and that all +** child-trees have already been initialized using IncrInit(INCRINIT_TASK). +** In this case vdbePmaReaderNext() is called on all child iterators and +** the current iterator set to point to the first key in its range. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; @@ -1855,39 +1894,42 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; + assert( eMode==INCRINIT_NORMAL || pIncr->bUseThread==1 ); rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr */ + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; if( pIncr->bUseThread==0 ){ if( pTask->file2.pFd==0 ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file2.pFd); assert( pTask->file2.iEof>0 ); - if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(db, pTask->file2.pFd, pTask->file2.iEof); - pTask->file2.iEof = 0; - } + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; } if( rc==SQLITE_OK ){ pIncr->aFile[1].pFd = pTask->file2.pFd; pIncr->iStartOff = pTask->file2.iEof; - pTask->file2.iEof += pIncr->mxSz; + pTask->file2.iEof += mxSz; } }else{ - rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[0].pFd); + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); } } } if( rc==SQLITE_OK && pIncr->bUseThread ){ - /* Use the current thread */ - assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); + /* Use the current thread to populate aFile[1], even though this + ** iterator is multi-threaded. The reason being that this function + ** is already running in background thread pIncr->pTask->thread. */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); } - if( rc==SQLITE_OK && eMode!=INCRINIT2_TASK ){ + if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ rc = vdbePmaReaderNext(pIter); } } @@ -1901,13 +1943,13 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ */ static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT2_TASK)); + void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT_TASK)); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT2_TASK) +** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pIter->pIncr @@ -2153,13 +2195,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL); rc = vdbePmaReaderIncrInit(pIter, eMode); } }else #endif { - rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From 958d261bb8fa8b559c76e15f20c90c3623c1c55a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 13:40:07 +0000 Subject: [PATCH 48/99] Fix harmless compiler warnings. FossilOrigin-Name: f8f72ecb9052a4cace1db75879fb8b5131ea4f50 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 5 +++-- src/vdbeInt.h | 4 ++-- src/vdbesort.c | 29 +++++++++++++++++------------ 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ae525eafb2..58e22e9235 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-18T12:38:54.966 +C Fix\sharmless\scompiler\swarnings. +D 2014-04-18T13:40:07.511 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c cba6c791a3621de633cc018f09c81aa472072d42 +F src/vdbe.c b50cd3009a2e3067746c73dce36153f19df2e42e F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d +F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c a41721a8e97735597afb80c930f04f195d86a817 +F src/vdbesort.c f93c8aaff5398a702a7b49aae128031e050300b9 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 12b190db1d20c34390c768614b40ff58a9d3b74c 2c5363873a6f990a0abaacac6303acd46b48befc -R bb69be65b78f81eaebedcfbb221ccbdc +P 8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac +R 1501533958e541baa710100e336c4bc8 U drh -Z b1d3cff21f914d2f704f3c429938823e +Z 328a0fa311f69db138a2df8298545ace diff --git a/manifest.uuid b/manifest.uuid index 4eee3d389b..3ae7a2ee4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac \ No newline at end of file +f8f72ecb9052a4cace1db75879fb8b5131ea4f50 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c8b5d64477..fa34a0b7f3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4234,6 +4234,7 @@ case OP_SorterCompare: { assert( pOp->p4type==P4_INT32 ); pIn3 = &aMem[pOp->p3]; nIgnore = pOp->p4.i; + res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nIgnore, &res); VdbeBranchTaken(res!=0,2); if( res ){ @@ -4483,7 +4484,7 @@ case OP_Rewind: { /* jump */ assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ - rc = sqlite3VdbeSorterRewind(db, pC, &res); + rc = sqlite3VdbeSorterRewind(pC, &res); }else{ pCrsr = pC->pCursor; assert( pCrsr ); @@ -4642,7 +4643,7 @@ case OP_IdxInsert: { /* in2 */ rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ if( isSorter(pC) ){ - rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 03a303ca1b..97c49446b2 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -442,8 +442,8 @@ void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbesort.c b/src/vdbesort.c index ec9849bf16..6fc4e4e109 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -589,7 +589,7 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); + rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); } return rc; } @@ -914,9 +914,9 @@ static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } -static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ +static void vdbeSorterRewindDebug(const char *zEvent){ i64 t; - sqlite3OsCurrentTimeInt64(db->pVfs, &t); + sqlite3OsCurrentTimeInt64(sqlite3_vfs_find(0), &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } static void vdbeSorterPopulateDebug( @@ -941,7 +941,7 @@ static void vdbeSorterBlockDebug( } #else # define vdbeSorterWorkDebug(x,y) -# define vdbeSorterRewindDebug(x,y) +# define vdbeSorterRewindDebug(y) # define vdbeSorterPopulateDebug(x,y) # define vdbeSorterBlockDebug(x,y,z) #endif @@ -1109,7 +1109,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsFetch(pFile, 0, (int)nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } } @@ -1484,6 +1484,7 @@ static int vdbeSorterNext( return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 /* ** The main routine for background threads that write level-0 PMAs. */ @@ -1495,6 +1496,7 @@ static void *vdbeSorterFlushThread(void *pCtx){ pTask->bDone = 1; return SQLITE_INT_TO_PTR(rc); } +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ /* ** Flush the current contents of VdbeSorter.list to a new PMA, possibly @@ -1561,14 +1563,13 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ } return rc; -#endif +#endif /* SQLITE_MAX_WORKER_THREADS!=0 */ } /* ** Add a record to the sorter. */ int sqlite3VdbeSorterWrite( - sqlite3 *db, /* Database handle */ const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ @@ -1643,7 +1644,7 @@ int sqlite3VdbeSorterWrite( pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (u8*)(pSorter->list.pList) - pSorter->list.aMemory; + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ @@ -1798,6 +1799,7 @@ static int vdbeIncrNew( return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 /* ** Set the "use-threads" flag on object pIncr. */ @@ -1807,6 +1809,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ pIncr->pTask->file2.iEof -= pIncr->mxSz; } } +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 @@ -2159,13 +2162,15 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; +#if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; - int iTask; +#endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS if( pSorter->bUseThreads ){ + int iTask; PmaReader *pIter; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); @@ -2219,7 +2224,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ ** this function is called to prepare for iterating through the records ** in sorted order. */ -int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ @@ -2246,7 +2251,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); - vdbeSorterRewindDebug(db, "rewind"); + vdbeSorterRewindDebug("rewind"); /* Assuming no errors have occurred, set up a merger structure to ** incrementally read and merge all remaining PMAs. */ @@ -2256,7 +2261,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ *pbEof = 0; } - vdbeSorterRewindDebug(db, "rewinddone"); + vdbeSorterRewindDebug("rewinddone"); return rc; } From 46a06bbe47c043cbc034b4e94f7b08eefb6172be Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 13:57:39 +0000 Subject: [PATCH 49/99] Add to speedtest1.c the --threads option for setting the SQLITE_CONFIG_WORKER_THREADS configuration. FossilOrigin-Name: 5fce40c44aacf883df2e8e9472c399a6e92197b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 58e22e9235..644d369a1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-04-18T13:40:07.511 +C Add\sto\sspeedtest1.c\sthe\s--threads\soption\sfor\ssetting\sthe\nSQLITE_CONFIG_WORKER_THREADS\sconfiguration. +D 2014-04-18T13:57:39.195 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -829,7 +829,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 +F test/speedtest1.c bd150a4c29e26ff4ddbdd505e0a0d96347ffca51 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac -R 1501533958e541baa710100e336c4bc8 +P f8f72ecb9052a4cace1db75879fb8b5131ea4f50 +R ec8b910c95550d011d611a610b4db207 U drh -Z 328a0fa311f69db138a2df8298545ace +Z 39c3a2898ecaace6ee1641d24df0dd67 diff --git a/manifest.uuid b/manifest.uuid index 3ae7a2ee4a..f698cc6543 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8f72ecb9052a4cace1db75879fb8b5131ea4f50 \ No newline at end of file +5fce40c44aacf883df2e8e9472c399a6e92197b3 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 28c24e0885..7d38b37b0d 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -27,6 +27,7 @@ static const char zHelp[] = " --stats Show statistics at the end\n" " --testset T Run test-set T\n" " --trace Turn on SQL tracing\n" + " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" " --without-rowid Use WITHOUT ROWID where appropriate\n" @@ -962,6 +963,7 @@ int main(int argc, char **argv){ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int nScratch = 0, szScratch=0;/* --scratch configuration */ int showStats = 0; /* True for --stats */ + int nThread = 0; /* --threads value */ const char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ @@ -1046,6 +1048,9 @@ int main(int argc, char **argv){ zTSet = argv[++i]; }else if( strcmp(z,"trace")==0 ){ doTrace = 1; + }else if( strcmp(z,"threads")==0 ){ + if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + nThread = integerValue(argv[++i]); }else if( strcmp(z,"utf16le")==0 ){ zEncoding = "utf16le"; }else if( strcmp(z,"utf16be")==0 ){ @@ -1092,6 +1097,11 @@ int main(int argc, char **argv){ rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); if( rc ) fatal_error("scratch configuration failed: %d\n", rc); } +#ifdef SQLITE_CONFIG_WORKER_THREADS + if( nThread>0 ){ + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); + } +#endif if( nLook>0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } From 3de4df27dda707357d3fc62bf1c318a7a2e9dfeb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Apr 2014 12:28:28 +0000 Subject: [PATCH 50/99] Improvements to comments. Store some extra information in SqliteThread that is useful for debugging. FossilOrigin-Name: 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 10 +++++++--- src/vdbesort.c | 17 ++++++++++------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 2caa7ed483..b71dec8c60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-23T12:57:55.344 +C Improvements\sto\scomments.\s\sStore\ssome\sextra\sinformation\sin\sSqliteThread\sthat\nis\suseful\sfor\sdebugging. +D 2014-04-24T12:28:28.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b8e7232f2b9c9d148d6886117160de394d172f85 +F src/threads.c e35de159f9ced746266ff4b2129b7a828e59119c F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c f93c8aaff5398a702a7b49aae128031e050300b9 +F src/vdbesort.c 2b13026e5d4afa4737a312715aa1cd7e2f4d07c2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5fce40c44aacf883df2e8e9472c399a6e92197b3 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 07151f40a802d55a98a20acba34885fa +P e2c9f71a451e44040624b9f255b4510743513019 +R 85ab93c26e2f3dfc5f24d719f3aa0846 U drh -Z 85be1e117efe898356ded29b053b1ff8 +Z 26e90d38e785197a28b1e4097f1c9b04 diff --git a/manifest.uuid b/manifest.uuid index 3cda1ddaf8..6ee6407693 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2c9f71a451e44040624b9f255b4510743513019 \ No newline at end of file +9fb5e212089d85cdd3b4787dd69c72e6d84560b6 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index eeab5379ce..64975801be 100644 --- a/src/threads.c +++ b/src/threads.c @@ -37,9 +37,11 @@ /* A running thread */ struct SQLiteThread { - pthread_t tid; - int done; - void *pOut; + pthread_t tid; /* Thread ID */ + int done; /* Set to true when thread finishes */ + void *pOut; /* Result returned by the thread */ + void *(*xTask)(void*); /* The thread routine */ + void *pIn; /* Argument to the thread */ }; /* Create a new thread */ @@ -56,6 +58,8 @@ int sqlite3ThreadCreate( p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); + p->xTask = xTask; + p->pIn = pIn; if( sqlite3GlobalConfig.bCoreMutex==0 || pthread_create(&p->tid, 0, xTask, pIn)!=0 ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index 6fc4e4e109..46de735284 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -15,7 +15,7 @@ ** using indexes and without LIMIT clauses. ** ** The VdbeSorter object implements a multi-threaded external merge sort -** algorithm that is efficient even if the number of element being sorted +** algorithm that is efficient even if the number of elements being sorted ** exceeds the available memory. ** ** Here is the (internal, non-API) interface between this module and the @@ -104,9 +104,7 @@ ** ** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted -** PMAs within temporary files on disk. Within a single file sorter is -** running in single threaded mode, or distributed between one or more files -** for multi-threaded sorters. +** PMAs within temporary files on disk. ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged @@ -158,7 +156,7 @@ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ typedef struct SorterList SorterList; /* In-memory list of records */ -typedef struct IncrMerger IncrMerger; +typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ /* ** A container for a temp file handle and the current amount of data @@ -170,11 +168,16 @@ struct SorterFile { }; /* -** In memory linked list of records. +** An in-memory list of objects to be sorted. +** +** If aMemory==0 then each object is allocated separately and the objects +** are connected using SorterRecord.u.pNext. If aMemory!=0 then all objects +** are stored in the aMemory[] bulk memory, one right after the other, and +** are connected using SorterRecord.u.iNext. */ struct SorterList { SorterRecord *pList; /* Linked list of records */ - u8 *aMemory; /* If non-NULL, blob of memory for pList */ + u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ int szPMA; /* Size of pList as PMA in bytes */ }; From f690c142806a545dbea5c3ed8cc3680ef2cbd78b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Apr 2014 16:25:25 +0000 Subject: [PATCH 51/99] Improved header comment on the vdbesort.c module. No changes to code. FossilOrigin-Name: bf09ce24d054bc68c226064f5f28d97e0e648a13 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 56 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0187e05510..b7837cbb94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reopen\sthe\sorderby-planning\sbranch\sand\smerge\sin\sthe\slatest\strunk\senhancements\nand\sfixes. -D 2014-04-24T15:06:25.003 +C Improved\sheader\scomment\son\sthe\svdbesort.c\smodule.\s\sNo\schanges\sto\scode. +D 2014-04-24T16:25:25.812 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 +F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 58f7ca29303c280229f86d01f418e1de5f5aebba 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R b46cdf624d79e68eb1974021b4d3f992 +P 6077ddcd93318e24b9756adaaf293ba9fb3cedf7 +R f7111ea680230454a4775d5c337b77f9 U drh -Z d3c484ede684435a1175d26a414ba92a +Z b4087c0ba17f1174753bea8bd453086a diff --git a/manifest.uuid b/manifest.uuid index 4866c44be0..a672982fcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6077ddcd93318e24b9756adaaf293ba9fb3cedf7 \ No newline at end of file +bf09ce24d054bc68c226064f5f28d97e0e648a13 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 791a2465ca..6a34cefa61 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,9 +10,55 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a external merge sort +** algorithm that is efficient even if the aggregate size of +** the elements being sorted exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). +** */ #include "sqliteInt.h" @@ -893,7 +939,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; From 7a0fd192ccf4efba952b38a0d5a7b38d18ba19fe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 15:25:24 +0000 Subject: [PATCH 52/99] Fix a faulty assert() statement. FossilOrigin-Name: 9196ce407379ca3b151b601b98848771e5cb4e8f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6166275bb9..789773a941 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\senhancements\sand\sfixes\sinto\sthe\sorderby-planning\sbranch. -D 2014-05-02T13:09:06.754 +C Fix\sa\sfaulty\sassert()\sstatement. +D 2014-05-02T15:25:24.157 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 +F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bf09ce24d054bc68c226064f5f28d97e0e648a13 3bc43594aaeee9225c0590677fcce480bedcb37b -R 54b60e57b7c4903b7427b2471b549ff9 +P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d +R f6e2655ab9521eb2aa3af53a52824666 U drh -Z 3f220c7986d95ea2653ad2dab9b5d42e +Z 7c7e4b02db28ce1795a4d7aa65a0e17d diff --git a/manifest.uuid b/manifest.uuid index 2809ea21e5..a5c49533a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84862d3a095629d20c8e7b8a16f4dc26cd41ab6d \ No newline at end of file +9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6a34cefa61..70fbbcf425 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -733,10 +733,10 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ while( p ){ SorterRecord *pNext; if( pSorter->aMemory ){ - assert( p->u.iNextnMemory ); if( (u8*)p==pSorter->aMemory ){ pNext = 0; }else{ + assert( p->u.iNextnMemory ); pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; } }else{ From 012e133772cec1e77fb132eab406d0af0ad636c9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 16:03:57 +0000 Subject: [PATCH 53/99] Remove a faulty assert() from vdbesort.c. FossilOrigin-Name: d95d68aa1d14b750888d50068380cc107f9070df --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b71dec8c60..07d5ceadf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments.\s\sStore\ssome\sextra\sinformation\sin\sSqliteThread\sthat\nis\suseful\sfor\sdebugging. -D 2014-04-24T12:28:28.628 +C Remove\sa\sfaulty\sassert()\sfrom\svdbesort.c. +D 2014-05-02T16:03:57.598 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2b13026e5d4afa4737a312715aa1cd7e2f4d07c2 +F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2c9f71a451e44040624b9f255b4510743513019 -R 85ab93c26e2f3dfc5f24d719f3aa0846 -U drh -Z 26e90d38e785197a28b1e4097f1c9b04 +P 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 +R ee91b52dc43b32be4c405d028722e931 +U dan +Z 8b01b62c4d596e9ee4b370ddc9560981 diff --git a/manifest.uuid b/manifest.uuid index 6ee6407693..89f9d98d45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fb5e212089d85cdd3b4787dd69c72e6d84560b6 \ No newline at end of file +d95d68aa1d14b750888d50068380cc107f9070df \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 46de735284..daf2477532 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1900,7 +1900,6 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - assert( eMode==INCRINIT_NORMAL || pIncr->bUseThread==1 ); rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object From c7f6c148d9b4dcf7c19b3edd19c82d8738d4048e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 16:22:55 +0000 Subject: [PATCH 54/99] Failure to extend a temp file for use with mmap() in vdbesort.c is benign. FossilOrigin-Name: d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 789773a941..d6c9c71387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sstatement. -D 2014-05-02T15:25:24.157 +C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. +D 2014-05-02T16:22:55.791 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 +F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d -R f6e2655ab9521eb2aa3af53a52824666 +P 9196ce407379ca3b151b601b98848771e5cb4e8f +R 51e21c516ddbd5830d43ea54f991a56e U drh -Z 7c7e4b02db28ce1795a4d7aa65a0e17d +Z 19bdd7c6b9f54ab4b2fbcff19d13a623 diff --git a/manifest.uuid b/manifest.uuid index a5c49533a7..e8d80914a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file +d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 70fbbcf425..86fef69256 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -857,17 +857,16 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ +static void vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } - return rc; } #else -# define vdbeSorterExtendFile(x,y) SQLITE_OK +# define vdbeSorterExtendFile(x,y) #endif /* @@ -907,8 +906,8 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile( - pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 ); } @@ -1132,7 +1131,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); } } From a7bf23c6b562711263b559c76a70f78036304726 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 17:12:41 +0000 Subject: [PATCH 55/99] Fix some broken asserts in btree.c and vdbeaux.c that may fail following an OOM error. FossilOrigin-Name: e15f47064bef431c0afd8bf93eb4e160c23ad562 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- src/vdbeaux.c | 13 ++++++++----- test/malloc.test | 20 ++++++++++++++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 07d5ceadf1..1b2b4c23fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sfaulty\sassert()\sfrom\svdbesort.c. -D 2014-05-02T16:03:57.598 +C Fix\ssome\sbroken\sasserts\sin\sbtree.c\sand\svdbeaux.c\sthat\smay\sfail\sfollowing\san\sOOM\serror. +D 2014-05-02T17:12:41.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 1e8fa0e597f73cc1b144285065f175f5a661ee75 +F src/btree.c e14daeed62b5b1499835a6523f9a47d660c297c8 F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 9ea11e29230d8f454580870465354c707bd42851 @@ -283,7 +283,7 @@ F src/vdbe.c b50cd3009a2e3067746c73dce36153f19df2e42e F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b +F src/vdbeaux.c 6e6993fa4be39ab96bdd565994bee8ec3bef06ab F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 @@ -663,7 +663,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test 26ae08a09cc15a98d147ee63925e3a66048e71c9 +F test/malloc.test ada8ae15193ffc6415bf431a6db31d4ec507dbfe F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 -R ee91b52dc43b32be4c405d028722e931 +P d95d68aa1d14b750888d50068380cc107f9070df +R 2642a9b7016b11d8b125fe44a5d1acd4 U dan -Z 8b01b62c4d596e9ee4b370ddc9560981 +Z 60595d854f9f1dca901aef6c64df6a77 diff --git a/manifest.uuid b/manifest.uuid index 89f9d98d45..7ac8504d07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d68aa1d14b750888d50068380cc107f9070df \ No newline at end of file +e15f47064bef431c0afd8bf93eb4e160c23ad562 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a1d888a137..f8f0373b65 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4735,7 +4735,7 @@ int sqlite3BtreeMovetoUnpacked( } assert( (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) - && (pIdxKey->errCode!=SQLITE_NOMEM || !pCur->pBtree->db->mallocFailed) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) ); if( c<0 ){ lwr = idx+1; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e7f4b12798..360525a5da 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3593,8 +3593,9 @@ int sqlite3VdbeRecordCompare( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( CORRUPT_DB + assert( CORRUPT_DB || pKeyInfo->db==0 || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || pKeyInfo->db->mallocFailed ); return pPKey2->default_rc; } @@ -3692,10 +3693,11 @@ static int vdbeRecordCompareInt( res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + assert( pPKey2->pKeyInfo->db==0 + || (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB + || CORRUPT_DB || pPKey2->pKeyInfo->db->mallocFailed ); return res; } @@ -3756,10 +3758,11 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + assert( pPKey2->pKeyInfo->db==0 + || (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB + || CORRUPT_DB || pPKey2->pKeyInfo->db->mallocFailed ); return res; } diff --git a/test/malloc.test b/test/malloc.test index 10d2a18c96..a2a7682777 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -902,6 +902,26 @@ do_faultsim_test 40.3 -faults oom-trans* -body { faultsim_integrity_check } +reset_db +add_test_utf16bin_collate db +set big [string repeat x 200] +do_execsql_test 41.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a COLLATE utf16bin); + INSERT INTO t1 VALUES('fghij' || $::big); + INSERT INTO t1 VALUES('pqrst' || $::big); + INSERT INTO t1 VALUES('abcde' || $::big); + INSERT INTO t1 VALUES('uvwxy' || $::big); + INSERT INTO t1 VALUES('klmno' || $::big); + CREATE INDEX i1 ON t1(a); +} +do_faultsim_test 41.2 -faults oom* -body { + execsql { SELECT * FROM t1 WHERE a = ('abcde' || $::big)} +} -test { + faultsim_test_result [list 0 "abcde$::big"] + faultsim_integrity_check +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 79211e194d465fba319faae7fbf70774a12415ae Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 17:33:16 +0000 Subject: [PATCH 56/99] Simplify assert() statements used to verify correct operation of record comparison routines. FossilOrigin-Name: 3300d62dcbe74842cf86ca436959fe4e77a89f84 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 40 +++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index d6c9c71387..8fd72b4b86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. -D 2014-05-02T16:22:55.791 +C Simplify\sassert()\sstatements\sused\sto\sverify\scorrect\soperation\sof\s\nrecord\scomparison\sroutines. +D 2014-05-02T17:33:16.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbe.c b3510cc71f706beffc66e2aa4bbda54bcd5e9668 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 +F src/vdbeaux.c c9a8c917776c941af99285594d5c30d99e21c99a F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9196ce407379ca3b151b601b98848771e5cb4e8f -R 51e21c516ddbd5830d43ea54f991a56e +P d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 +R defe6572bf21263fd4b2b33706bcd8e7 U drh -Z 19bdd7c6b9f54ab4b2fbcff19d13a623 +Z 26e6dec065eafceab8263c10d377378f diff --git a/manifest.uuid b/manifest.uuid index e8d80914a0..ea44721a8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file +3300d62dcbe74842cf86ca436959fe4e77a89f84 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a6b536720..172d12042b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3137,10 +3137,14 @@ void sqlite3VdbeRecordUnpack( ** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used ** in assert() statements to ensure that the optimized code in ** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -3202,7 +3206,7 @@ static int vdbeRecordCompareDebug( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } - return rc; + goto debugCompareEnd; } i++; }while( idx1nField ); @@ -3216,7 +3220,15 @@ static int vdbeRecordCompareDebug( /* rc==0 here means that one of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - return pPKey2->default_rc; + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; } #endif @@ -3564,11 +3576,7 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB - || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || pKeyInfo->db->mallocFailed - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.zMalloc==0 ); /* See comment below */ return rc; } @@ -3587,9 +3595,7 @@ int sqlite3VdbeRecordCompare( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( CORRUPT_DB - || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) ); return pPKey2->default_rc; } @@ -3686,11 +3692,7 @@ static int vdbeRecordCompareInt( res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } @@ -3750,11 +3752,7 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } From 84de690b4f7c46b9dd70bdf6b24b4089f19475ef Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 18:46:52 +0000 Subject: [PATCH 57/99] Do not run the vdbeRecordCompareDebug() assert if pKeyInfo->db is NULL since in that case there would be no way to check for a memory allocation failure. FossilOrigin-Name: 63ed2d6acb82be8a74dbf6a61388be6da6113985 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ac820ac5a9..d82c20faf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sorderby-planning\swith\sthis\sbranch. -D 2014-05-02T18:05:38.568 +C Do\snot\srun\sthe\svdbeRecordCompareDebug()\sassert\sif\spKeyInfo->db\sis\sNULL\ssince\nin\sthat\scase\sthere\swould\sbe\sno\sway\sto\scheck\sfor\sa\smemory\sallocation\sfailure. +D 2014-05-02T18:46:52.058 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/vdbe.c 89ab2ded5123e823b47293aedd7931a4742fb6bd F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c f651c4de1193326597b3da2b8f91d993e8a6e97b +F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e15f47064bef431c0afd8bf93eb4e160c23ad562 3300d62dcbe74842cf86ca436959fe4e77a89f84 -R 6fa5ff1df255fa8dfad56d587950c625 -U dan -Z 63362d13bfcb7640057f1ca0f3721e45 +P d9549de31741239ece060e448b592ce8fc5b8042 +R bd79c5eed170ef831b62fddde9ae2198 +U drh +Z beeab334f565837c4ac52f73c3e92e7b diff --git a/manifest.uuid b/manifest.uuid index 55e5e7ae81..fc320531a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9549de31741239ece060e448b592ce8fc5b8042 \ No newline at end of file +63ed2d6acb82be8a74dbf6a61388be6da6113985 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 882bb6908e..3f52dd6017 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3156,6 +3156,7 @@ static int vdbeRecordCompareDebug( Mem mem1; pKeyInfo = pPKey2->pKeyInfo; + if( pKeyInfo->db==0 ) return 1; mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ From e500747cfee58695a2f02df0b9cf095746f0a574 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 2 May 2014 19:12:37 +0000 Subject: [PATCH 58/99] Fix typo in Windows makefile. Make sure the WaitForSingleObjectEx system call is always available. FossilOrigin-Name: d7ed529fa2aa5cb13edaabca2acaad06dffef569 --- Makefile.msc | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 4 ---- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 226d464785..db47f02755 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -564,7 +564,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo threads.o tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo diff --git a/manifest b/manifest index d82c20faf7..68b54aaa89 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Do\snot\srun\sthe\svdbeRecordCompareDebug()\sassert\sif\spKeyInfo->db\sis\sNULL\ssince\nin\sthat\scase\sthere\swould\sbe\sno\sway\sto\scheck\sfor\sa\smemory\sallocation\sfailure. -D 2014-05-02T18:46:52.058 +C Fix\stypo\sin\sWindows\smakefile.\s\sMake\ssure\sthe\sWaitForSingleObjectEx\ssystem\scall\sis\salways\savailable. +D 2014-05-02T19:12:37.015 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 153eb9b9725bc7b8e4dbe963219298e0c4a644b0 +F Makefile.msc 7d6981e8e14d8189a2a2f9b4a485f40b2032cb2e F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -205,7 +205,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c 8245fe9184300e641d02e29a8ca95cefe0cb0fd0 +F src/os_win.c 187fad4d385b3b26ec6fd4b703b1b087ad6a5c4d F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d9549de31741239ece060e448b592ce8fc5b8042 -R bd79c5eed170ef831b62fddde9ae2198 -U drh -Z beeab334f565837c4ac52f73c3e92e7b +P 63ed2d6acb82be8a74dbf6a61388be6da6113985 +R 820b790c56c4122e03b250f07fbb06ef +U mistachkin +Z 87a0e192077836669877b548f0fe55d8 diff --git a/manifest.uuid b/manifest.uuid index fc320531a2..30c06534cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ed2d6acb82be8a74dbf6a61388be6da6113985 \ No newline at end of file +d7ed529fa2aa5cb13edaabca2acaad06dffef569 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 44eb7e686f..10fea2638c 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -948,11 +948,7 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, -#else - { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, -#endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ BOOL))aSyscall[64].pCurrent) From a09c8855b7d41088d40df18b4b7b41a0c0c7ad28 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 11:22:09 +0000 Subject: [PATCH 59/99] Add the SQLITE_DEFAULT_WORKER_THREADS compile-time option. Fix a NULL-pointer dereference that can occur following OOM. FossilOrigin-Name: e0dea89b3e9f295f80210fcca007681bf1b08692 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/global.c | 2 +- src/sqliteInt.h | 7 +++++++ src/vdbesort.c | 1 + 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 68b54aaa89..a8edf3f089 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sWindows\smakefile.\s\sMake\ssure\sthe\sWaitForSingleObjectEx\ssystem\scall\sis\salways\savailable. -D 2014-05-02T19:12:37.015 +C Add\sthe\sSQLITE_DEFAULT_WORKER_THREADS\scompile-time\soption.\s\nFix\sa\sNULL-pointer\sdereference\sthat\scan\soccur\sfollowing\sOOM. +D 2014-05-03T11:22:09.616 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c deadd872189b92aca4ee2566332a86315839f811 +F src/global.c b7943ff485c31660ec0b17c68467034804df01b1 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -223,7 +223,7 @@ F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 F src/sqlite.h.in 579aebacdea59386d9cdf01fd4a16f4cafbb248f F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 76bd7ba54008909d9e437807565a54b1075983dc +F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 +F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 63ed2d6acb82be8a74dbf6a61388be6da6113985 -R 820b790c56c4122e03b250f07fbb06ef -U mistachkin -Z 87a0e192077836669877b548f0fe55d8 +P d7ed529fa2aa5cb13edaabca2acaad06dffef569 +R bf6d771799174fcada0f5c36289734b7 +U drh +Z 521faa6705b79213c3ba83b826f10b41 diff --git a/manifest.uuid b/manifest.uuid index 30c06534cc..03cf71aa4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7ed529fa2aa5cb13edaabca2acaad06dffef569 \ No newline at end of file +e0dea89b3e9f295f80210fcca007681bf1b08692 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 091b750803..4cd637f83b 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - 0, /* nWorker */ + SQLITE_DEFAULT_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0c8fe65003..43f5d34d66 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -433,6 +433,13 @@ #ifndef SQLITE_MAX_WORKER_THREADS # define SQLITE_MAX_WORKER_THREADS 0 #endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif /* diff --git a/src/vdbesort.c b/src/vdbesort.c index daf2477532..413e85c3b5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2179,6 +2179,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); pSorter->pReader = pIter; + if( pIter==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); From 7bd3c89114aa93b791a232fa39c6ec7f25cd54b4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 12:00:01 +0000 Subject: [PATCH 60/99] Add two new static mutexes, SQLITE_MUTEX_STATIC_APP1 and _APP2, for use by the application program. First intended use is in test programs for the memory allocation logic where one does not want to allocating a _FAST or _RECURSIVE mutex since that would involve using the memory allocation system under test. FossilOrigin-Name: 13686035dd1cf67ad9c6d282ab13c3259e7273d1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/mutex_unix.c | 6 +++++- src/mutex_w32.c | 6 +++++- src/sqlite.h.in | 8 ++++++-- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a8edf3f089..ed70315ed0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_DEFAULT_WORKER_THREADS\scompile-time\soption.\s\nFix\sa\sNULL-pointer\sdereference\sthat\scan\soccur\sfollowing\sOOM. -D 2014-05-03T11:22:09.616 +C Add\stwo\snew\sstatic\smutexes,\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2,\sfor\suse\sby\nthe\sapplication\sprogram.\s\sFirst\sintended\suse\sis\sin\stest\sprograms\sfor\sthe\nmemory\sallocation\slogic\swhere\sone\sdoes\snot\swant\sto\sallocating\sa\s_FAST\nor\s_RECURSIVE\smutex\ssince\sthat\swould\sinvolve\susing\sthe\smemory\sallocation\nsystem\sunder\stest. +D 2014-05-03T12:00:01.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -198,8 +198,8 @@ F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 -F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 6108c88e1cb38d8fbb3534b170793815cbedbf97 +F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c +F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c a5ed3fdc82ebab5b9b095ea1971515a7f8a303d2 F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 -F src/sqlite.h.in 579aebacdea59386d9cdf01fd4a16f4cafbb248f +F src/sqlite.h.in ed6d0cc90da850340c3863c84351e6e164c0ef00 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d7ed529fa2aa5cb13edaabca2acaad06dffef569 -R bf6d771799174fcada0f5c36289734b7 +P e0dea89b3e9f295f80210fcca007681bf1b08692 +R 29c0adc487699967d8fac6f96f2dadac U drh -Z 521faa6705b79213c3ba83b826f10b41 +Z ccb706fc99bd01bffced28b368a2dc97 diff --git a/manifest.uuid b/manifest.uuid index 03cf71aa4d..62ecc8633b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0dea89b3e9f295f80210fcca007681bf1b08692 \ No newline at end of file +13686035dd1cf67ad9c6d282ab13c3259e7273d1 \ No newline at end of file diff --git a/src/mutex_unix.c b/src/mutex_unix.c index eca7295831..ea1203c047 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -96,10 +96,12 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -133,6 +135,8 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 4b88c17452..7c17cc501e 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -99,6 +99,8 @@ static sqlite3_mutex winMutex_staticMutexes[6] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; static int winMutex_isInit = 0; @@ -159,10 +161,12 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f09c374a74..2f39101a31 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5828,10 +5828,12 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** )^ ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6035,6 +6037,8 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection From 2fc7bc08bdde4b3da490c3990e1f06168eded03d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 13:53:37 +0000 Subject: [PATCH 61/99] Get SQLITE_MUTEX_STATIC_APP1 and _APP2 working for the debugMutex implementation. FossilOrigin-Name: f49ba1c926c63ee1c4609930138389fca182c845 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.c | 2 +- src/mutex_noop.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ed70315ed0..a2cac08c7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stwo\snew\sstatic\smutexes,\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2,\sfor\suse\sby\nthe\sapplication\sprogram.\s\sFirst\sintended\suse\sis\sin\stest\sprograms\sfor\sthe\nmemory\sallocation\slogic\swhere\sone\sdoes\snot\swant\sto\sallocating\sa\s_FAST\nor\s_RECURSIVE\smutex\ssince\sthat\swould\sinvolve\susing\sthe\smemory\sallocation\nsystem\sunder\stest. -D 2014-05-03T12:00:01.738 +C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation. +D 2014-05-03T13:53:37.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,9 +195,9 @@ F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 -F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc +F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 +F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e0dea89b3e9f295f80210fcca007681bf1b08692 -R 29c0adc487699967d8fac6f96f2dadac +P 13686035dd1cf67ad9c6d282ab13c3259e7273d1 +R 7c657c53542f045bb02fc7656a8fcff4 U drh -Z ccb706fc99bd01bffced28b368a2dc97 +Z af16da93d438f6495f6a372d4f94c5d4 diff --git a/manifest.uuid b/manifest.uuid index 62ecc8633b..1fae5ede0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13686035dd1cf67ad9c6d282ab13c3259e7273d1 \ No newline at end of file +f49ba1c926c63ee1c4609930138389fca182c845 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index b567e7c27e..bad5a7c113 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -81,7 +81,7 @@ int sqlite3MutexEnd(void){ */ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 456e82a25e..4cfc6f3a76 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[6]; + static sqlite3_debug_mutex aStatic[8]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: From 0d51def29e2734627d8ea4009dcfbbf837de30b7 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 14:28:14 +0000 Subject: [PATCH 62/99] Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5. FossilOrigin-Name: 3a66c4e1bf311d38668dfcdcd77867feff6db7bd --- manifest | 28 ++++----- manifest.uuid | 2 +- src/test1.c | 35 ++++++++++++ src/test_config.c | 8 +-- src/vdbesort.c | 2 +- test/permutations.test | 6 ++ test/sort.test | 126 ++++++++++++++++++++++++++++++++++++++++- test/sort2.test | 3 + test/sort3.test | 4 ++ test/sortfault.test | 37 +++++++----- 10 files changed, 213 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index a2cac08c7a..53d89eafaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation. -D 2014-05-03T13:53:37.085 +C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5. +D 2014-05-03T14:28:14.676 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2 +F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -241,7 +241,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 +F src/test_config.c bf2e0bf49ebd8fe3dccabb4542157f9571fe48fa F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3 +F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d +F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 -F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af -F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577 +F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73 +F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 +F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 13686035dd1cf67ad9c6d282ab13c3259e7273d1 -R 7c657c53542f045bb02fc7656a8fcff4 -U drh -Z af16da93d438f6495f6a372d4f94c5d4 +P f49ba1c926c63ee1c4609930138389fca182c845 +R ecf8f8bb6befc88e6cfdbe14a65100aa +U dan +Z 4d5ad6cf671998327bccef4dcd098843 diff --git a/manifest.uuid b/manifest.uuid index 1fae5ede0f..500fa40c2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49ba1c926c63ee1c4609930138389fca182c845 \ No newline at end of file +3a66c4e1bf311d38668dfcdcd77867feff6db7bd \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 44e96c2c1f..403a7fafa7 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6336,6 +6336,40 @@ static int tclLoadStaticExtensionCmd( return TCL_OK; } +/* +** sorter_test_fakeheap BOOL +** +*/ +static int sorter_test_fakeheap( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int bArg; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + + if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){ + return TCL_ERROR; + } + + if( bArg ){ + if( sqlite3GlobalConfig.pHeap==0 ){ + sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1); + } + }else{ + if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){ + sqlite3GlobalConfig.pHeap = 0; + } + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. @@ -6569,6 +6603,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "getrusage", test_getrusage }, #endif { "load_static_extension", tclLoadStaticExtensionCmd }, + { "sorter_test_fakeheap", sorter_test_fakeheap }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test_config.c b/src/test_config.c index 2201fba007..33898a759c 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,11 +99,9 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif -#if SQLITE_MAX_WORKER_THREADS>0 - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); -#endif + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", + STRINGVALUE(SQLITE_MAX_WORKER_THREADS), TCL_GLOBAL_ONLY + ); #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); diff --git a/src/vdbesort.c b/src/vdbesort.c index 413e85c3b5..5efd6f112c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1556,7 +1556,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( aMem ){ pSorter->list.aMemory = aMem; pSorter->nMemory = sqlite3MallocSize(aMem); - }else{ + }else if( pSorter->list.aMemory ){ pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } diff --git a/test/permutations.test b/test/permutations.test index 4487af055b..b547e7b1bc 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -332,6 +332,12 @@ test_suite "coverage-analyze" -description { analyze.test analyzeB.test mallocA.test } +test_suite "coverage-sorter" -description { + Coverage tests for file vdbesort.c. +} -files { + sort.test sortfault.test +} + lappend ::testsuitelist xxx #------------------------------------------------------------------------- diff --git a/test/sort.test b/test/sort.test index ccbfdda2b3..c6c7fc6e41 100644 --- a/test/sort.test +++ b/test/sort.test @@ -8,10 +8,10 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the CREATE TABLE statement. # -# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $ +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sorter (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -485,5 +485,125 @@ do_execsql_test sort-13.3 { SELECT a, b FROM t10 ORDER BY a; } [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +#------------------------------------------------------------------------- +# Sort some large ( > 4KiB) records. +# +proc cksum {x} { + set i1 1 + set i2 2 + binary scan $x c* L + foreach {a b} $L { + set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] + set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] + } + list $i1 $i2 +} +db func cksum cksum + +do_execsql_test sort-14.0 { + PRAGMA cache_size = 5; + CREATE TABLE t11(a, b); + INSERT INTO t11 VALUES(randomblob(5000), NULL); + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 + UPDATE t11 SET b = cksum(a); +} + +foreach {tn mmap_limit} { + 1 0 + 2 1000000 +} { + do_test sort-14.$tn { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + set prev "" + db eval { SELECT * FROM t11 ORDER BY b } { + if {$b != [cksum $a]} {error "checksum failed"} + if {[string compare $b $prev] < 0} {error "sort failed"} + set prev $b + } + set {} {} + } {} +} + +#------------------------------------------------------------------------- +# +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { + 1 0 3 file true false + 2 0 3 file true true + 3 0 0 file true false + 4 1000000 3 file true false + 5 0 0 memory false true +} { + db close + + sqlite3_shutdown + sqlite3_config_worker_threads $nWorker + if {$coremutex} { + sqlite3_config multithread + } else { + sqlite3_config singlethread + } + sqlite3_initialize + + sorter_test_fakeheap $fakeheap + + reset_db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + execsql "PRAGMA temp_store = $tmpstore" + + set ten [string repeat X 10300] + set one [string repeat y 200] + + do_execsql_test 15.$tn.1 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one UNION ALL + SELECT 1, $ten UNION ALL + SELECT 3, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 1 $ten 2 $one 3 $one 4 $ten] + + do_execsql_test 15.$tn.2 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(4); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(1); + CREATE INDEX i1 ON t1(a); + SELECT * FROM t1 ORDER BY a; + } {1 2 3 4 5 6} + + do_execsql_test 15.$tn.3 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 2 $one 4 $ten] + + sorter_test_fakeheap 0 +} + +db close +sqlite3_shutdown +#sqlite3_config_worker_threads $sqlite_options(worker_threads) +sqlite3_config_worker_threads 0 +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize finish_test + diff --git a/test/sort2.test b/test/sort2.test index 4fb6a9462b..e4e40dab74 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, the tests in this file attempt to verify that +# multi-threaded sorting works. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sort3.test b/test/sort3.test index 9963dcccd3..80d8bbca3f 100644 --- a/test/sort3.test +++ b/test/sort3.test @@ -10,6 +10,10 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# The tests in this file verify that sorting works when the library is +# configured to use mmap(), but the temporary files generated by the +# sorter are too large to be completely mapped. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sortfault.test b/test/sortfault.test index bdf57b45ba..abe9af6854 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, it tests the effects of fault injection on the sorter +# module (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -20,20 +23,26 @@ do_execsql_test 1.0 { PRAGMA cache_size = 5; } -do_faultsim_test 1 -prep { - sqlite3 db test.db -} -body { - execsql { - WITH r(x,y) AS ( - SELECT 1, randomblob(1000) - UNION ALL - SELECT x+1, randomblob(1000) FROM r - LIMIT 500 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } -} -test { - faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} +foreach {tn mmap_limit} { + 1 0 + 2 100000 +} { + do_faultsim_test 1.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 500 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } + } -test { + faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} + } } finish_test From e18e90ebafe89d444c0524f91088b0c9316cbbc0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 19:33:00 +0000 Subject: [PATCH 63/99] Fix a race condition in the sorter. FossilOrigin-Name: 32ccf3ae18531682dfd039fa8df6ad9a907ac455 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 21 ++++++++++++++++++--- test/sort.test | 3 --- test/sortfault.test | 38 ++++++++++++++++++++++++++++---------- 5 files changed, 55 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 53d89eafaf..917d67a6b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5. -D 2014-05-03T14:28:14.676 +C Fix\sa\srace\scondition\sin\sthe\ssorter. +D 2014-05-03T19:33:00.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0 +F src/vdbesort.c c443cdf00fc8b90b17fbeaa1ad833d3091b0bf5c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73 +F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512 +F test/sortfault.test f875d29c58b2eafdca1b51c0810075570d5a3cbc F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f49ba1c926c63ee1c4609930138389fca182c845 -R ecf8f8bb6befc88e6cfdbe14a65100aa +P 3a66c4e1bf311d38668dfcdcd77867feff6db7bd +R 07ad433b1f531db0a3f09b334b377488 U dan -Z 4d5ad6cf671998327bccef4dcd098843 +Z 8b4c7c15ef224489143b17033ef5ced9 diff --git a/manifest.uuid b/manifest.uuid index 500fa40c2b..6bfb75656c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a66c4e1bf311d38668dfcdcd77867feff6db7bd \ No newline at end of file +32ccf3ae18531682dfd039fa8df6ad9a907ac455 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5efd6f112c..5676d96e37 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -651,6 +651,7 @@ static int vdbePmaReaderNext(PmaReader *pIter){ int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ + if( pIter->iReadOff>=pIter->iEof ){ IncrMerger *pIncr = pIter->pIncr; int bEof = 1; @@ -1844,10 +1845,18 @@ static int vdbeIncrInitMerger( ){ int rc = SQLITE_OK; /* Return code */ int i; /* For iterating through PmaReader objects */ + int nTree = pMerger->nTree; - for(i=0; rc==SQLITE_OK && inTree; i++){ + for(i=0; rc==SQLITE_OK && iaIter[i]); + /* Iterators should be normally initialized in order, as if they are + ** reading from the same temp file this makes for more linear file IO. + ** However, in the INCRINIT_ROOT case, if iterator aIter[nTask-1] is + ** in use it will block the vdbePmaReaderNext() call while it uses + ** the main thread to fill its buffer. So calling PmaReaderNext() + ** on this iterator before any of the multi-threaded iterators takes + ** better advantage of multi-processor hardware. */ + rc = vdbePmaReaderNext(&pMerger->aIter[nTree-i-1]); }else{ rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); } @@ -2196,7 +2205,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ PmaReader *p = &pMain->aIter[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } + if( p->pIncr ){ + if( iTask==pSorter->nTask-1 ){ + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + }else{ + rc = vdbePmaReaderBgIncrInit(p); + } + } } } } diff --git a/test/sort.test b/test/sort.test index c6c7fc6e41..0c87dd6441 100644 --- a/test/sort.test +++ b/test/sort.test @@ -541,7 +541,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { 5 0 0 memory false true } { db close - sqlite3_shutdown sqlite3_config_worker_threads $nWorker if {$coremutex} { @@ -550,7 +549,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { sqlite3_config singlethread } sqlite3_initialize - sorter_test_fakeheap $fakeheap reset_db @@ -597,7 +595,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { db close sqlite3_shutdown -#sqlite3_config_worker_threads $sqlite_options(worker_threads) sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread diff --git a/test/sortfault.test b/test/sortfault.test index abe9af6854..7c58bb6833 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -23,27 +23,45 @@ do_execsql_test 1.0 { PRAGMA cache_size = 5; } -foreach {tn mmap_limit} { - 1 0 - 2 100000 +foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { + 1 0 0 file multithread false + 2 100000 0 file multithread false } { + catch { db close } + sqlite3_shutdown + sqlite3_config_worker_threads $nWorker + sqlite3_config $threadsmode + sqlite3_initialize + sorter_test_fakeheap $fakeheap + + set str [string repeat a 1000] + do_faultsim_test 1.$tn -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } } -body { execsql { WITH r(x,y) AS ( - SELECT 1, randomblob(1000) + SELECT 1, $::str UNION ALL - SELECT x+1, randomblob(1000) FROM r - LIMIT 500 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } + SELECT x+1, $::str FROM r + LIMIT 200 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } } -test { - faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} + faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } } +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 0 +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize finish_test From f7f425d6006c5181cf5e3771d35766c5de456271 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 20:43:13 +0000 Subject: [PATCH 64/99] Add an extra fault-injection test to sortfault.test. Remove an unreachable branch from vdbesort.c. FossilOrigin-Name: a33a366ba8a0da81ddd895d552a348441ef8529a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 12 +++++------- test/sortfault.test | 22 ++++++++++++++++++++++ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 917d67a6b3..903e7795ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\srace\scondition\sin\sthe\ssorter. -D 2014-05-03T19:33:00.713 +C Add\san\sextra\sfault-injection\stest\sto\ssortfault.test.\sRemove\san\sunreachable\sbranch\sfrom\svdbesort.c. +D 2014-05-03T20:43:13.986 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c c443cdf00fc8b90b17fbeaa1ad833d3091b0bf5c +F src/vdbesort.c db67b5b54c476163f3068a194a734c394ecb675e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test f875d29c58b2eafdca1b51c0810075570d5a3cbc +F test/sortfault.test f2f94227d97f7efb6351aff29e77498e877d6216 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3a66c4e1bf311d38668dfcdcd77867feff6db7bd -R 07ad433b1f531db0a3f09b334b377488 +P 32ccf3ae18531682dfd039fa8df6ad9a907ac455 +R 5ede29f57c1094d89ff6cfb82c23c871 U dan -Z 8b4c7c15ef224489143b17033ef5ced9 +Z 83a8a7f60269415a80d26ca6a02bda49 diff --git a/manifest.uuid b/manifest.uuid index 6bfb75656c..4a6d01e399 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32ccf3ae18531682dfd039fa8df6ad9a907ac455 \ No newline at end of file +a33a366ba8a0da81ddd895d552a348441ef8529a \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5676d96e37..d0a6be84c9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1807,11 +1807,9 @@ static int vdbeIncrNew( /* ** Set the "use-threads" flag on object pIncr. */ -static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ - if( bUseThread ){ - pIncr->bUseThread = 1; - pIncr->pTask->file2.iEof -= pIncr->mxSz; - } +static void vdbeIncrSetThreads(IncrMerger *pIncr){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; } #endif /* SQLITE_MAX_WORKER_THREADS>0 */ @@ -2193,11 +2191,11 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + vdbeIncrSetThreads(pIter->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aIter[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } diff --git a/test/sortfault.test b/test/sortfault.test index 7c58bb6833..2f746fcc24 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -26,6 +26,7 @@ do_execsql_test 1.0 { foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { 1 0 0 file multithread false 2 100000 0 file multithread false + 3 100000 1 file multithread false } { catch { db close } sqlite3_shutdown @@ -53,6 +54,27 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { } -test { faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } + + + do_faultsim_test 2.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + add_test_utf16bin_collate db + execsql { PRAGMA cache_size = 5 } + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 100, $::str + UNION ALL + SELECT x-1, $::str FROM r + LIMIT 100 + ) + SELECT count(x), length(y) FROM r GROUP BY y COLLATE utf16bin, (x%5) + } + } -test { + faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} + } + } catch { db close } From d94d4ee7bd462e9a00e47207cac8a0119e7bffa8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 09:08:54 +0000 Subject: [PATCH 65/99] Add tests so that the "coverage-sorter" test permutation covers all branches in vdbesort.c. Fix a few minor problems in the same file. FossilOrigin-Name: bde28b702dabd02269e333535cc41481351c5efc --- main.mk | 1 + manifest | 20 +++++----- manifest.uuid | 2 +- src/threads.c | 10 +++++ src/vdbesort.c | 54 +++++++++++++------------- test/sort.test | 50 ++++++++++++++++++++---- test/sortfault.test | 94 +++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 178 insertions(+), 53 deletions(-) diff --git a/main.mk b/main.mk index 76285b0d76..d61f715e80 100644 --- a/main.mk +++ b/main.mk @@ -312,6 +312,7 @@ TESTSRC2 = \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ diff --git a/manifest b/manifest index 903e7795ef..a8debded75 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\sfault-injection\stest\sto\ssortfault.test.\sRemove\san\sunreachable\sbranch\sfrom\svdbesort.c. -D 2014-05-03T20:43:13.986 +C Add\stests\sso\sthat\sthe\s"coverage-sorter"\stest\spermutation\scovers\sall\sbranches\sin\svdbesort.c.\sFix\sa\sfew\sminor\sproblems\sin\sthe\ssame\sfile. +D 2014-05-05T09:08:54.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,7 +144,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 97673939d94f210b7218a8236df6c8799d4a0da6 +F main.mk f51f401a87fd07be255bcecae8b03105ca2024e8 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -273,7 +273,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c e35de159f9ced746266ff4b2129b7a828e59119c +F src/threads.c b9daffcb6ae3a9c080da37b905e790d45a8f99db F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c db67b5b54c476163f3068a194a734c394ecb675e +F src/vdbesort.c af752fa4c125ab2bfeee1c4e0b50504e79aa7051 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 +F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test f2f94227d97f7efb6351aff29e77498e877d6216 +F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32ccf3ae18531682dfd039fa8df6ad9a907ac455 -R 5ede29f57c1094d89ff6cfb82c23c871 +P a33a366ba8a0da81ddd895d552a348441ef8529a +R 48fa4af9d614a9372c1dc3ff956ba091 U dan -Z 83a8a7f60269415a80d26ca6a02bda49 +Z 1076615c7d8998a9f1b953b2c0f517bb diff --git a/manifest.uuid b/manifest.uuid index 4a6d01e399..8a5cb48075 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a33a366ba8a0da81ddd895d552a348441ef8529a \ No newline at end of file +bde28b702dabd02269e333535cc41481351c5efc \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 64975801be..6b59ce90d2 100644 --- a/src/threads.c +++ b/src/threads.c @@ -208,6 +208,7 @@ int sqlite3ThreadCreate( /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ @@ -216,6 +217,15 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ *ppOut = p->pResult; } sqlite3_free(p); + +#if defined(SQLITE_TEST) + { + void *pTstAlloc = sqlite3Malloc(10); + if (!pTstAlloc) return SQLITE_NOMEM; + sqlite3_free(pTstAlloc); + } +#endif + return SQLITE_OK; } diff --git a/src/vdbesort.c b/src/vdbesort.c index d0a6be84c9..90a611914d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1251,11 +1251,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ pList->pList = p; sqlite3_free(aSlot); - if( pTask->pUnpacked->errCode ){ - assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); - return SQLITE_NOMEM; - } - return SQLITE_OK; + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM + ); + return pTask->pUnpacked->errCode; } /* @@ -1485,7 +1484,7 @@ static int vdbeSorterNext( *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } - return rc; + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } #if SQLITE_MAX_WORKER_THREADS>0 @@ -1534,7 +1533,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( pTask->bDone ){ rc = vdbeSorterJoinThread(pTask); } - if( pTask->pThread==0 || rc!=SQLITE_OK ) break; + if( rc!=SQLITE_OK || pTask->pThread==0 ) break; } if( rc==SQLITE_OK ){ @@ -1864,7 +1863,7 @@ static int vdbeIncrInitMerger( rc = vdbeSorterDoCompare(pTask, pMerger, i); } - return rc; + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } /* @@ -2074,9 +2073,10 @@ static int vdbeSorterAddToTree( rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); } } - - p = pIter->pIncr->pMerger; - nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + if( rc==SQLITE_OK ){ + p = pIter->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + } } if( rc==SQLITE_OK ){ @@ -2178,6 +2178,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS + assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; PmaReader *pIter; @@ -2199,16 +2200,14 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pIncr->pTask!=pLast ); } } - if( pSorter->nTask>1 ){ - for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ - if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); - }else{ - rc = vdbePmaReaderBgIncrInit(p); - } + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ + if( iTask==pSorter->nTask-1 ){ + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + }else{ + rc = vdbePmaReaderBgIncrInit(p); } } } @@ -2216,8 +2215,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL); - rc = vdbePmaReaderIncrInit(pIter, eMode); + rc = vdbePmaReaderIncrInit(pIter, INCRINIT_ROOT); } }else #endif @@ -2259,10 +2257,12 @@ int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ return rc; } - /* Write the current in-memory list to a PMA. */ - if( pSorter->list.pList ){ - rc = vdbeSorterFlushPMA(pSorter); - } + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + ** function flushes the contents of memory to disk, it immediately always + ** creates a new list consisting of a single key immediately afterwards. + ** So the list is never empty at this point. */ + assert( pSorter->list.pList ); + rc = vdbeSorterFlushPMA(pSorter); /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); diff --git a/test/sort.test b/test/sort.test index 0c87dd6441..e75740e9c3 100644 --- a/test/sort.test +++ b/test/sort.test @@ -533,12 +533,14 @@ foreach {tn mmap_limit} { #------------------------------------------------------------------------- # -foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { - 1 0 3 file true false - 2 0 3 file true true - 3 0 0 file true false - 4 1000000 3 file true false - 5 0 0 memory false true +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { + 1 0 3 file true false 0 + 2 0 3 file true true 0 + 3 0 0 file true false 0 + 4 1000000 3 file true false 0 + 5 0 0 memory false true 0 + 6 0 0 file false true 1000000 + 7 0 0 file false true 10000 } { db close sqlite3_shutdown @@ -550,6 +552,7 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { } sqlite3_initialize sorter_test_fakeheap $fakeheap + sqlite3_soft_heap_limit $softheaplimit reset_db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit @@ -558,8 +561,13 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { set ten [string repeat X 10300] set one [string repeat y 200] + if {$softheaplimit} { + execsql { PRAGMA cache_size = 20 }; + } else { + execsql { PRAGMA cache_size = 5 }; + } + do_execsql_test 15.$tn.1 { - PRAGMA cache_size = 5; WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one UNION ALL @@ -582,7 +590,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { } {1 2 3 4 5 6} do_execsql_test 15.$tn.3 { - PRAGMA cache_size = 5; WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one @@ -601,6 +608,33 @@ set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) sqlite3_initialize +sqlite3_soft_heap_limit 0 + +reset_db +do_catchsql_test 16.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(NULL, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {0 {}} +reset_db +do_catchsql_test 16.2 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {1 {UNIQUE constraint failed: t1.b, t1.a, t1.c}} + +reset_db +do_execsql_test 17.1 { + SELECT * FROM sqlite_master ORDER BY sql; +} {} finish_test diff --git a/test/sortfault.test b/test/sortfault.test index 2f746fcc24..4c199ab212 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -18,24 +18,32 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sortfault - do_execsql_test 1.0 { PRAGMA cache_size = 5; } -foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { - 1 0 0 file multithread false - 2 100000 0 file multithread false - 3 100000 1 file multithread false +foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { + 1 0 0 file multithread false false + 2 100000 0 file multithread false false + 3 100000 1 file multithread false false + 4 2000000 0 file singlethread false true } { + if {$sqlite_options(threadsafe)} { set threadsmode singlethread } + catch { db close } sqlite3_shutdown sqlite3_config_worker_threads $nWorker sqlite3_config $threadsmode + if { $lookaside } { + sqlite3_config_lookaside 100 500 + } else { + sqlite3_config_lookaside 0 0 + } sqlite3_initialize sorter_test_fakeheap $fakeheap set str [string repeat a 1000] + puts $threadsmode do_faultsim_test 1.$tn -prep { sqlite3 db test.db @@ -55,8 +63,7 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } - - do_faultsim_test 2.$tn -prep { + do_faultsim_test 2.$tn -faults oom* -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit add_test_utf16bin_collate db @@ -75,6 +82,30 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} } + if {$mmap_limit > 1000000} { + set str2 [string repeat $str 10] + + sqlite3_memdebug_vfs_oom_test 0 + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } + + do_faultsim_test 3.$tn -faults oom-trans* -body { + execsql { + WITH r(x,y) AS ( + SELECT 300, $::str2 + UNION ALL + SELECT x-1, $::str2 FROM r + LIMIT 300 + ) + SELECT count(x), length(y) FROM r GROUP BY y, (x%5) + } + } -test { + faultsim_test_result {0 {60 10000 60 10000 60 10000 60 10000 60 10000}} + } + + sqlite3_memdebug_vfs_oom_test 1 + } } catch { db close } @@ -84,6 +115,55 @@ set t(0) singlethread set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_config_lookaside 100 500 sqlite3_initialize + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} +do_test 4.1 { + for {set i 0} {$i < 256} {incr i} { + execsql { + INSERT INTO t1 SELECT + ((a<<3) + b) & 2147483647, + ((b<<3) + c) & 2147483647, + ((c<<3) + a) & 2147483647 + FROM t1 ORDER BY rowid DESC LIMIT 1; + } + } +} {} + +faultsim_save_and_close + +do_faultsim_test 4.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE UNIQUE INDEX i1 ON t1(a,b,c) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set a [string repeat a 500] +set b [string repeat b 500] +set c [string repeat c 500] +do_execsql_test 5.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES($a, $b, $c); + INSERT INTO t1 VALUES($c, $b, $a); +} + +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT * FROM t1 ORDER BY a } +} -test { + faultsim_test_result [list 0 [list $::a $::b $::c $::c $::b $::a]] +} + finish_test From 0d3a4085e5f33768726abe9450ca95af182ee082 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 15:58:40 +0000 Subject: [PATCH 66/99] Fix a race condition in the sorter code. FossilOrigin-Name: 2d2edfe58db101d42a96772b856e6e55b401aab6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a8debded75..e411d8e3de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sso\sthat\sthe\s"coverage-sorter"\stest\spermutation\scovers\sall\sbranches\sin\svdbesort.c.\sFix\sa\sfew\sminor\sproblems\sin\sthe\ssame\sfile. -D 2014-05-05T09:08:54.007 +C Fix\sa\srace\scondition\sin\sthe\ssorter\scode. +D 2014-05-05T15:58:40.542 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c af752fa4c125ab2bfeee1c4e0b50504e79aa7051 +F src/vdbesort.c b36070436d33372237541e9eaa6068e1a61bc402 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a33a366ba8a0da81ddd895d552a348441ef8529a -R 48fa4af9d614a9372c1dc3ff956ba091 +P bde28b702dabd02269e333535cc41481351c5efc +R e53afa0ba1bd427219aede77058d3d04 U dan -Z 1076615c7d8998a9f1b953b2c0f517bb +Z ba1cae0a2780ce3568cf6b58bd76a820 diff --git a/manifest.uuid b/manifest.uuid index 8a5cb48075..82ab91d82e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bde28b702dabd02269e333535cc41481351c5efc \ No newline at end of file +2d2edfe58db101d42a96772b856e6e55b401aab6 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 90a611914d..9c57135dd3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -991,7 +991,15 @@ static int vdbeSorterCreateThread( static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inTask; i++){ + + /* This function is always called by the main user thread. + ** + ** If this function is being called after SorterRewind() has been called, + ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread + ** is currently attempt to join one of the other threads. To avoid a race + ** condition where this thread also attempts to join the same object, join + ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ + for(i=pSorter->nTask-1; i>=0; i--){ SortSubtask *pTask = &pSorter->aTask[i]; int rc2 = vdbeSorterJoinThread(pTask); if( rc==SQLITE_OK ) rc = rc2; From 449cb9a5af9762d6eba3343802c601723f19929f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 20:03:50 +0000 Subject: [PATCH 67/99] Add test file sort4.test, containing brute force tests for the multi-theaded sorter. FossilOrigin-Name: 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 --- manifest | 13 ++-- manifest.uuid | 2 +- test/permutations.test | 2 +- test/sort4.test | 145 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 test/sort4.test diff --git a/manifest b/manifest index e411d8e3de..ac504dddc6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\srace\scondition\sin\sthe\ssorter\scode. -D 2014-05-05T15:58:40.542 +C Add\stest\sfile\ssort4.test,\scontaining\sbrute\sforce\stests\sfor\sthe\smulti-theaded\ssorter. +D 2014-05-05T20:03:50.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb +F test/permutations.test 465bc22a873ced74a6d1dedc0dde5da424be6e6a F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -824,6 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sort4.test a29761a7d05f194e516e91a03d38f8194ac9849d F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1170,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bde28b702dabd02269e333535cc41481351c5efc -R e53afa0ba1bd427219aede77058d3d04 +P 2d2edfe58db101d42a96772b856e6e55b401aab6 +R a3f7784643670968322db5c49315411c U dan -Z ba1cae0a2780ce3568cf6b58bd76a820 +Z cca87b3c456f151b38b423a6310da351 diff --git a/manifest.uuid b/manifest.uuid index 82ab91d82e..f77d0e5e6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d2edfe58db101d42a96772b856e6e55b401aab6 \ No newline at end of file +9cc364c42cc64ab7b55b5c55e303fb63a456cf00 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index b547e7b1bc..3cf4aa80b2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,7 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test - sort3.test + sort3.test sort4.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] diff --git a/test/sort4.test b/test/sort4.test new file mode 100644 index 0000000000..3fb335cb2f --- /dev/null +++ b/test/sort4.test @@ -0,0 +1,145 @@ +# 2014 May 6. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# The tests in this file are brute force tests of the multi-threaded +# sorter. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort4 + +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 3 +sqlite3_initialize +reset_db + +#-------------------------------------------------------------------- +# Set up a table "t1" containing $nRow rows. Each row contains also +# contains blob fields that total to at least $nPayload bytes of +# content. +# +proc populate_table {nRow nPayload} { + set nCol 0 + + set n 0 + for {set nCol 0} {$n < $nPayload} {incr nCol} { + incr n [expr (4 << $nCol)] + } + + set cols [lrange [list xxx c0 c1 c2 c3 c4 c5 c6 c7] 1 $nCol] + set data [lrange [list xxx \ + randomblob(4) randomblob(8) randomblob(16) randomblob(32) \ + randomblob(64) randomblob(128) randomblob(256) randomblob(512) \ + ] 1 $nCol] + + execsql { DROP TABLE IF EXISTS t1 } + + db transaction { + execsql "CREATE TABLE t1(a, [join $cols ,], b);" + set insert "INSERT INTO t1 VALUES(:k, [join $data ,], :k)" + for {set i 0} {$i < $nRow} {incr i} { + set k [expr int(rand()*1000000000)] + execsql $insert + } + } +} + +# Helper for [do_sorter_test] +# +proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { + db eval "PRAGMA cache_size = $cache_size" + set res [list] + + set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow] + + set nPayload [expr (($nPayload+3)/4) * 4] + set cols [list] + foreach {mask col} { + 0x04 c0 0x08 c1 0x10 c2 0x20 c3 + 0x40 c4 0x80 c5 0x100 c6 0x200 c7 + } { + if {$nPayload & $mask} { lappend cols $col } + } + + set n 0 + db eval "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" { + if {$a!=$b} { error "a!=b (a=$a b=$b)" } + lappend res $a + incr n + if {$n==$nLoad} break + } + + + set sql {SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead} + if {$res != [db eval $sql]} { + puts $res + puts [db eval {SELECT a FROM t1 WHERE rowid<=$nLoad ORDER BY a}] + error "data no good" + } + + set {} {} +} + +# Usage: +# +# do_sorter_test ... +# +# where are any of the following switches: +# +# -rows N (number of rows to have sorter sort) +# -read N (number of rows to read out of sorter) +# -payload N (bytes of payload to read with each row) +# -cachesize N (Value for "PRAGMA cache_size = ?") +# -repeats N (number of times to repeat test) +# +proc do_sorter_test {tn args} { + set a(-rows) 1000 + set a(-repeats) 1 + set a(-read) 100 + set a(-payload) 100 + set a(-cachesize) 100 + + foreach {s val} $args { + if {[info exists a($s)]==0} { + unset a(-cachesize) + set optlist "[join [array names a] ,] or -cachesize" + error "Unknown option $s, expected $optlist" + } + set a($s) $val + } + + for {set i 0} {$i < $a(-repeats)} {incr i} { + set cmd [list sorter_test $a(-rows) $a(-read) $a(-payload) $a(-cachesize)] + do_test $tn.$i $cmd {} + } +} + +do_test 1 { + execsql "PRAGMA page_size = 4096" + populate_table 100000 500 +} {} + +do_sorter_test 2 -repeats 10 -rows 1000 -read 100 +do_sorter_test 3 -repeats 10 -rows 100000 -read 1000 +do_sorter_test 4 -repeats 10 -rows 100000 -read 1000 -payload 500 +do_sorter_test 5 -repeats 10 -rows 100000 -read 100000 -payload 8 +do_sorter_test 6 -repeats 10 -rows 100000 -read 10 -payload 8 + +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 0 +sqlite3_initialize +finish_test + + From dfea45330e5187e37a7de7750fcef6884630a1bf Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 May 2014 15:38:07 +0000 Subject: [PATCH 68/99] Re-implement the core of the multi-threaded sorter tests in sort4.test using C. Run each test in sort4.test ten times, or repeat all tests for 300 seconds as part of the "multithread" permutation test. FossilOrigin-Name: 208b2b04d4d282bec4424ea7160a123ba549d118 --- manifest | 16 +++---- manifest.uuid | 2 +- src/test1.c | 74 ++++++++++++++++++++++++++++++ test/permutations.test | 2 +- test/sort4.test | 101 +++++++++++++++++++++++++++++------------ 5 files changed, 156 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index ac504dddc6..e5ca2b242a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\ssort4.test,\scontaining\sbrute\sforce\stests\sfor\sthe\smulti-theaded\ssorter. -D 2014-05-05T20:03:50.967 +C Re-implement\sthe\score\sof\sthe\smulti-threaded\ssorter\stests\sin\ssort4.test\susing\sC.\sRun\seach\stest\sin\ssort4.test\sten\stimes,\sor\srepeat\sall\stests\sfor\s300\sseconds\sas\spart\sof\sthe\s"multithread"\spermutation\stest. +D 2014-05-06T15:38:07.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9 +F src/test1.c a0e59104fec5626bb1d1bd746157f43e85245e4b F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 465bc22a873ced74a6d1dedc0dde5da424be6e6a +F test/permutations.test 33e7e239ba494fdb30e2f4ffc64c508b145ff42f F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test a29761a7d05f194e516e91a03d38f8194ac9849d +F test/sort4.test 5cce4601abc9b1b63ac25d087723b257710577bb F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2d2edfe58db101d42a96772b856e6e55b401aab6 -R a3f7784643670968322db5c49315411c +P 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 +R 3ed4f872bc36ee2cfbd000cc3f4dde43 U dan -Z cca87b3c456f151b38b423a6310da351 +Z 5684026d864e64d9d77cb71d6b4cd16b diff --git a/manifest.uuid b/manifest.uuid index f77d0e5e6f..4e0ab1c048 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9cc364c42cc64ab7b55b5c55e303fb63a456cf00 \ No newline at end of file +208b2b04d4d282bec4424ea7160a123ba549d118 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 403a7fafa7..2d0efb949e 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6370,6 +6370,79 @@ static int sorter_test_fakeheap( return TCL_OK; } +/* +** sorter_test_sort4_helper DB SQL1 NSTEP SQL2 +** +** Compile SQL statement $SQL1 and step it $NSTEP times. For each row, +** check that the leftmost and rightmost columns returned are both integers, +** and that both contain the same value. +** +** Then execute statement $SQL2. Check that the statement returns the same +** set of integers in the same order as in the previous step (using $SQL1). +*/ +static int sorter_test_sort4_helper( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zSql1; + const char *zSql2; + int nStep; + int iStep; + int iCksum1 = 0; + int iCksum2 = 0; + int rc; + int iB; + sqlite3 *db; + sqlite3_stmt *pStmt; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SQL1 NSTEP SQL2"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSql1 = Tcl_GetString(objv[2]); + if( Tcl_GetIntFromObj(interp, objv[3], &nStep) ) return TCL_ERROR; + zSql2 = Tcl_GetString(objv[4]); + + rc = sqlite3_prepare_v2(db, zSql1, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) goto sql_error; + + iB = sqlite3_column_count(pStmt)-1; + for(iStep=0; iStep, b INTEGER); +# +# For each row, the values of columns "a" and "b" are set to the same +# pseudo-randomly selected integer. The "extra-columns", of which there +# are at most eight, are named c0, c1, c2 etc. Column c0 contains a 4 +# byte string. Column c1 an 8 byte string. Field c2 16 bytes, and so on. +# +# This table is intended to be used for testing queries of the form: +# +# SELECT a, , b FROM t1 ORDER BY a; +# +# The test code checks that rows are returned in order, and that the +# values of "a" and "b" are the same for each row (the idea being that +# if field "b" at the end of the sorter record has not been corrupted, +# the rest of the record is probably Ok as well). # proc populate_table {nRow nPayload} { set nCol 0 @@ -57,8 +80,7 @@ proc populate_table {nRow nPayload} { # Helper for [do_sorter_test] # -proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { - db eval "PRAGMA cache_size = $cache_size" +proc sorter_test {nRow nRead nPayload} { set res [list] set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow] @@ -72,22 +94,25 @@ proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { if {$nPayload & $mask} { lappend cols $col } } - set n 0 - db eval "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" { - if {$a!=$b} { error "a!=b (a=$a b=$b)" } - lappend res $a - incr n - if {$n==$nLoad} break - } - - - set sql {SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead} - if {$res != [db eval $sql]} { - puts $res - puts [db eval {SELECT a FROM t1 WHERE rowid<=$nLoad ORDER BY a}] - error "data no good" - } + # Create two SELECT statements. Statement $sql1 uses the sorter to sort + # $nRow records of a bit over $nPayload bytes each read from the "t1" + # table created by [populate_table] proc above. Rows are sorted in order + # of the integer field in each "t1" record. + # + # The second SQL statement sorts the same set of rows as the first, but + # uses a LIMIT clause, causing SQLite to use a temp table instead of the + # sorter for sorting. + # + set sql1 "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" + set sql2 "SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead" + # Pass the two SQL statements to a helper command written in C. This + # command steps statement $sql1 $nRead times and compares the integer + # values in the rows returned with the results of executing $sql2. If + # the comparison fails (indicating some bug in the sorter), a Tcl + # exception is thrown. + # + sorter_test_sort4_helper db $sql1 $nRead $sql2 set {} {} } @@ -119,22 +144,41 @@ proc do_sorter_test {tn args} { set a($s) $val } - for {set i 0} {$i < $a(-repeats)} {incr i} { - set cmd [list sorter_test $a(-rows) $a(-read) $a(-payload) $a(-cachesize)] - do_test $tn.$i $cmd {} - } + db eval "PRAGMA cache_size = $a(-cachesize)" + + do_test $tn [subst -nocommands { + for {set i 0} {[set i] < $a(-repeats)} {incr i} { + sorter_test $a(-rows) $a(-read) $a(-payload) + } + }] {} } +proc clock_seconds {} { + db one {SELECT strftime('%s')} +} + +#------------------------------------------------------------------------- +# Begin tests here. + +# Create a test database. do_test 1 { execsql "PRAGMA page_size = 4096" populate_table 100000 500 } {} -do_sorter_test 2 -repeats 10 -rows 1000 -read 100 -do_sorter_test 3 -repeats 10 -rows 100000 -read 1000 -do_sorter_test 4 -repeats 10 -rows 100000 -read 1000 -payload 500 -do_sorter_test 5 -repeats 10 -rows 100000 -read 100000 -payload 8 -do_sorter_test 6 -repeats 10 -rows 100000 -read 10 -payload 8 +set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT] + +for {set tn 2} {1} {incr tn} { + do_sorter_test $tn.2 -repeats 10 -rows 1000 -read 100 + do_sorter_test $tn.3 -repeats 10 -rows 100000 -read 1000 + do_sorter_test $tn.4 -repeats 10 -rows 100000 -read 1000 -payload 500 + do_sorter_test $tn.5 -repeats 10 -rows 100000 -read 100000 -payload 8 + do_sorter_test $tn.6 -repeats 10 -rows 100000 -read 10 -payload 8 + + set iNow [clock_seconds] + if {$iNow>=$iTimeLimit} break + do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {} +} catch { db close } sqlite3_shutdown @@ -142,4 +186,3 @@ sqlite3_config_worker_threads 0 sqlite3_initialize finish_test - From 5a7dbc70eac74506513f2e69141039fb8ee0efd3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 May 2014 16:21:30 +0000 Subject: [PATCH 69/99] Add a little extra variety to the tests in sort4.test. FossilOrigin-Name: 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/sort4.test | 24 +++++++++++++++++------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e5ca2b242a..e4b717ce1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Re-implement\sthe\score\sof\sthe\smulti-threaded\ssorter\stests\sin\ssort4.test\susing\sC.\sRun\seach\stest\sin\ssort4.test\sten\stimes,\sor\srepeat\sall\stests\sfor\s300\sseconds\sas\spart\sof\sthe\s"multithread"\spermutation\stest. -D 2014-05-06T15:38:07.762 +C Add\sa\slittle\sextra\svariety\sto\sthe\stests\sin\ssort4.test. +D 2014-05-06T16:21:30.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test 5cce4601abc9b1b63ac25d087723b257710577bb +F test/sort4.test 971452fd4e2928e6fc05c3868396ad7d5f9ce2ad F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 -R 3ed4f872bc36ee2cfbd000cc3f4dde43 +P 208b2b04d4d282bec4424ea7160a123ba549d118 +R b3799ca8720be78abdc748ed84ab7526 U dan -Z 5684026d864e64d9d77cb71d6b4cd16b +Z f689b9a36b0734ece63c5c5e094898f6 diff --git a/manifest.uuid b/manifest.uuid index 4e0ab1c048..0fc1ce1baf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -208b2b04d4d282bec4424ea7160a123ba549d118 \ No newline at end of file +7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa \ No newline at end of file diff --git a/test/sort4.test b/test/sort4.test index 7bee0aad27..4e8336cd84 100644 --- a/test/sort4.test +++ b/test/sort4.test @@ -127,6 +127,7 @@ proc sorter_test {nRow nRead nPayload} { # -payload N (bytes of payload to read with each row) # -cachesize N (Value for "PRAGMA cache_size = ?") # -repeats N (number of times to repeat test) +# -fakeheap BOOL (true to use separate allocations for in-memory records) # proc do_sorter_test {tn args} { set a(-rows) 1000 @@ -134,6 +135,7 @@ proc do_sorter_test {tn args} { set a(-read) 100 set a(-payload) 100 set a(-cachesize) 100 + set a(-fakeheap) 0 foreach {s val} $args { if {[info exists a($s)]==0} { @@ -143,14 +145,20 @@ proc do_sorter_test {tn args} { } set a($s) $val } + if {[permutation] == "memsys3" || [permutation] == "memsys5"} { + set a(-fakeheap) 0 + } + if {$a(-fakeheap)} { sorter_test_fakeheap 1 } + db eval "PRAGMA cache_size = $a(-cachesize)" - do_test $tn [subst -nocommands { for {set i 0} {[set i] < $a(-repeats)} {incr i} { sorter_test $a(-rows) $a(-read) $a(-payload) } }] {} + + if {$a(-fakeheap)} { sorter_test_fakeheap 0 } } proc clock_seconds {} { @@ -168,12 +176,14 @@ do_test 1 { set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT] -for {set tn 2} {1} {incr tn} { - do_sorter_test $tn.2 -repeats 10 -rows 1000 -read 100 - do_sorter_test $tn.3 -repeats 10 -rows 100000 -read 1000 - do_sorter_test $tn.4 -repeats 10 -rows 100000 -read 1000 -payload 500 - do_sorter_test $tn.5 -repeats 10 -rows 100000 -read 100000 -payload 8 - do_sorter_test $tn.6 -repeats 10 -rows 100000 -read 10 -payload 8 +for {set t 2} {1} {incr tn} { + do_sorter_test $t.2 -repeats 10 -rows 1000 -read 100 + do_sorter_test $t.3 -repeats 10 -rows 100000 -read 1000 + do_sorter_test $t.4 -repeats 10 -rows 100000 -read 1000 -payload 500 + do_sorter_test $t.5 -repeats 10 -rows 100000 -read 100000 -payload 8 + do_sorter_test $t.6 -repeats 10 -rows 100000 -read 10 -payload 8 + do_sorter_test $t.7 -repeats 10 -rows 10000 -read 10000 -payload 8 -fakeheap 1 + do_sorter_test $t.8 -repeats 10 -rows 100000 -read 10000 -cachesize 250 set iNow [clock_seconds] if {$iNow>=$iTimeLimit} break From df24cf81a6ccd41a75c348e8e9c342e502527c5c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 8 May 2014 22:01:08 +0000 Subject: [PATCH 70/99] Fix static variable declaration issue on Windows. FossilOrigin-Name: a41d29691307067523c8637b486941c5f7c33775 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e4b717ce1a..4aa2464bf2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\slittle\sextra\svariety\sto\sthe\stests\sin\ssort4.test. -D 2014-05-06T16:21:30.441 +C Fix\sstatic\svariable\sdeclaration\sissue\son\sWindows. +D 2014-05-08T22:01:08.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -199,7 +199,7 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c -F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 +F src/mutex_w32.c c4726d3bfe80996665a39d27598ebd056e2d6b89 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 208b2b04d4d282bec4424ea7160a123ba549d118 -R b3799ca8720be78abdc748ed84ab7526 -U dan -Z f689b9a36b0734ece63c5c5e094898f6 +P 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa +R 117bf7d69b405485b3cb8ed74140d14c +U mistachkin +Z 6be3c46bf656f034e9d870a5f4b03d3e diff --git a/manifest.uuid b/manifest.uuid index 0fc1ce1baf..80fca0f480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa \ No newline at end of file +a41d29691307067523c8637b486941c5f7c33775 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 7c17cc501e..05d2d5516b 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -93,7 +93,7 @@ static int winMutexNotheld(sqlite3_mutex *p){ /* ** Initialize and deinitialize the mutex subsystem. */ -static sqlite3_mutex winMutex_staticMutexes[6] = { +static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, From dcb1a840ff101edd3b3d7388aba431fcf733d68e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 May 2014 11:15:57 +0000 Subject: [PATCH 71/99] Add new static mutex SQLITE_MUTEX_STATIC_APP3. FossilOrigin-Name: ee0ab09c80a648e9202757fc04122952375e7c8c --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/mutex_noop.c | 2 +- src/mutex_unix.c | 2 ++ src/mutex_w32.c | 2 ++ src/sqlite.h.in | 1 + 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 4aa2464bf2..60a99091c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sstatic\svariable\sdeclaration\sissue\son\sWindows. -D 2014-05-08T22:01:08.441 +C Add\snew\sstatic\smutex\sSQLITE_MUTEX_STATIC_APP3. +D 2014-05-09T11:15:57.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -197,9 +197,9 @@ F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 -F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c -F src/mutex_w32.c c4726d3bfe80996665a39d27598ebd056e2d6b89 +F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 +F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_w32.c 6509b34042b0a8cdd8ea849f5987e187a969f225 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c a5ed3fdc82ebab5b9b095ea1971515a7f8a303d2 F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 -F src/sqlite.h.in ed6d0cc90da850340c3863c84351e6e164c0ef00 +F src/sqlite.h.in 3f3934dd2ff0adbd79d259fbbb2eee38a2c12367 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa -R 117bf7d69b405485b3cb8ed74140d14c -U mistachkin -Z 6be3c46bf656f034e9d870a5f4b03d3e +P a41d29691307067523c8637b486941c5f7c33775 +R dfeb990bf911fd0420714aaa7823f840 +U dan +Z 6074041396d198afcfcf58f3847bacd2 diff --git a/manifest.uuid b/manifest.uuid index 80fca0f480..a410acecfb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a41d29691307067523c8637b486941c5f7c33775 \ No newline at end of file +ee0ab09c80a648e9202757fc04122952375e7c8c \ No newline at end of file diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 4cfc6f3a76..1a900c225a 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[8]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: diff --git a/src/mutex_unix.c b/src/mutex_unix.c index ea1203c047..c8663144e8 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -102,6 +102,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -137,6 +138,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 05d2d5516b..9a1663e596 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -101,6 +101,7 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; static int winMutex_isInit = 0; @@ -167,6 +168,7 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2f39101a31..d3fe5694d8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6039,6 +6039,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection From b0f935e4da79d87c5f2fb1c518bf48d24d7e865f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 May 2014 15:30:00 +0000 Subject: [PATCH 72/99] In the sorter, only use large memory allocations if scratch memory has not been configured. Add #ifdefs to disable unused code when SQLITE_MAX_WORKER_THREADS is zero. Other sorter changes in support of testability. FossilOrigin-Name: d7e2b0d9cb099eda3341bc934bedff9facfe88bd --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/main.c | 2 ++ src/vdbesort.c | 53 ++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 7252ec3ac5..0a8a624dbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-05-09T15:00:32.924 +C In\sthe\ssorter,\sonly\suse\slarge\smemory\sallocations\sif\sscratch\smemory\shas\snot\nbeen\sconfigured.\s\sAdd\s#ifdefs\sto\sdisable\sunused\scode\swhen\s\nSQLITE_MAX_WORKER_THREADS\sis\szero.\s\sOther\ssorter\schanges\sin\ssupport\nof\stestability. +D 2014-05-12T15:30:00.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 2ba4f6eeb84cf8efd8bb632c1abe203dc86adcac +F src/main.c 274a72f5de174a8eb20d01bbd83393347c092592 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b36070436d33372237541e9eaa6068e1a61bc402 +F src/vdbesort.c 0daa029978b50d9c1d103d179ac1e2b38d9daa20 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P ee0ab09c80a648e9202757fc04122952375e7c8c 116bed5af664899a73b46dca528ac0c021fc50c3 -R e8e0d3ce5fd31e02042e8e0c64134243 -U dan -Z b9842070469fd517448a2bc084149835 +P 9ac8f1e7115bc50663235adedeb0d3e1234c5740 +R 6fb72f0b0bf6c011a8428254c9753d68 +U drh +Z 8d87c7cf2c9f964aeaecc8906e4a1927 diff --git a/manifest.uuid b/manifest.uuid index cc8556fc36..695d315026 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ac8f1e7115bc50663235adedeb0d3e1234c5740 \ No newline at end of file +d7e2b0d9cb099eda3341bc934bedff9facfe88bd \ No newline at end of file diff --git a/src/main.c b/src/main.c index d76bcb607e..4f8a128508 100644 --- a/src/main.c +++ b/src/main.c @@ -516,9 +516,11 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_WORKER_THREADS: { +#if SQLITE_MAX_WORKER_THREADS>0 int n = va_arg(ap, int); if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; if( n>=0 ) sqlite3GlobalConfig.nWorker = n; +#endif break; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 9c57135dd3..01404e4f12 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -449,7 +449,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - if( pIter->pIncr ) vdbeIncrFree(pIter->pIncr); + vdbeIncrFree(pIter->pIncr); memset(pIter, 0, sizeof(PmaReader)); } @@ -828,7 +828,11 @@ int sqlite3VdbeSorterInit( int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; +#if SQLITE_MAX_WORKER_THREADS==0 + const int nWorker = 0; +#else int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); +#endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); @@ -858,10 +862,13 @@ int sqlite3VdbeSorterInit( if( mxCachemxPmaSize = mxCache * pgsz; - /* If the application is using memsys3 or memsys5, use a separate - ** allocation for each sort-key in memory. Otherwise, use a single big - ** allocation at pSorter->aMemory for all sort-keys. */ - if( sqlite3GlobalConfig.pHeap==0 ){ + /* If the application has not configure scratch memory using + ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory + ** allocations. If scratch memory has been configured, then assume + ** large memory allocations should be avoided to prevent heap + ** fragmentation. + */ + if( sqlite3GlobalConfig.pScratch==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); @@ -1921,7 +1928,15 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ int mxSz = pIncr->mxSz; - if( pIncr->bUseThread==0 ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ if( pTask->file2.pFd==0 ){ assert( pTask->file2.iEof>0 ); rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); @@ -1932,14 +1947,10 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ pIncr->iStartOff = pTask->file2.iEof; pTask->file2.iEof += mxSz; } - }else{ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); - } } } +#if SQLITE_MAX_WORKER_THREADS>0 if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this ** iterator is multi-threaded. The reason being that this function @@ -1947,6 +1958,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); } +#endif if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ rc = vdbePmaReaderNext(pIter); @@ -2111,6 +2123,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ int rc = SQLITE_OK; int iTask; +#if SQLITE_MAX_WORKER_THREADS>0 /* If the sorter uses more than one task, then create the top-level ** MergeEngine here. This MergeEngine will read data from exactly ** one PmaReader per sub-task. */ @@ -2119,6 +2132,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ pMain = vdbeMergeEngineNew(pSorter->nTask); if( pMain==0 ) rc = SQLITE_NOMEM; } +#endif for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; @@ -2301,10 +2315,13 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ assert( pSorter->pReader==0 || pSorter->pMerger==0 ); assert( pSorter->bUseThreads==0 || pSorter->pReader ); assert( pSorter->bUseThreads==1 || pSorter->pMerger ); +#if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->bUseThreads ){ rc = vdbePmaReaderNext(pSorter->pReader); *pbEof = (pSorter->pReader->pFile==0); - }else{ + }else +#endif + /*if( !pSorter->bUseThreads )*/ { rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); } }else{ @@ -2328,9 +2345,15 @@ static void *vdbeSorterRowkey( ){ void *pKey; if( pSorter->bUsePMA ){ - PmaReader *pReader = (pSorter->bUseThreads ? - pSorter->pReader : &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]] - ); + PmaReader *pReader; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + pReader = pSorter->pReader; + }else +#endif + /*if( !pSorter->bUseThreads )*/{ + pReader = &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]]; + } *pnKey = pReader->nKey; pKey = pReader->aKey; }else{ From 6cc375938318a5a68a2fe5e6a79bcc6beb208b38 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 May 2014 16:56:56 +0000 Subject: [PATCH 73/99] Use #ifdef to omit code that is not used when SQLITE_MAX_WORKER_THREADS is zero. FossilOrigin-Name: 2e8d287d4f41d395a488bf65b20e956b0e311177 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0a8a624dbd..2bf8b35c71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\ssorter,\sonly\suse\slarge\smemory\sallocations\sif\sscratch\smemory\shas\snot\nbeen\sconfigured.\s\sAdd\s#ifdefs\sto\sdisable\sunused\scode\swhen\s\nSQLITE_MAX_WORKER_THREADS\sis\szero.\s\sOther\ssorter\schanges\sin\ssupport\nof\stestability. -D 2014-05-12T15:30:00.944 +C Use\s#ifdef\sto\somit\scode\sthat\sis\snot\sused\swhen\sSQLITE_MAX_WORKER_THREADS\sis\szero. +D 2014-05-15T16:56:56.755 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0daa029978b50d9c1d103d179ac1e2b38d9daa20 +F src/vdbesort.c 0cb40c336c04e2ada936fc4e17a6c6cb885af8c6 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 9ac8f1e7115bc50663235adedeb0d3e1234c5740 -R 6fb72f0b0bf6c011a8428254c9753d68 +P d7e2b0d9cb099eda3341bc934bedff9facfe88bd +R e9a885bf89bf1a3baa55ce358814bd90 U drh -Z 8d87c7cf2c9f964aeaecc8906e4a1927 +Z 872834e0bfb672176645b5e47ef04abb diff --git a/manifest.uuid b/manifest.uuid index 695d315026..9ae9c294db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7e2b0d9cb099eda3341bc934bedff9facfe88bd \ No newline at end of file +2e8d287d4f41d395a488bf65b20e956b0e311177 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 01404e4f12..ef8e7f6349 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1077,11 +1077,14 @@ static void vdbeIncrFree(IncrMerger *pIncr){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + assert( pSorter->bUseThreads || pSorter->pReader==0 ); +#if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->pReader ){ vdbePmaReaderClear(pSorter->pReader); sqlite3DbFree(db, pSorter->pReader); pSorter->pReader = 0; } +#endif vdbeMergeEngineFree(pSorter->pMerger); pSorter->pMerger = 0; for(i=0; inTask; i++){ From 8f0dab37cb6800ade1d73386009cc8384240c6bb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 12:18:08 +0000 Subject: [PATCH 74/99] Use #ifdef logic to avoid an always-true branch when SQLITE_MAX_WORKER_THREADS=0 FossilOrigin-Name: 88cfe6d7de5f19f484304d0db585eac5de6c00ae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2bf8b35c71..5971b7a8b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\s#ifdef\sto\somit\scode\sthat\sis\snot\sused\swhen\sSQLITE_MAX_WORKER_THREADS\sis\szero. -D 2014-05-15T16:56:56.755 +C Use\s#ifdef\slogic\sto\savoid\san\salways-true\sbranch\swhen\nSQLITE_MAX_WORKER_THREADS=0 +D 2014-05-16T12:18:08.658 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0cb40c336c04e2ada936fc4e17a6c6cb885af8c6 +F src/vdbesort.c 7946115f7c3e0d6156121d5e39512e3e980da134 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P d7e2b0d9cb099eda3341bc934bedff9facfe88bd -R e9a885bf89bf1a3baa55ce358814bd90 +P 2e8d287d4f41d395a488bf65b20e956b0e311177 +R e7717e2e9621a181baf0cb2712e277d2 U drh -Z 872834e0bfb672176645b5e47ef04abb +Z 12a20c396c3fdadc5d2762a7d632efff diff --git a/manifest.uuid b/manifest.uuid index 9ae9c294db..ec40eff80f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e8d287d4f41d395a488bf65b20e956b0e311177 \ No newline at end of file +88cfe6d7de5f19f484304d0db585eac5de6c00ae \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ef8e7f6349..3bd19e1c36 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -829,7 +829,7 @@ int sqlite3VdbeSorterInit( int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 - const int nWorker = 0; +# define nWorker 0 #else int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); #endif @@ -879,6 +879,7 @@ int sqlite3VdbeSorterInit( return rc; } +#undef nWorker /* Defined at the top of this function */ /* ** Free the list of sorted records starting at pRecord. @@ -1806,9 +1807,8 @@ static int vdbeIncrNew( IncrMerger **ppOut ){ int rc = SQLITE_OK; - IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); + IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3MallocZero(sizeof(*pIncr)); if( pIncr ){ - memset(pIncr, 0, sizeof(IncrMerger)); pIncr->pMerger = pMerger; pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); From 190d6959673c8855243911808b9c5efd98cf7764 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 17:31:42 +0000 Subject: [PATCH 75/99] Add a pair of sqlite3FaultSim(100) calls to vdbesort.c to facilitate testing of obscure and hard to reach error conditions. FossilOrigin-Name: cceac14fd83ddd8f868c1767cdc66635607cb159 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 008a8c89ec..963e9310be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\schanges\sfrom\strunk,\sand\sespecially\sthe\snew\ssqlite3FaultSim()\ninterface. -D 2014-05-16T14:27:05.717 +C Add\sa\spair\sof\ssqlite3FaultSim(100)\scalls\sto\svdbesort.c\sto\sfacilitate\stesting\nof\sobscure\sand\shard\sto\sreach\serror\sconditions. +D 2014-05-16T17:31:42.435 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 7946115f7c3e0d6156121d5e39512e3e980da134 +F src/vdbesort.c f126a837c4bea832f3b91796081131b22d7b9c7e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 88cfe6d7de5f19f484304d0db585eac5de6c00ae 0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 -R ab0eda2a23c6a8769439757acdbfc7f8 +P 43fcbd9116401f30781fdcbe55d1674d6b96311b +R 2be9e504fbe8e48aafd8e17341d9b966 U drh -Z b6ed5f464b035536d72301437f5c1385 +Z 9396ea859eaf958c2b1d37dca9017593 diff --git a/manifest.uuid b/manifest.uuid index 48bb400eb1..5102b5e8a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43fcbd9116401f30781fdcbe55d1674d6b96311b \ No newline at end of file +cceac14fd83ddd8f868c1767cdc66635607cb159 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 3bd19e1c36..7c844065b4 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1032,7 +1032,7 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ while( NnTree = N; pNew->aIter = (PmaReader*)&pNew[1]; @@ -1807,7 +1807,8 @@ static int vdbeIncrNew( IncrMerger **ppOut ){ int rc = SQLITE_OK; - IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3MallocZero(sizeof(*pIncr)); + IncrMerger *pIncr = *ppOut = (IncrMerger*) + (sqlite3FaultSim(100) ? 0 : sqlite3MallocZero(sizeof(*pIncr))); if( pIncr ){ pIncr->pMerger = pMerger; pIncr->pTask = pTask; From 5f4a4790485f14102c55ba4ea38413b554e5e275 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 20:24:51 +0000 Subject: [PATCH 76/99] Rearrange some conditionals and add #if statements to make the code more testable. FossilOrigin-Name: 17afd77057f8695733a9a60225646c1d8813b1a0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- src/vdbesort.c | 29 ++++++++++++++++++++--------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 963e9310be..01d72475f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\spair\sof\ssqlite3FaultSim(100)\scalls\sto\svdbesort.c\sto\sfacilitate\stesting\nof\sobscure\sand\shard\sto\sreach\serror\sconditions. -D 2014-05-16T17:31:42.435 +C Rearrange\ssome\sconditionals\sand\sadd\s#if\sstatements\sto\smake\sthe\scode\smore\ntestable. +D 2014-05-16T20:24:51.024 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,14 +282,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 89ab2ded5123e823b47293aedd7931a4742fb6bd +F src/vdbe.c 4d11273a6a6644d948c534a17bdeaf42a2793d30 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c f126a837c4bea832f3b91796081131b22d7b9c7e +F src/vdbesort.c 15d1405fcfbb2f4a2ad667872be211153d69fa36 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 43fcbd9116401f30781fdcbe55d1674d6b96311b -R 2be9e504fbe8e48aafd8e17341d9b966 +P cceac14fd83ddd8f868c1767cdc66635607cb159 +R 48ad2b172d9e67d362d3e27e91c64651 U drh -Z 9396ea859eaf958c2b1d37dca9017593 +Z 9751e875adf689a22af045a3a24a99e5 diff --git a/manifest.uuid b/manifest.uuid index 5102b5e8a8..d6d88fd0bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cceac14fd83ddd8f868c1767cdc66635607cb159 \ No newline at end of file +17afd77057f8695733a9a60225646c1d8813b1a0 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b8bf727b53..a2a59406a4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1128,7 +1128,7 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + sqlite3VdbeMemRelease(pOut); zMalloc = pOut->zMalloc; memcpy(pOut, pIn1, sizeof(Mem)); #ifdef SQLITE_DEBUG diff --git a/src/vdbesort.c b/src/vdbesort.c index 7c844065b4..aa08879aef 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -108,8 +108,9 @@ ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged -** incrementally as keys are retreived from the sorter by the VDBE. See -** comments above object MergeEngine below for details. +** incrementally as keys are retreived from the sorter by the VDBE. The +** MergeEngine object, described in further detail below, performs this +** merge. ** ** Or, if running in multi-threaded mode, then a background thread is ** launched to merge the existing PMAs. Once the background thread has @@ -900,11 +901,17 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); pTask->pUnpacked = 0; - if( pTask->list.aMemory==0 ){ - vdbeSorterRecordFree(0, pTask->list.pList); - }else{ +#if SQLITE_MAX_WORKER_THREADS>0 + /* pTask->list.aMemory can only be non-zero if it was handed memory + ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ + if( pTask->list.aMemory ){ sqlite3_free(pTask->list.aMemory); pTask->list.aMemory = 0; + }else +#endif + { + assert( pTask->list.aMemory==0 ); + vdbeSorterRecordFree(0, pTask->list.pList); } pTask->list.pList = 0; if( pTask->file.pFd ){ @@ -2138,7 +2145,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } #endif - for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; if( pTask->nPMA ){ MergeEngine *pRoot = 0; /* Root node of tree for this task */ @@ -2165,10 +2172,14 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } if( rc==SQLITE_OK ){ - if( pMain==0 ){ - pMain = pRoot; - }else{ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pMain!=0 ){ rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + }else +#endif + { + assert( pMain==0 ); + pMain = pRoot; } }else{ vdbeMergeEngineFree(pRoot); From de823bedefdbadcffb818c5bafdd90f344fc94bb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 May 2014 11:03:53 +0000 Subject: [PATCH 77/99] In vdbesort.c, change the names of PmaReader variables "pIter" to "pReadr". Other related comment changes. The generated object code should be the same. FossilOrigin-Name: 99efb235a08784020535a770d7e7d96d9ccee12d --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 330 ++++++++++++++++++++++++------------------------- 3 files changed, 172 insertions(+), 172 deletions(-) diff --git a/manifest b/manifest index 01dfb41694..48c7be4c03 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\schanges\sfrom\strunk. -D 2014-05-19T23:17:33.708 +C In\svdbesort.c,\schange\sthe\snames\sof\sPmaReader\svariables\s"pIter"\sto\s"pReadr".\nOther\srelated\scomment\schanges.\s\sThe\sgenerated\sobject\scode\sshould\sbe\sthe\ssame. +D 2014-05-20T11:03:53.818 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 15d1405fcfbb2f4a2ad667872be211153d69fa36 +F src/vdbesort.c a8b8798f23baa4b293804597532a94470e44f248 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1177,7 +1177,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 17afd77057f8695733a9a60225646c1d8813b1a0 8180e320ee4090e41511836678e49a98c0b228e8 -R caf94a64ae09103b6f818ad8457d731e +P 6eefdad946da6a5f4052ac51d327777890fa3f18 +R 7346935112c4ac3d3d573eb5af19111a U drh -Z 654b19280087cc59a60d4e21f62a0c05 +Z 3f1d10c1aba3efc362d8ec22de1a631d diff --git a/manifest.uuid b/manifest.uuid index ea7a5b8d59..0e33e66f25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6eefdad946da6a5f4052ac51d327777890fa3f18 \ No newline at end of file +99efb235a08784020535a770d7e7d96d9ccee12d \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index aa08879aef..e9776914c2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -188,11 +188,11 @@ struct SorterList { ** combined into one big PMA in order to be able to step through the sorted ** records in order. ** -** The aIter[] array contains a PmaReader object for each of the PMAs being -** merged. An aIter[] object either points to a valid key or else is at EOF. +** The aReadr[] array contains a PmaReader object for each of the PMAs being +** merged. An aReadr[] object either points to a valid key or else is at EOF. ** For the purposes of the paragraphs below, we assume that the array is ** actually N elements in size, where N is the smallest power of 2 greater -** to or equal to the number of PMAs being merged. The extra aIter[] elements +** to or equal to the number of PMAs being merged. The extra aReadr[] elements ** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in @@ -200,7 +200,7 @@ struct SorterList { ** ** The final (N/2) elements of aTree[] contain the results of comparing ** pairs of PMA keys together. Element i contains the result of -** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** ** For the purposes of this comparison, EOF is considered greater than any @@ -208,34 +208,34 @@ struct SorterList { ** values), it doesn't matter which index is stored. ** ** The (N/4) elements of aTree[] that precede the final (N/2) described -** above contains the index of the smallest of each block of 4 iterators. -** And so on. So that aTree[1] contains the index of the iterator that +** above contains the index of the smallest of each block of 4 PmaReaders +** And so on. So that aTree[1] contains the index of the PmaReader that ** currently points to the smallest key value. aTree[0] is unused. ** ** Example: ** -** aIter[0] -> Banana -** aIter[1] -> Feijoa -** aIter[2] -> Elderberry -** aIter[3] -> Currant -** aIter[4] -> Grapefruit -** aIter[5] -> Apple -** aIter[6] -> Durian -** aIter[7] -> EOF +** aReadr[0] -> Banana +** aReadr[1] -> Feijoa +** aReadr[2] -> Elderberry +** aReadr[3] -> Currant +** aReadr[4] -> Grapefruit +** aReadr[5] -> Apple +** aReadr[6] -> Durian +** aReadr[7] -> EOF ** ** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } ** ** The current element is "Apple" (the value of the key indicated by -** iterator 5). When the Next() operation is invoked, iterator 5 will +** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will ** be advanced to the next key in its segment. Say the next key is ** "Eggplant": ** -** aIter[5] -> Eggplant +** aReadr[5] -> Eggplant ** -** The contents of aTree[] are updated first by comparing the new iterator -** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** The contents of aTree[] are updated first by comparing the new PmaReader +** 5 key to the current key of PmaReader 4 (still "Grapefruit"). The PmaReader ** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. -** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** The value of PmaReader 6 - "Durian" - is now smaller than that of PmaReader ** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (BananaaAlloc); - sqlite3_free(pIter->aBuffer); - if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - vdbeIncrFree(pIter->pIncr); - memset(pIter, 0, sizeof(PmaReader)); +static void vdbePmaReaderClear(PmaReader *pReadr){ + sqlite3_free(pReadr->aAlloc); + sqlite3_free(pReadr->aBuffer); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + vdbeIncrFree(pReadr->pIncr); + memset(pReadr, 0, sizeof(PmaReader)); } /* @@ -464,7 +464,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ ** next call to this function. */ static int vdbePmaReadBlob( - PmaReader *p, /* Iterator */ + PmaReader *p, /* PmaReader from which to take the blob */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ @@ -495,7 +495,7 @@ static int vdbePmaReadBlob( } assert( nRead>0 ); - /* Read data from the file. Return early if an error occurs. */ + /* Readr data from the file. Return early if an error occurs. */ rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; @@ -599,43 +599,43 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ } /* -** Seek iterator pIter to offset iOff within file pFile. Return SQLITE_OK +** Seek PmaReader pReadr to offset iOff within file pFile. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ static int vdbePmaReaderSeek( SortSubtask *pTask, /* Task context */ - PmaReader *pIter, /* Iterate to populate */ + PmaReader *pReadr, /* Iterate to populate */ SorterFile *pFile, /* Sorter file to read from */ i64 iOff /* Offset in pFile */ ){ int rc = SQLITE_OK; - assert( pIter->pIncr==0 || pIter->pIncr->bEof==0 ); + assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); - if( pIter->aMap ){ - sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - pIter->aMap = 0; + if( pReadr->aMap ){ + sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + pReadr->aMap = 0; } - pIter->iReadOff = iOff; - pIter->iEof = pFile->iEof; - pIter->pFile = pFile->pFd; + pReadr->iReadOff = iOff; + pReadr->iEof = pFile->iEof; + pReadr->pFile = pFile->pFd; - rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); - if( rc==SQLITE_OK && pIter->aMap==0 ){ + rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); + if( rc==SQLITE_OK && pReadr->aMap==0 ){ int pgsz = pTask->pSorter->pgsz; - int iBuf = pIter->iReadOff % pgsz; - if( pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pgsz; + int iBuf = pReadr->iReadOff % pgsz; + if( pReadr->aBuffer==0 ){ + pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM; + pReadr->nBuffer = pgsz; } if( rc==SQLITE_OK && iBuf ){ int nRead = pgsz - iBuf; - if( (pIter->iReadOff + nRead) > pIter->iEof ){ - nRead = (int)(pIter->iEof - pIter->iReadOff); + if( (pReadr->iReadOff + nRead) > pReadr->iEof ){ + nRead = (int)(pReadr->iEof - pReadr->iReadOff); } rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } @@ -645,22 +645,22 @@ static int vdbePmaReaderSeek( } /* -** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if +** Advance PmaReader pReadr to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbePmaReaderNext(PmaReader *pIter){ +static int vdbePmaReaderNext(PmaReader *pReadr){ int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ - if( pIter->iReadOff>=pIter->iEof ){ - IncrMerger *pIncr = pIter->pIncr; + if( pReadr->iReadOff>=pReadr->iEof ){ + IncrMerger *pIncr = pReadr->pIncr; int bEof = 1; if( pIncr ){ rc = vdbeIncrSwap(pIncr); if( rc==SQLITE_OK && pIncr->bEof==0 ){ rc = vdbePmaReaderSeek( - pIncr->pTask, pIter, &pIncr->aFile[0], pIncr->iStartOff + pIncr->pTask, pReadr, &pIncr->aFile[0], pIncr->iStartOff ); bEof = 0; } @@ -668,26 +668,26 @@ static int vdbePmaReaderNext(PmaReader *pIter){ if( bEof ){ /* This is an EOF condition */ - vdbePmaReaderClear(pIter); + vdbePmaReaderClear(pReadr); return rc; } } if( rc==SQLITE_OK ){ - rc = vdbePmaReadVarint(pIter, &nRec); + rc = vdbePmaReadVarint(pReadr, &nRec); } if( rc==SQLITE_OK ){ - pIter->nKey = (int)nRec; - rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); + pReadr->nKey = (int)nRec; + rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); } return rc; } /* -** Initialize iterator pIter to scan through the PMA stored in file pFile +** Initialize PmaReader pReadr to scan through the PMA stored in file pFile ** starting at offset iStart and ending at offset iEof-1. This function -** leaves the iterator pointing to the first key in the PMA (or EOF if the +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the ** PMA is empty). ** ** If the pnByte parameter is NULL, then it is assumed that the file @@ -697,26 +697,26 @@ static int vdbePmaReaderInit( SortSubtask *pTask, /* Task context */ SorterFile *pFile, /* Sorter file to read from */ i64 iStart, /* Start offset in pFile */ - PmaReader *pIter, /* Iterator to populate */ + PmaReader *pReadr, /* PmaReader to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc; assert( pFile->iEof>iStart ); - assert( pIter->aAlloc==0 && pIter->nAlloc==0 ); - assert( pIter->aBuffer==0 ); - assert( pIter->aMap==0 ); + assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); + assert( pReadr->aBuffer==0 ); + assert( pReadr->aMap==0 ); - rc = vdbePmaReaderSeek(pTask, pIter, pFile, iStart); + rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - rc = vdbePmaReadVarint(pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; + rc = vdbePmaReadVarint(pReadr, &nByte); + pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderNext(pIter); + rc = vdbePmaReaderNext(pReadr); } return rc; } @@ -748,7 +748,7 @@ static int vdbeSorterCompare( } /* -** This function is called to compare two iterator keys when merging +** This function is called to compare two PmaReader keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ @@ -773,8 +773,8 @@ static int vdbeSorterDoCompare( i2 = pMerger->aTree[iOut*2+1]; } - p1 = &pMerger->aIter[i1]; - p2 = &pMerger->aIter[i2]; + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; if( p1->pFile==0 ){ iRes = i2; @@ -1027,23 +1027,23 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new MergeEngine object with space for nIter iterators. +** Allocate a new MergeEngine object with space for nReader PmaReaders. */ -static MergeEngine *vdbeMergeEngineNew(int nIter){ - int N = 2; /* Smallest power of two >= nIter */ +static MergeEngine *vdbeMergeEngineNew(int nReader){ + int N = 2; /* Smallest power of two >= nReader */ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - assert( nIter<=SORTER_MAX_MERGE_COUNT ); + assert( nReader<=SORTER_MAX_MERGE_COUNT ); - while( NnTree = N; - pNew->aIter = (PmaReader*)&pNew[1]; - pNew->aTree = (int*)&pNew->aIter[N]; + pNew->aReadr = (PmaReader*)&pNew[1]; + pNew->aTree = (int*)&pNew->aReadr[N]; } return pNew; } @@ -1055,7 +1055,7 @@ static void vdbeMergeEngineFree(MergeEngine *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ - vdbePmaReaderClear(&pMerger->aIter[i]); + vdbePmaReaderClear(&pMerger->aReadr[i]); } } sqlite3_free(pMerger); @@ -1439,8 +1439,8 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine iterator passed as the second argument to -** the next entry. Set *pbEof to true if this means the iterator has +** Advance the MergeEngine PmaReader passed as the second argument to +** the next entry. Set *pbEof to true if this means the PmaReader has ** reached EOF. ** ** Return SQLITE_OK if successful or an error code if an error occurs. @@ -1451,63 +1451,63 @@ static int vdbeSorterNext( int *pbEof ){ int rc; - int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ + int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ - /* Advance the current iterator */ - rc = vdbePmaReaderNext(&pMerger->aIter[iPrev]); + /* Advance the current PmaReader */ + rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); /* Update contents of aTree[] */ if( rc==SQLITE_OK ){ int i; /* Index of aTree[] to recalculate */ - PmaReader *pIter1; /* First iterator to compare */ - PmaReader *pIter2; /* Second iterator to compare */ - u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + PmaReader *pReadr1; /* First PmaReader to compare */ + PmaReader *pReadr2; /* Second PmaReader to compare */ + u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ - /* Find the first two iterators to compare. The one that was just + /* Find the first two PmaReaders to compare. The one that was just ** advanced (iPrev) and the one next to it in the array. */ - pIter1 = &pMerger->aIter[(iPrev & 0xFFFE)]; - pIter2 = &pMerger->aIter[(iPrev | 0x0001)]; - pKey2 = pIter2->aKey; + pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; + pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; + pKey2 = pReadr2->aKey; for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ - /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ int iRes; - if( pIter1->pFile==0 ){ + if( pReadr1->pFile==0 ){ iRes = +1; - }else if( pIter2->pFile==0 ){ + }else if( pReadr2->pFile==0 ){ iRes = -1; }else{ iRes = vdbeSorterCompare(pTask, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey + pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey ); } - /* If pIter1 contained the smaller value, set aTree[i] to its index. - ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pTask->pUnpacked, so set - ** pKey2 to point to the record belonging to pIter2. + /* If pReadr1 contained the smaller value, set aTree[i] to its index. + ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this + ** case there is no cache of pReadr2 in pTask->pUnpacked, so set + ** pKey2 to point to the record belonging to pReadr2. ** - ** Alternatively, if pIter2 contains the smaller of the two values, - ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** Alternatively, if pReadr2 contains the smaller of the two values, + ** set aTree[i] to its index and update pReadr1. If vdbeSorterCompare() ** was actually called above, then pTask->pUnpacked now contains - ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. + ** a value equivalent to pReadr2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pReadr2 again. ** ** If the two values were equal, then the value from the oldest - ** PMA should be considered smaller. The VdbeSorter.aIter[] array - ** is sorted from oldest to newest, so pIter1 contains older values - ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pMerger->aIter); - pIter2 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; - pKey2 = pIter2->aKey; + ** PMA should be considered smaller. The VdbeSorter.aReadr[] array + ** is sorted from oldest to newest, so pReadr1 contains older values + ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); + pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + pKey2 = pReadr2->aKey; }else{ - if( pIter1->pFile ) pKey2 = 0; - pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); - pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + if( pReadr1->pFile ) pKey2 = 0; + pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); + pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } } - *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFile==0); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); @@ -1709,7 +1709,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; - PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; + PmaReader *pReader = &pMerger->aReadr[ pMerger->aTree[1] ]; int nKey = pReader->nKey; i64 iEof = writer.iWriteOff + writer.iBufEnd; @@ -1755,7 +1755,7 @@ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ /* ** This function is called when the PmaReader corresponding to pIncr has ** finished reading the contents of aFile[0]. Its purpose is to "refill" -** aFile[0] such that the iterator should start rereading it from the +** aFile[0] such that the PmaReader should start rereading it from the ** beginning. ** ** For single-threaded objects, this is accomplished by literally reading @@ -1841,7 +1841,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* ** Initialize the merger argument passed as the second argument. Once this @@ -1852,7 +1852,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); ** objects attached to the PmaReader objects that the merger reads from have ** already been populated, but that they have not yet populated aFile[0] and ** set the PmaReader objects up to read from it. In this case all that is -** required is to call vdbePmaReaderNext() on each iterator to point it at +** required is to call vdbePmaReaderNext() on each PmaReader to point it at ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use @@ -1867,21 +1867,21 @@ static int vdbeIncrInitMerger( int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ - int i; /* For iterating through PmaReader objects */ + int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; for(i=0; rc==SQLITE_OK && iaIter[nTree-i-1]); + rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } } @@ -1894,20 +1894,20 @@ static int vdbeIncrInitMerger( /* ** If the PmaReader passed as the first argument is not an incremental-reader -** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge -** object at (pIter->pIncr). +** object at (pReadr->pIncr). ** -** If argument eMode is set to INCRINIT_NORMAL, then PmaReader iterators -** in the sub-tree headed by pIter are also initialized. Data is then loaded -** into the buffers belonging to this iterator, pIter, and it is set to +** If argument eMode is set to INCRINIT_NORMAL, then PmaReader PmaReaders +** in the sub-tree headed by pReadr are also initialized. Data is then loaded +** into the buffers belonging to this PmaReader, pReadr, and it is set to ** point to the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed -** to be a multi-threaded iterator and this function is being called in a -** background thread. In this case all iterators in the sub-tree are +** to be a multi-threaded PmaReader and this function is being called in a +** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to -** pIter is populated. However, the iterator itself is not set up to point +** pReadr is populated. However, the PmaReader itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do ** that. ** @@ -1918,16 +1918,16 @@ static int vdbeIncrInitMerger( ** lead to the current background thread attempting to join itself. ** ** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed -** that pIter->pIncr is a multi-threaded IncrMerge objects, and that all +** that pReadr->pIncr is a multi-threaded IncrMerge objects, and that all ** child-trees have already been initialized using IncrInit(INCRINIT_TASK). -** In this case vdbePmaReaderNext() is called on all child iterators and -** the current iterator set to point to the first key in its range. +** In this case vdbePmaReaderNext() is called on all child PmaReaders and +** the current PmaReader set to point to the first key in its range. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; - IncrMerger *pIncr = pIter->pIncr; + IncrMerger *pIncr = pReadr->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; @@ -1964,7 +1964,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this - ** iterator is multi-threaded. The reason being that this function + ** PmaReader is multi-threaded. The reason being that this function ** is already running in background thread pIncr->pTask->thread. */ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); @@ -1972,7 +1972,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ #endif if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ - rc = vdbePmaReaderNext(pIter); + rc = vdbePmaReaderNext(pReadr); } } return rc; @@ -1994,13 +1994,13 @@ static void *vdbePmaReaderBgInit(void *pCtx){ ** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** -** This call will initialize the various fields of the pIter->pIncr +** This call will initialize the various fields of the pReadr->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a ** background thread to populate aFile[1]. */ -static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ - void *pCtx = (void*)pIter; - return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbePmaReaderBgInit, pCtx); +static int vdbePmaReaderBgIncrInit(PmaReader *pReadr){ + void *pCtx = (void*)pReadr; + return vdbeSorterCreateThread(pReadr->pIncr->pTask, vdbePmaReaderBgInit, pCtx); } #endif @@ -2019,7 +2019,7 @@ static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ static int vdbeMergeEngineLevel0( SortSubtask *pTask, /* Sorter task to read from */ int nPMA, /* Number of PMAs to read */ - i64 *piOffset, /* IN/OUT: Read offset in pTask->file */ + i64 *piOffset, /* IN/OUT: Readr offset in pTask->file */ MergeEngine **ppOut /* OUT: New merge-engine */ ){ MergeEngine *pNew; /* Merge engine to return */ @@ -2032,9 +2032,9 @@ static int vdbeMergeEngineLevel0( for(i=0; iaIter[i]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pIter, &nDummy); - iOff = pIter->iEof; + PmaReader *pReadr = &pNew->aReadr[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); + iOff = pReadr->iEof; } if( rc!=SQLITE_OK ){ @@ -2094,24 +2094,24 @@ static int vdbeSorterAddToTree( for(i=1; iaIter[iIter]; + PmaReader *pReadr = &p->aReadr[iIter]; - if( pIter->pIncr==0 ){ + if( pReadr->pIncr==0 ){ MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); + rc = vdbeIncrNew(pTask, pNew, &pReadr->pIncr); } } if( rc==SQLITE_OK ){ - p = pIter->pIncr->pMerger; + p = pReadr->pIncr->pMerger; nDiv = nDiv / SORTER_MAX_MERGE_COUNT; } } if( rc==SQLITE_OK ){ - p->aIter[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + p->aReadr[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; }else{ vdbeIncrFree(pIncr); } @@ -2174,7 +2174,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS>0 if( pMain!=0 ){ - rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + rc = vdbeIncrNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); }else #endif { @@ -2218,27 +2218,27 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; - PmaReader *pIter; + PmaReader *pReadr; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ - pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); - pSorter->pReader = pIter; - if( pIter==0 ) rc = SQLITE_NOMEM; + pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReadr; + if( pReadr==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ - rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); + rc = vdbeIncrNew(pLast, pMain, &pReadr->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pIter->pIncr); + vdbeIncrSetThreads(pReadr->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; - if( (pIncr = pMain->aIter[iTask].pIncr) ){ + if( (pIncr = pMain->aReadr[iTask].pIncr) ){ vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; + PmaReader *p = &pMain->aReadr[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); if( p->pIncr ){ if( iTask==pSorter->nTask-1 ){ @@ -2252,7 +2252,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pIter, INCRINIT_ROOT); + rc = vdbePmaReaderIncrInit(pReadr, INCRINIT_ROOT); } }else #endif @@ -2367,7 +2367,7 @@ static void *vdbeSorterRowkey( }else #endif /*if( !pSorter->bUseThreads )*/{ - pReader = &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]]; + pReader = &pSorter->pMerger->aReadr[pSorter->pMerger->aTree[1]]; } *pnKey = pReader->nKey; pKey = pReader->aKey; From cb6effafecd4bbd73a55002e4df081c2e687451b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 May 2014 19:11:50 +0000 Subject: [PATCH 78/99] Improvements to the testability of the threads.c module. FossilOrigin-Name: 386e088868b44b02646e452147838d2e97b093ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 48c7be4c03..86d0d6e9cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\schange\sthe\snames\sof\sPmaReader\svariables\s"pIter"\sto\s"pReadr".\nOther\srelated\scomment\schanges.\s\sThe\sgenerated\sobject\scode\sshould\sbe\sthe\ssame. -D 2014-05-20T11:03:53.818 +C Improvements\sto\sthe\stestability\sof\sthe\sthreads.c\smodule. +D 2014-05-20T19:11:50.626 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -275,7 +275,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b9daffcb6ae3a9c080da37b905e790d45a8f99db +F src/threads.c 3c63f60ce0aae4c40ed4b8dc745490ff42a05308 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -1177,7 +1177,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6eefdad946da6a5f4052ac51d327777890fa3f18 -R 7346935112c4ac3d3d573eb5af19111a +P 99efb235a08784020535a770d7e7d96d9ccee12d +R 41b3a138387644de9249d3787f3c7d11 U drh -Z 3f1d10c1aba3efc362d8ec22de1a631d +Z 514d5c30144e64644ef12bde520c79ea diff --git a/manifest.uuid b/manifest.uuid index 0e33e66f25..bffaf5bd3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99efb235a08784020535a770d7e7d96d9ccee12d \ No newline at end of file +386e088868b44b02646e452147838d2e97b093ee \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 6b59ce90d2..1cfe2cc6ce 100644 --- a/src/threads.c +++ b/src/threads.c @@ -54,15 +54,16 @@ int sqlite3ThreadCreate( assert( ppThread!=0 ); assert( xTask!=0 ); + /* This routine is never used in single-threaded mode */ + assert( sqlite3GlobalConfig.bCoreMutex!=0 ); + *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - if( sqlite3GlobalConfig.bCoreMutex==0 - || pthread_create(&p->tid, 0, xTask, pIn)!=0 - ){ + if( sqlite3FaultSim(200) ? 1 : pthread_create(&p->tid, 0, xTask, pIn) ){ p->done = 1; p->pOut = xTask(pIn); } @@ -80,10 +81,10 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ *ppOut = p->pOut; rc = SQLITE_OK; }else{ - rc = pthread_join(p->tid, ppOut); + rc = pthread_join(p->tid, ppOut) ? SQLITE_ERROR : SQLITE_OK; } sqlite3_free(p); - return rc ? SQLITE_ERROR : SQLITE_OK; + return rc; } #endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ From ac65196e4d002ae36ea82871134f445728342b5b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 14:54:50 +0000 Subject: [PATCH 79/99] Improvements to comments in the multi-threaded sorter. Also include a function name change for clarity. And add a test to help show that the MergeEngine object is only used by a single thread. FossilOrigin-Name: 9af50a878f67c1c2a4f1520160cc989650d7196a --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 55 ++++++++++++++++++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index ab912afdb5..1af171baaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-07-24T16:54:28.599 +C Improvements\sto\scomments\sin\sthe\smulti-threaded\ssorter.\s\sAlso\sinclude\sa\nfunction\sname\schange\sfor\sclarity.\s\sAnd\sadd\sa\stest\sto\shelp\sshow\sthat\sthe\nMergeEngine\sobject\sis\sonly\sused\sby\sa\ssingle\sthread. +D 2014-07-28T14:54:50.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 68ef480fa75b27d5860fb96ca4f5a9af98ba102f F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e2784e2e1f1819a55ce6f22c6ab22eca576ae6d8 +F src/vdbesort.c ef998096c8b2a1a85fbd730183a9b62f652e1af3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ae23a65eb1547fbe8b86ab71477071990a22d31d fb1048cb2b613a0dbfe625a5df05e9dcd736a433 -R 5b949bc8daa338f7894b4848fcc3e5ce +P 770685892c8f09b9cddb2fbb2877cfb291e19425 +R d6f4c6811acb8e4555290f0ceab890b8 U drh -Z 1a0a2770bde58d6f05cf30741962cf0a +Z 8438d80edf2a72a2fe3b270fb06c810d diff --git a/manifest.uuid b/manifest.uuid index d06af4d0d3..64a549188e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -770685892c8f09b9cddb2fbb2877cfb291e19425 \ No newline at end of file +9af50a878f67c1c2a4f1520160cc989650d7196a \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8065f6d982..96d9e2976e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -85,7 +85,7 @@ ** The threshold for the amount of main memory to use before flushing ** records to a PMA is roughly the same as the limit configured for the ** page-cache of the main database. Specifically, the threshold is set to -** the value returned multiplied by "PRAGMA main.page_size" multipled by +** the value returned by "PRAGMA main.page_size" multipled by ** that returned by "PRAGMA main.cache_size", in bytes. ** ** If the sorter is running in single-threaded mode, then all PMAs generated @@ -190,6 +190,7 @@ struct SorterList { ** ** The aReadr[] array contains a PmaReader object for each of the PMAs being ** merged. An aReadr[] object either points to a valid key or else is at EOF. +** ("EOF" means "End Of File". When aReadr[] is at EOF there is no more data.) ** For the purposes of the paragraphs below, we assume that the array is ** actually N elements in size, where N is the smallest power of 2 greater ** to or equal to the number of PMAs being merged. The extra aReadr[] elements @@ -247,15 +248,19 @@ struct SorterList { */ struct MergeEngine { int nTree; /* Used size of aTree/aReadr (power of 2) */ + SortSubtask *pTask; /* Used by this thread only */ int *aTree; /* Current state of incremental merge */ PmaReader *aReadr; /* Array of PmaReaders to merge data from */ }; /* +** This object represents a single thread of control in a sort operation. ** Exactly VdbeSorter.nTask instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any ** other way. VdbeSorter.nTask is set to the number of worker threads allowed -** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). Thus for +** single-threaded operation, there is exactly one instance of this object +** and for multi-threaded operation there are two or more instances. ** ** Essentially, this structure contains all those fields of the VdbeSorter ** structure for which each thread requires a separate instance. For example, @@ -443,7 +448,7 @@ static int vdbeIncrSwap(IncrMerger*); static void vdbeIncrFree(IncrMerger *); /* -** Free all memory belonging to the PmaReader object passed as the second +** Free all memory belonging to the PmaReader object passed as the ** argument. All structure fields are set to zero before returning. */ static void vdbePmaReaderClear(PmaReader *pReadr){ @@ -455,12 +460,12 @@ static void vdbePmaReaderClear(PmaReader *pReadr){ } /* -** Read nByte bytes of data from the stream of data iterated by object p. +** Read the next nByte bytes of data from the PMA p. ** If successful, set *ppOut to point to a buffer containing the data ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite ** error code. ** -** The buffer indicated by *ppOut may only be considered valid until the +** The buffer returned in *ppOut is only valid until the ** next call to this function. */ static int vdbePmaReadBlob( @@ -594,6 +599,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); } return rc; } @@ -604,7 +610,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ */ static int vdbePmaReaderSeek( SortSubtask *pTask, /* Task context */ - PmaReader *pReadr, /* Iterate to populate */ + PmaReader *pReadr, /* Reader whose cursor is to be moved */ SorterFile *pFile, /* Sorter file to read from */ i64 iOff /* Offset in pFile */ ){ @@ -637,6 +643,7 @@ static int vdbePmaReaderSeek( rc = sqlite3OsRead( pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); + testcase( rc!=SQLITE_OK ); } } @@ -668,6 +675,7 @@ static int vdbePmaReaderNext(PmaReader *pReadr){ if( bEof ){ /* This is an EOF condition */ vdbePmaReaderClear(pReadr); + testcase( rc!=SQLITE_OK ); return rc; } } @@ -678,6 +686,7 @@ static int vdbePmaReaderNext(PmaReader *pReadr){ if( rc==SQLITE_OK ){ pReadr->nKey = (int)nRec; rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); + testcase( rc!=SQLITE_OK ); } return rc; @@ -1026,7 +1035,11 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new MergeEngine object with space for nReader PmaReaders. +** Allocate a new MergeEngine object capable of handling up to +** nReader PmaReader inputs. +** +** nReader is automatically rounded up to the next power of two. +** nReader may not exceed SORTER_MAX_MERGE_COUNT even after rounding up. */ static MergeEngine *vdbeMergeEngineNew(int nReader){ int N = 2; /* Smallest power of two >= nReader */ @@ -1041,6 +1054,7 @@ static MergeEngine *vdbeMergeEngineNew(int nReader){ pNew = sqlite3FaultSim(100) ? 0 : (MergeEngine*)sqlite3MallocZero(nByte); if( pNew ){ pNew->nTree = N; + pNew->pTask = 0; pNew->aReadr = (PmaReader*)&pNew[1]; pNew->aTree = (int*)&pNew->aReadr[N]; } @@ -1438,20 +1452,24 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine PmaReader passed as the second argument to -** the next entry. Set *pbEof to true if this means the PmaReader has -** reached EOF. +** Advance the MergeEngine pMerge (passed as the second argument) to +** its next entry. Set *pbEof to true there is no next entry because +** the MergeEngine has reached the end of all its inputs. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ -static int vdbeSorterNext( - SortSubtask *pTask, - MergeEngine *pMerger, - int *pbEof +static int vdbeMergeEngineStep( + SortSubtask *pTask, /* The thread in which this MergeEngine runs */ + MergeEngine *pMerger, /* The merge engine to advance to the next row */ + int *pbEof /* Set TRUE at EOF. Set false for more content */ ){ int rc; int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ + /* A MergeEngine object is only used by a single thread */ + assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + pMerger->pTask = pTask; + /* Advance the current PmaReader */ rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); @@ -1720,7 +1738,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeSorterNext(pTask, pIncr->pMerger, &dummy); + rc = vdbeMergeEngineStep(pTask, pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); @@ -2128,7 +2146,10 @@ static int vdbeSorterAddToTree( ** error occurs, an SQLite error code is returned and the final value ** of *ppOut is undefined. */ -static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ +static int vdbeSorterMergeTreeBuild( + VdbeSorter *pSorter, /* The VDBE cursor that implements the sort */ + MergeEngine **ppOut /* Write the MergeEngine here */ +){ MergeEngine *pMain = 0; int rc = SQLITE_OK; int iTask; @@ -2336,7 +2357,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else #endif /*if( !pSorter->bUseThreads )*/ { - rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + rc = vdbeMergeEngineStep(&pSorter->aTask[0], pSorter->pMerger, pbEof); } }else{ SorterRecord *pFree = pSorter->list.pList; From a4c8ca049e7833e78df61e0f866056b6f39afea5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 17:18:28 +0000 Subject: [PATCH 80/99] In vdbesort.c, rename all pointers to sqlite3_file objects "pFd" and use the name "pFile" only for pointers to SortFile objects. Other comment enhancements. FossilOrigin-Name: 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 --- manifest | 12 ++--- manifest.uuid | 2 +- src/vdbesort.c | 116 ++++++++++++++++++++++++++----------------------- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 006de9f272..cd56be770f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-07-28T15:01:37.313 +C In\svdbesort.c,\srename\sall\spointers\sto\ssqlite3_file\sobjects\s"pFd"\sand\suse\sthe\nname\s"pFile"\sonly\sfor\spointers\sto\sSortFile\sobjects.\s\sOther\scomment\senhancements. +D 2014-07-28T17:18:28.406 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c ef998096c8b2a1a85fbd730183a9b62f652e1af3 +F src/vdbesort.c cbd9f27e3521737ad87149550c6d30994f74df7a F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9af50a878f67c1c2a4f1520160cc989650d7196a 5350229b52b18a4961858a30538c5c75e5bd3048 -R c247ffbe0be6e5ba20860955ecb933f2 +P 163c247bd8280ab14fe577329c631c8bd884707f +R 502fd7906a0bc29e23e7a03a06d6bf2a U drh -Z eec00caa889efc5f1f3b6e693b6885aa +Z e1002bd8dfbb36530151cc4f8c9e6ac6 diff --git a/manifest.uuid b/manifest.uuid index 2637b6c6f7..1b6e273eef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -163c247bd8280ab14fe577329c631c8bd884707f \ No newline at end of file +518290a7fc0994f9593c5c3ba5b2610a1b86dae1 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 96d9e2976e..ce26e9acc8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -327,20 +327,24 @@ struct VdbeSorter { /* ** An instance of the following object is used to read records out of a ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. -** pFile==0 at EOF. +** aKey might point into aMap or into aBuffer. If neither of those locations +** contain a contiguous representation of the key, then aAlloc is allocated +** and the key is copied into aAlloc and aKey is made to poitn to aAlloc. +** +** pFd==0 at EOF. */ struct PmaReader { - i64 iReadOff; /* Current read offset */ - i64 iEof; /* 1 byte past EOF for this PmaReader */ - int nAlloc; /* Bytes of space at aAlloc */ - int nKey; /* Number of bytes in key */ - sqlite3_file *pFile; /* File we are reading from */ - u8 *aAlloc; /* Allocated space */ - u8 *aKey; /* Pointer to current key */ - u8 *aBuffer; /* Current read buffer */ - int nBuffer; /* Size of read buffer in bytes */ - u8 *aMap; /* Pointer to mapping of entire file */ - IncrMerger *pIncr; /* Incremental merger */ + i64 iReadOff; /* Current read offset */ + i64 iEof; /* 1 byte past EOF for this PmaReader */ + int nAlloc; /* Bytes of space at aAlloc */ + int nKey; /* Number of bytes in key */ + sqlite3_file *pFd; /* File handle we are reading from */ + u8 *aAlloc; /* Space for aKey if aBuffer and pMap wont work */ + u8 *aKey; /* Pointer to current key */ + u8 *aBuffer; /* Current read buffer */ + int nBuffer; /* Size of read buffer in bytes */ + u8 *aMap; /* Pointer to mapping of entire file */ + IncrMerger *pIncr; /* Incremental merger */ }; /* @@ -400,7 +404,7 @@ struct PmaWriter { int iBufStart; /* First byte of buffer to write */ int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ - sqlite3_file *pFile; /* File to write to */ + sqlite3_file *pFd; /* File handle to write to */ }; /* @@ -454,7 +458,7 @@ static void vdbeIncrFree(IncrMerger *); static void vdbePmaReaderClear(PmaReader *pReadr){ sqlite3_free(pReadr->aAlloc); sqlite3_free(pReadr->aBuffer); - if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); vdbeIncrFree(pReadr->pIncr); memset(pReadr, 0, sizeof(PmaReader)); } @@ -501,7 +505,7 @@ static int vdbePmaReadBlob( assert( nRead>0 ); /* Readr data from the file. Return early if an error occurs. */ - rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + rc = sqlite3OsRead(p->pFd, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; } @@ -619,12 +623,12 @@ static int vdbePmaReaderSeek( assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); if( pReadr->aMap ){ - sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); pReadr->aMap = 0; } pReadr->iReadOff = iOff; pReadr->iEof = pFile->iEof; - pReadr->pFile = pFile->pFd; + pReadr->pFd = pFile->pFd; rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); if( rc==SQLITE_OK && pReadr->aMap==0 ){ @@ -641,7 +645,7 @@ static int vdbePmaReaderSeek( nRead = (int)(pReadr->iEof - pReadr->iReadOff); } rc = sqlite3OsRead( - pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff + pReadr->pFd, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); testcase( rc!=SQLITE_OK ); } @@ -784,9 +788,9 @@ static int vdbeSorterDoCompare( p1 = &pMerger->aReadr[i1]; p2 = &pMerger->aReadr[i2]; - if( p1->pFile==0 ){ + if( p1->pFd==0 ){ iRes = i2; - }else if( p2->pFile==0 ){ + }else if( p2->pFd==0 ){ iRes = i1; }else{ int res; @@ -1147,13 +1151,13 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ - int rc = sqlite3OsTruncate(pFile, nByte); + int rc = sqlite3OsTruncate(pFd, nByte); if( rc==SQLITE_OK ){ void *p = 0; - sqlite3OsFetch(pFile, 0, (int)nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); } } } @@ -1163,25 +1167,25 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ /* ** Allocate space for a file-handle and open a temporary file. If successful, -** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. -** Otherwise, set *ppFile to 0 and return an SQLite error code. +** set *ppFd to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFd to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile( sqlite3 *db, /* Database handle doing sort */ i64 nExtend, /* Attempt to extend file to this size */ - sqlite3_file **ppFile + sqlite3_file **ppFd ){ int rc; - rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); if( rc==SQLITE_OK ){ i64 max = SQLITE_MAX_MMAP_SIZE; - sqlite3OsFileControlHint(*ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + sqlite3OsFileControlHint(*ppFd, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); if( nExtend>0 ){ - vdbeSorterExtendFile(db, *ppFile, nExtend); + vdbeSorterExtendFile(db, *ppFd, nExtend); } } return rc; @@ -1300,10 +1304,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ ** Initialize a PMA-writer object. */ static void vdbePmaWriterInit( - sqlite3_file *pFile, /* File to write to */ + sqlite3_file *pFd, /* File handle to write to */ PmaWriter *p, /* Object to populate */ int nBuf, /* Buffer size */ - i64 iStart /* Offset of pFile to begin writing at */ + i64 iStart /* Offset of pFd to begin writing at */ ){ memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); @@ -1313,7 +1317,7 @@ static void vdbePmaWriterInit( p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; p->nBuffer = nBuf; - p->pFile = pFile; + p->pFd = pFd; } } @@ -1332,7 +1336,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); p->iBufEnd += nCopy; if( p->iBufEnd==p->nBuffer ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -1357,7 +1361,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -1489,9 +1493,9 @@ static int vdbeMergeEngineStep( for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ int iRes; - if( pReadr1->pFile==0 ){ + if( pReadr1->pFd==0 ){ iRes = +1; - }else if( pReadr2->pFile==0 ){ + }else if( pReadr2->pFd==0 ){ iRes = -1; }else{ iRes = vdbeSorterCompare(pTask, @@ -1519,12 +1523,12 @@ static int vdbeMergeEngineStep( pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; pKey2 = pReadr2->aKey; }else{ - if( pReadr1->pFile ) pKey2 = 0; + if( pReadr1->pFd ) pKey2 = 0; pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } } - *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFile==0); + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); @@ -1732,7 +1736,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Check if the output file is full or if the input has been exhausted. ** In either case exit the loop. */ - if( pReader->pFile==0 ) break; + if( pReader->pFd==0 ) break; if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; /* Write the next key to the output. */ @@ -1826,9 +1830,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ ** pMerger argument before returning. */ static int vdbeIncrNew( - SortSubtask *pTask, - MergeEngine *pMerger, - IncrMerger **ppOut + SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ + MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ + IncrMerger **ppOut /* Write the new IncrMerger here */ ){ int rc = SQLITE_OK; IncrMerger *pIncr = *ppOut = (IncrMerger*) @@ -1861,9 +1865,9 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* -** Initialize the merger argument passed as the second argument. Once this -** function returns, the first key of merged data may be read from the merger -** object in the usual fashion. +** Initialize the MergeEngine object passed as the second argument. Once this +** function returns, the first key of merged data may be read from the +** MergeEngine object in the usual fashion. ** ** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge ** objects attached to the PmaReader objects that the merger reads from have @@ -1879,14 +1883,18 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int vdbeIncrInitMerger( - SortSubtask *pTask, - MergeEngine *pMerger, + SortSubtask *pTask, /* Thread that will run pMerger */ + MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; + /* Verify that the MergeEngine is assigned to a single thread */ + assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + pMerger->pTask = pTask; + for(i=0; rc==SQLITE_OK && ipIncr). ** -** If argument eMode is set to INCRINIT_NORMAL, then PmaReader PmaReaders +** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders ** in the sub-tree headed by pReadr are also initialized. Data is then loaded -** into the buffers belonging to this PmaReader, pReadr, and it is set to +** into the buffers belonging to pReadr and it is set to ** point to the first key in its range. ** -** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed +** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed ** to be a multi-threaded PmaReader and this function is being called in a ** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to -** pReadr is populated. However, the PmaReader itself is not set up to point +** pReadr is populated. However, pReadr itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do ** that. ** ** The reason this function does not call vdbePmaReaderNext() immediately -** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that has +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has ** to block on thread (pTask->thread) before accessing aFile[1]. But, since ** this entire function is being run by thread (pTask->thread), that will ** lead to the current background thread attempting to join itself. @@ -2353,7 +2361,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->bUseThreads ){ rc = vdbePmaReaderNext(pSorter->pReader); - *pbEof = (pSorter->pReader->pFile==0); + *pbEof = (pSorter->pReader->pFd==0); }else #endif /*if( !pSorter->bUseThreads )*/ { From 8a4865f139e210ea92d41171b7f39a424d981f8e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 18:57:40 +0000 Subject: [PATCH 81/99] In vdbesort.c, rename vdbeSorterDoCompare() to vdbeMergeEngineCompare() and move it closer to the one place where it is called. Other minor comment changes. FossilOrigin-Name: 09d50d9f0fe7df26dadb0a332731683a07a89fde --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 150 +++++++++++++++++++++++++++---------------------- 3 files changed, 89 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index cd56be770f..9b892425af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\srename\sall\spointers\sto\ssqlite3_file\sobjects\s"pFd"\sand\suse\sthe\nname\s"pFile"\sonly\sfor\spointers\sto\sSortFile\sobjects.\s\sOther\scomment\senhancements. -D 2014-07-28T17:18:28.406 +C In\svdbesort.c,\srename\svdbeSorterDoCompare()\sto\svdbeMergeEngineCompare()\sand\nmove\sit\scloser\sto\sthe\sone\splace\swhere\sit\sis\scalled.\s\sOther\sminor\scomment\nchanges. +D 2014-07-28T18:57:40.217 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c cbd9f27e3521737ad87149550c6d30994f74df7a +F src/vdbesort.c 2ee0867189cd40199e7d4ed944190b9453e82eec F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 163c247bd8280ab14fe577329c631c8bd884707f -R 502fd7906a0bc29e23e7a03a06d6bf2a +P 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 +R c47502cced56ced8e0a3892ddc45fd18 U drh -Z e1002bd8dfbb36530151cc4f8c9e6ac6 +Z 1aa9f494eefb9d25759f33de8d16a270 diff --git a/manifest.uuid b/manifest.uuid index 1b6e273eef..e77bb124e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -518290a7fc0994f9593c5c3ba5b2610a1b86dae1 \ No newline at end of file +09d50d9f0fe7df26dadb0a332731683a07a89fde \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ce26e9acc8..d69a48618a 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -609,7 +609,8 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ } /* -** Seek PmaReader pReadr to offset iOff within file pFile. Return SQLITE_OK +** Attach PmaReader pReadr to file pFile (if it is not already attached to +** that file) and seek it to offset iOff within the file. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ static int vdbePmaReaderSeek( @@ -759,56 +760,6 @@ static int vdbeSorterCompare( return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } -/* -** This function is called to compare two PmaReader keys when merging -** multiple b-tree segments. Parameter iOut is the index of the aTree[] -** value to recalculate. -*/ -static int vdbeSorterDoCompare( - SortSubtask *pTask, - MergeEngine *pMerger, - int iOut -){ - int i1; - int i2; - int iRes; - PmaReader *p1; - PmaReader *p2; - - assert( iOutnTree && iOut>0 ); - - if( iOut>=(pMerger->nTree/2) ){ - i1 = (iOut - pMerger->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pMerger->aTree[iOut*2]; - i2 = pMerger->aTree[iOut*2+1]; - } - - p1 = &pMerger->aReadr[i1]; - p2 = &pMerger->aReadr[i2]; - - if( p1->pFd==0 ){ - iRes = i2; - }else if( p2->pFd==0 ){ - iRes = i1; - }else{ - int res; - assert( pTask->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ - res = vdbeSorterCompare( - pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey - ); - if( res<=0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pMerger->aTree[iOut] = iRes; - return SQLITE_OK; -} - /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** @@ -1859,10 +1810,69 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ } #endif /* SQLITE_MAX_WORKER_THREADS>0 */ + + +/* +** Recompute pMerger->aTree[iOut] by comparing the next keys on the +** two PmaReaders that feed that entry. Neither of the PmaReaders +** are advanced. This routine merely does the comparison. +*/ +static void vdbeMergeEngineCompare( + MergeEngine *pMerger, /* Merge engine containing PmaReaders to compare */ + int iOut /* Store the result in pMerger->aTree[iOut] */ +){ + int i1; + int i2; + int iRes; + PmaReader *p1; + PmaReader *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; + } + + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; + + if( p1->pFd==0 ){ + iRes = i2; + }else if( p2->pFd==0 ){ + iRes = i1; + }else{ + int res; + assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = vdbeSorterCompare( + pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pMerger->aTree[iOut] = iRes; +} + +/* +** Allowed values for the eMode parameter to vdbeIncrMergerInit() +** and vdbePmaReaderIncrMergeInit(). +*/ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); + +/* Forward reference. +** The vdbeIncrMergeInit() and vdbePmaReaderIncrMergeInit() routines call each +** other (when building a merge tree). +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); /* ** Initialize the MergeEngine object passed as the second argument. Once this @@ -1877,12 +1887,12 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use -** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data ** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbeIncrInitMerger( +static int vdbeIncrMergerInit( SortSubtask *pTask, /* Thread that will run pMerger */ MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ @@ -1895,7 +1905,7 @@ static int vdbeIncrInitMerger( assert( pMerger->pTask==0 || pMerger->pTask==pTask ); pMerger->pTask = pTask; - for(i=0; rc==SQLITE_OK && iaReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrMergeInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } + if( rc!=SQLITE_OK ) return rc; } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pTask, pMerger, i); + for(i=pMerger->nTree-1; i>0; i--){ + vdbeMergeEngineCompare(pMerger, i); } - - return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); + return pTask->pUnpacked->errCode; } /* +** Initialize the IncrMerge field of a PmaReader. +** ** If the PmaReader passed as the first argument is not an incremental-reader ** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge @@ -1950,14 +1962,14 @@ static int vdbeIncrInitMerger( ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); + rc = vdbeIncrMergerInit(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object @@ -2005,18 +2017,20 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 /* -** The main routine for vdbePmaReaderIncrInit() operations run in +** The main routine for vdbePmaReaderIncrMergeInit() operations run in ** background threads. */ static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT_TASK)); + void *pRet = SQLITE_INT_TO_PTR( + vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) + ); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) +** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pReadr->pIncr @@ -2270,7 +2284,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); if( p->pIncr ){ if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + rc = vdbePmaReaderIncrMergeInit(p, INCRINIT_TASK); }else{ rc = vdbePmaReaderBgIncrInit(p); } @@ -2280,12 +2294,12 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pReadr, INCRINIT_ROOT); + rc = vdbePmaReaderIncrMergeInit(pReadr, INCRINIT_ROOT); } }else #endif { - rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); + rc = vdbeIncrMergerInit(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From d906514d98822ce5eaaca164fdbaa0cf955f53c6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 19:58:41 +0000 Subject: [PATCH 82/99] Rename vdbeIncrMergerInit() to vdbeMergeEngineInit() - a much more accurate name. FossilOrigin-Name: 5b084a2dd581141b2d0cd9d1a5975625f65ec34d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9b892425af..061768f24d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\srename\svdbeSorterDoCompare()\sto\svdbeMergeEngineCompare()\sand\nmove\sit\scloser\sto\sthe\sone\splace\swhere\sit\sis\scalled.\s\sOther\sminor\scomment\nchanges. -D 2014-07-28T18:57:40.217 +C Rename\svdbeIncrMergerInit()\sto\svdbeMergeEngineInit()\s-\sa\smuch\smore\saccurate\nname. +D 2014-07-28T19:58:41.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 2ee0867189cd40199e7d4ed944190b9453e82eec +F src/vdbesort.c 143de4b9da4e42a4270cb615dddfc9e8d5224373 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 -R c47502cced56ced8e0a3892ddc45fd18 +P 09d50d9f0fe7df26dadb0a332731683a07a89fde +R 174b1c88fd5de2d3dafe85889f7f84b2 U drh -Z 1aa9f494eefb9d25759f33de8d16a270 +Z dbe81e54bc35480cf09bd2aae1bd678b diff --git a/manifest.uuid b/manifest.uuid index e77bb124e6..565619f60c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09d50d9f0fe7df26dadb0a332731683a07a89fde \ No newline at end of file +5b084a2dd581141b2d0cd9d1a5975625f65ec34d \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d69a48618a..5f83493061 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1861,7 +1861,7 @@ static void vdbeMergeEngineCompare( } /* -** Allowed values for the eMode parameter to vdbeIncrMergerInit() +** Allowed values for the eMode parameter to vdbeMergeEngineInit() ** and vdbePmaReaderIncrMergeInit(). */ #define INCRINIT_NORMAL 0 @@ -1892,7 +1892,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbeIncrMergerInit( +static int vdbeMergeEngineInit( SortSubtask *pTask, /* Thread that will run pMerger */ MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ @@ -1969,7 +1969,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - rc = vdbeIncrMergerInit(pTask, pIncr->pMerger, eMode); + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object @@ -2299,7 +2299,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ }else #endif { - rc = vdbeIncrMergerInit(pTask0, pMain, INCRINIT_NORMAL); + rc = vdbeMergeEngineInit(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From bde27aaa5a439a1d91dbcbcd5ae9d4e60933145d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 20:16:41 +0000 Subject: [PATCH 83/99] Remove an unnecessary parameter from vdbeMergeEngineStep(). Rename a couple other routines to be more descriptive of what they do. FossilOrigin-Name: f2407a40f339fa6c2cec194f78ae7c93655b1ec1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 34 ++++++++++++++++------------------ 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 061768f24d..5ef711c807 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\svdbeIncrMergerInit()\sto\svdbeMergeEngineInit()\s-\sa\smuch\smore\saccurate\nname. -D 2014-07-28T19:58:41.375 +C Remove\san\sunnecessary\sparameter\sfrom\svdbeMergeEngineStep().\s\sRename\sa\scouple\nother\sroutines\sto\sbe\smore\sdescriptive\sof\swhat\sthey\sdo. +D 2014-07-28T20:16:41.054 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 143de4b9da4e42a4270cb615dddfc9e8d5224373 +F src/vdbesort.c e78efeed9c87d47d7a09df7d176f3fbf6dcb77e2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 09d50d9f0fe7df26dadb0a332731683a07a89fde -R 174b1c88fd5de2d3dafe85889f7f84b2 +P 5b084a2dd581141b2d0cd9d1a5975625f65ec34d +R 73873dcfe41a2b7643bb45186ddcf360 U drh -Z dbe81e54bc35480cf09bd2aae1bd678b +Z 40695df9717fb1b9f124285f8dbce47e diff --git a/manifest.uuid b/manifest.uuid index 565619f60c..ec9fe1279a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b084a2dd581141b2d0cd9d1a5975625f65ec34d \ No newline at end of file +f2407a40f339fa6c2cec194f78ae7c93655b1ec1 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5f83493061..e9fa6b456f 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1407,23 +1407,19 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine pMerge (passed as the second argument) to -** its next entry. Set *pbEof to true there is no next entry because +** Advance the MergeEngine to its next entry. +** Set *pbEof to true there is no next entry because ** the MergeEngine has reached the end of all its inputs. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeMergeEngineStep( - SortSubtask *pTask, /* The thread in which this MergeEngine runs */ MergeEngine *pMerger, /* The merge engine to advance to the next row */ int *pbEof /* Set TRUE at EOF. Set false for more content */ ){ int rc; int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ - - /* A MergeEngine object is only used by a single thread */ - assert( pMerger->pTask==0 || pMerger->pTask==pTask ); - pMerger->pTask = pTask; + SortSubtask *pTask = pMerger->pTask; /* Advance the current PmaReader */ rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); @@ -1693,7 +1689,8 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeMergeEngineStep(pTask, pIncr->pMerger, &dummy); + assert( pIncr->pMerger->pTask==pTask ); + rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); @@ -1780,7 +1777,7 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ ** If an OOM condition is encountered, return NULL. In this case free the ** pMerger argument before returning. */ -static int vdbeIncrNew( +static int vdbeIncrMergerNew( SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ IncrMerger **ppOut /* Write the new IncrMerger here */ @@ -1804,7 +1801,7 @@ static int vdbeIncrNew( /* ** Set the "use-threads" flag on object pIncr. */ -static void vdbeIncrSetThreads(IncrMerger *pIncr){ +static void vdbeIncrMergerSetThreads(IncrMerger *pIncr){ pIncr->bUseThread = 1; pIncr->pTask->file2.iEof -= pIncr->mxSz; } @@ -1902,7 +1899,7 @@ static int vdbeMergeEngineInit( int nTree = pMerger->nTree; /* Verify that the MergeEngine is assigned to a single thread */ - assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + assert( pMerger->pTask==0 ); // || pMerger->pTask==pTask ); pMerger->pTask = pTask; for(i=0; ipIncr); + rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); } } if( rc==SQLITE_OK ){ @@ -2216,7 +2213,7 @@ static int vdbeSorterMergeTreeBuild( if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS>0 if( pMain!=0 ){ - rc = vdbeIncrNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); + rc = vdbeIncrMergerNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); }else #endif { @@ -2269,13 +2266,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( pReadr==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ - rc = vdbeIncrNew(pLast, pMain, &pReadr->pIncr); + rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pReadr->pIncr); + vdbeIncrMergerSetThreads(pReadr->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aReadr[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr); + vdbeIncrMergerSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } @@ -2379,7 +2376,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else #endif /*if( !pSorter->bUseThreads )*/ { - rc = vdbeMergeEngineStep(&pSorter->aTask[0], pSorter->pMerger, pbEof); + assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); + rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); } }else{ SorterRecord *pFree = pSorter->list.pList; From 2b49327d0845e375571bb8dce0f511706acd197a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 00:23:08 +0000 Subject: [PATCH 84/99] Fix the build on windows. FossilOrigin-Name: 2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 7 ------- src/sqliteInt.h | 7 ------- src/threads.c | 8 ++++++++ 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 5ef711c807..dd136942c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sparameter\sfrom\svdbeMergeEngineStep().\s\sRename\sa\scouple\nother\sroutines\sto\sbe\smore\sdescriptive\sof\swhat\sthey\sdo. -D 2014-07-28T20:16:41.054 +C Fix\sthe\sbuild\son\swindows. +D 2014-07-29T00:23:08.770 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 5f8c5568cc749d6ab44006124e7701f463559223 +F src/os_win.c 047e903174b018c50f425be793eafa8d849737a9 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -227,7 +227,7 @@ F src/shell.c 05e9e7f667a6340643b647c4be0db15dd7627d92 F src/sqlite.h.in a30af69fcbc8fab8b4a00032f9f1d24ba2f01c2c F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h f2b28ce01099fdeb75a222aeddeece7384c9d56a +F src/sqliteInt.h 9c1e5b965191c54157b155e321fa473b9734e062 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 3c63f60ce0aae4c40ed4b8dc745490ff42a05308 +F src/threads.c afdab9acbda8b96dd0e55052de59b87d972cadc1 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b084a2dd581141b2d0cd9d1a5975625f65ec34d -R 73873dcfe41a2b7643bb45186ddcf360 +P f2407a40f339fa6c2cec194f78ae7c93655b1ec1 +R bc66c7b33352e0660fc93d07e7c8b3a0 U drh -Z 40695df9717fb1b9f124285f8dbce47e +Z c1a505884ff19c661f20155161198823 diff --git a/manifest.uuid b/manifest.uuid index ec9fe1279a..2e59c45d3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2407a40f339fa6c2cec194f78ae7c93655b1ec1 \ No newline at end of file +2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5c246a8586..fa485978d3 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1275,13 +1275,6 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } -DWORD sqlite3Win32Wait(HANDLE hObject){ - DWORD rc; - while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, - TRUE))==WAIT_IO_COMPLETION ){} - return rc; -} - /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4ad80e4373..8761556ae2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3723,11 +3723,4 @@ int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); #endif -/* -** Win32 interface -*/ -#if SQLITE_OS_WIN - DWORD sqlite3Win32Wait(HANDLE hObject); -#endif - #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c index 1cfe2cc6ce..634dda3b7d 100644 --- a/src/threads.c +++ b/src/threads.c @@ -147,6 +147,14 @@ int sqlite3ThreadCreate( return SQLITE_OK; } +/* Wait on an object */ +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} + /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ DWORD rc; From fc129698e046e6c2341aa8aa2d07fe6c30ad8608 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 00:42:39 +0000 Subject: [PATCH 85/99] Add some asserts to the Win32 mutex subsystem. FossilOrigin-Name: e8f2dc5fadae96252649875c234fcdef1108bd48 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dd136942c2..04cafa5227 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\son\swindows. -D 2014-07-29T00:23:08.770 +C Add\ssome\sasserts\sto\sthe\sWin32\smutex\ssubsystem. +D 2014-07-29T00:42:39.711 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,7 +201,7 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 -F src/mutex_w32.c 13a6b8a3902f3750a3f251ee6640d983e7ca4d29 +F src/mutex_w32.c 92e8ead41598a289c93ac64dfd5e78820c4dabc1 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f2407a40f339fa6c2cec194f78ae7c93655b1ec1 -R bc66c7b33352e0660fc93d07e7c8b3a0 -U drh -Z c1a505884ff19c661f20155161198823 +P 2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c +R 9e10c19fd0ddd3066b284eac0abe3eac +U mistachkin +Z 477d633de18b2c200b7eed3df7c1c412 diff --git a/manifest.uuid b/manifest.uuid index 2e59c45d3e..7bff472a40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c \ No newline at end of file +e8f2dc5fadae96252649875c234fcdef1108bd48 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 1f5775ccec..78d1a6fba4 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -245,6 +245,7 @@ static void winMutexFree(sqlite3_mutex *p){ assert( p ); assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + assert( winMutex_isInit==1 ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -265,6 +266,7 @@ static void winMutexEnter(sqlite3_mutex *p){ DWORD tid = GetCurrentThreadId(); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); #endif + assert( winMutex_isInit==1 ); EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); @@ -293,6 +295,7 @@ static int winMutexTry(sqlite3_mutex *p){ ** ticket #2685. */ #if 0 + assert( winMutex_isInit==1 ); if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ p->owner = tid; p->nRef++; @@ -324,6 +327,7 @@ static void winMutexLeave(sqlite3_mutex *p){ if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif + assert( winMutex_isInit==1 ); LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ From 19f30dc171b8addb457e1032dba0f126cce61d85 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 15:18:00 +0000 Subject: [PATCH 86/99] Fix the threads build on Windows when SQLITE_MAX_WORKER_THREADS is greater than 0. FossilOrigin-Name: f37db3a03d95b508066745613029b7dd1240b31c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 114e1759be..6f77fc7896 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sR-Tree\sfix\sand\sthe\snew\sSQLITE_TESTCTRL_ISINIT\stest\scontrol\sfrom\ntrunk. -D 2014-07-29T14:16:42.855 +C Fix\sthe\sthreads\sbuild\son\sWindows\swhen\sSQLITE_MAX_WORKER_THREADS\sis\sgreater\nthan\s0. +D 2014-07-29T15:18:00.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c afdab9acbda8b96dd0e55052de59b87d972cadc1 +F src/threads.c 28c72cb7818fea8df83116c54d774e9541e60121 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e8f2dc5fadae96252649875c234fcdef1108bd48 8b651d4d6cde7efbc0cc7155948f477477be100e -R cf6ae596a6404c515cc34a8685610ab8 +P b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 +R d111ee75a275c87deb7a4599506773e2 U drh -Z da4425a950252cc837b6f80952633a27 +Z e24fba9313c2b62a7de12b4c1acec1ff diff --git a/manifest.uuid b/manifest.uuid index c91039efc5..9e436c5c79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 \ No newline at end of file +f37db3a03d95b508066745613029b7dd1240b31c \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 634dda3b7d..0f7f8a83bf 100644 --- a/src/threads.c +++ b/src/threads.c @@ -150,7 +150,7 @@ int sqlite3ThreadCreate( /* Wait on an object */ DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; - while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + while( (rc = WaitForSingleObjectEx(hObject, INFINITE, TRUE))==WAIT_IO_COMPLETION ){} return rc; } From b1ac2bc8da60b01ecd639a2aa35aba953431da3e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 16:37:53 +0000 Subject: [PATCH 87/99] Make the Win32 thread handles are available after the threads exit. FossilOrigin-Name: 565c5af7a75ad5c759ce1a61dab3a61c42819644 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/os_win.c | 7 +++++++ src/threads.c | 23 +++++++++++++---------- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 6f77fc7896..d98cd45916 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sthreads\sbuild\son\sWindows\swhen\sSQLITE_MAX_WORKER_THREADS\sis\sgreater\nthan\s0. -D 2014-07-29T15:18:00.197 +C Make\sthe\sWin32\sthread\shandles\sare\savailable\safter\sthe\sthreads\sexit. +D 2014-07-29T16:37:53.918 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 047e903174b018c50f425be793eafa8d849737a9 +F src/os_win.c 5f8c5568cc749d6ab44006124e7701f463559223 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 28c72cb7818fea8df83116c54d774e9541e60121 +F src/threads.c dfc566f8b5744914bb3e6fe77b5ed63037b9b35d F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 -R d111ee75a275c87deb7a4599506773e2 -U drh -Z e24fba9313c2b62a7de12b4c1acec1ff +P f37db3a03d95b508066745613029b7dd1240b31c +Q -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c +R 3e55555e18d084a6f84ef756240b91ed +U mistachkin +Z e9ccfd5ccb7ae9590b75500531d4b897 diff --git a/manifest.uuid b/manifest.uuid index 9e436c5c79..ed3eb9f7ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f37db3a03d95b508066745613029b7dd1240b31c \ No newline at end of file +565c5af7a75ad5c759ce1a61dab3a61c42819644 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fa485978d3..5c246a8586 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1275,6 +1275,13 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/threads.c b/src/threads.c index 0f7f8a83bf..5813b34893 100644 --- a/src/threads.c +++ b/src/threads.c @@ -100,21 +100,25 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /* A running thread */ struct SQLiteThread { uintptr_t tid; /* The thread handle */ + unsigned id; /* The thread identifier */ void *(*xTask)(void*); /* The routine to run as a thread */ void *pIn; /* Argument to xTask */ void *pResult; /* Result of xTask */ }; /* Thread procedure Win32 compatibility shim */ -static void sqlite3ThreadProc( +static unsigned __stdcall sqlite3ThreadProc( void *pArg /* IN: Pointer to the SQLiteThread structure */ ){ SQLiteThread *p = (SQLiteThread *)pArg; assert( p!=0 ); + assert( p->id==GetCurrentThreadId() ); assert( p->xTask!=0 ); p->pResult = p->xTask(p->pIn); - _endthread(); + + _endthreadex(0); + return 0; /* NOT REACHED */ } /* Create a new thread */ @@ -135,37 +139,36 @@ int sqlite3ThreadCreate( }else{ p->xTask = xTask; p->pIn = pIn; - p->tid = _beginthread(sqlite3ThreadProc, 0, p); + p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); if( p->tid==(uintptr_t)-1 ){ memset(p, 0, sizeof(*p)); } } if( p->xTask==0 ){ + p->id = GetCurrentThreadId(); p->pResult = xTask(pIn); } *ppThread = p; return SQLITE_OK; } -/* Wait on an object */ -DWORD sqlite3Win32Wait(HANDLE hObject){ - DWORD rc; - while( (rc = WaitForSingleObjectEx(hObject, INFINITE, - TRUE))==WAIT_IO_COMPLETION ){} - return rc; -} +DWORD sqlite3Win32Wait(HANDLE hObject); /* os_win.c */ /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ DWORD rc; + BOOL bRc; assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask==0 ){ rc = WAIT_OBJECT_0; }else{ + assert( p->id!=0 && p->id!=GetCurrentThreadId() ); rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); + bRc = CloseHandle((HANDLE)p->tid); + assert( bRc ); } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; sqlite3_free(p); From cd4b637760f66cfab3c4c8a9ae3bdadadbaa609e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 17:22:12 +0000 Subject: [PATCH 88/99] Fix a harmless compiler warning. FossilOrigin-Name: 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d98cd45916..de6ec8a279 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sWin32\sthread\shandles\sare\savailable\safter\sthe\sthreads\sexit. -D 2014-07-29T16:37:53.918 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-07-29T17:22:12.808 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e78efeed9c87d47d7a09df7d176f3fbf6dcb77e2 +F src/vdbesort.c 48b99b15ac27229a67ed601a711aad1a36cc8631 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,8 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f37db3a03d95b508066745613029b7dd1240b31c -Q -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c -R 3e55555e18d084a6f84ef756240b91ed -U mistachkin -Z e9ccfd5ccb7ae9590b75500531d4b897 +P 565c5af7a75ad5c759ce1a61dab3a61c42819644 +R c113c051fb0a0f7ee541f07d792965ad +U drh +Z 1016c6f05046b2d8d5d597484d5273e9 diff --git a/manifest.uuid b/manifest.uuid index ed3eb9f7ff..f715ad6084 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -565c5af7a75ad5c759ce1a61dab3a61c42819644 \ No newline at end of file +216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e9fa6b456f..96f5b64d57 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1113,7 +1113,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ } } #else -# define vdbeSorterExtendFile(x,y,z) SQLITE_OK +# define vdbeSorterExtendFile(x,y,z) #endif /* From b92284de1167f5114614e3be9e2dba88048ec7d5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 18:46:30 +0000 Subject: [PATCH 89/99] Fix unreachable branches in the threads.c module. FossilOrigin-Name: 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 14 ++++++++++---- src/vdbesort.c | 6 +++--- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index de6ec8a279..064bb2589f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2014-07-29T17:22:12.808 +C Fix\sunreachable\sbranches\sin\sthe\sthreads.c\smodule. +D 2014-07-29T18:46:30.709 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c dfc566f8b5744914bb3e6fe77b5ed63037b9b35d +F src/threads.c b4152ced8515d11a2293c036109069a4e23d78a8 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 48b99b15ac27229a67ed601a711aad1a36cc8631 +F src/vdbesort.c 2198e33de4c2c32bdb3fb4f06dbe2ae90a0fce33 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 565c5af7a75ad5c759ce1a61dab3a61c42819644 -R c113c051fb0a0f7ee541f07d792965ad +P 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 +R ccb9d21b3a566e207d222a64f9b56de4 U drh -Z 1016c6f05046b2d8d5d597484d5273e9 +Z 93aca20bb7a5a87f1d3c643bafb97dd4 diff --git a/manifest.uuid b/manifest.uuid index f715ad6084..cdfed7b950 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 \ No newline at end of file +3175e366bbf7579ec9ab27214b0a4e5cd27ea204 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 5813b34893..7cd2256b49 100644 --- a/src/threads.c +++ b/src/threads.c @@ -51,6 +51,7 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; + int rc; assert( ppThread!=0 ); assert( xTask!=0 ); @@ -63,7 +64,12 @@ int sqlite3ThreadCreate( memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - if( sqlite3FaultSim(200) ? 1 : pthread_create(&p->tid, 0, xTask, pIn) ){ + if( sqlite3FaultSim(200) ){ + rc = 1; + }else{ + rc = pthread_create(&p->tid, 0, xTask, pIn); + } + if( rc ){ p->done = 1; p->pOut = xTask(pIn); } @@ -76,7 +82,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->done ){ *ppOut = p->pOut; rc = SQLITE_OK; @@ -160,7 +166,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ BOOL bRc; assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ rc = WAIT_OBJECT_0; }else{ @@ -222,7 +228,7 @@ int sqlite3ThreadCreate( int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ diff --git a/src/vdbesort.c b/src/vdbesort.c index 96f5b64d57..d4a2cc757e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -938,11 +938,11 @@ static int vdbeSorterJoinThread(SortSubtask *pTask){ #ifdef SQLITE_DEBUG_SORTER_THREADS int bDone = pTask->bDone; #endif - void *pRet; + void *pRet = SQLITE_INT_TO_PTR(SQLITE_ERROR); vdbeSorterBlockDebug(pTask, !bDone, "enter"); - rc = sqlite3ThreadJoin(pTask->pThread, &pRet); + (void)sqlite3ThreadJoin(pTask->pThread, &pRet); vdbeSorterBlockDebug(pTask, !bDone, "exit"); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + rc = SQLITE_PTR_TO_INT(pRet); assert( pTask->bDone==1 ); pTask->bDone = 0; pTask->pThread = 0; From 7c2231cf3b9400f1dca70974b3cbbb0d3e660203 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 18:53:01 +0000 Subject: [PATCH 90/99] Add a couple more assert statements. FossilOrigin-Name: 4e816db235025c7998c649fddabfd807290a08b9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 064bb2589f..968ac9514b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sunreachable\sbranches\sin\sthe\sthreads.c\smodule. -D 2014-07-29T18:46:30.709 +C Add\sa\scouple\smore\sassert\sstatements. +D 2014-07-29T18:53:01.887 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b4152ced8515d11a2293c036109069a4e23d78a8 +F src/threads.c c0c04b9c8beed10327412819126edbc6f3b3d4d7 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 -R ccb9d21b3a566e207d222a64f9b56de4 -U drh -Z 93aca20bb7a5a87f1d3c643bafb97dd4 +P 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 +R f89aa137e4caaa8d5d9c03d775d53084 +U mistachkin +Z 86b91bc53f3caa97895c33b5247f45da diff --git a/manifest.uuid b/manifest.uuid index cdfed7b950..bb2eb8265b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3175e366bbf7579ec9ab27214b0a4e5cd27ea204 \ No newline at end of file +4e816db235025c7998c649fddabfd807290a08b9 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 7cd2256b49..8bb24003b7 100644 --- a/src/threads.c +++ b/src/threads.c @@ -168,7 +168,9 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ + assert( p->id==GetCurrentThreadId() ); rc = WAIT_OBJECT_0; + assert( p->tid==0 ); }else{ assert( p->id!=0 && p->id!=GetCurrentThreadId() ); rc = sqlite3Win32Wait((HANDLE)p->tid); From acd57ea0c2bbd41cbb1d49a2ac6d97fda9854094 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 19:00:43 +0000 Subject: [PATCH 91/99] Update return value checking to conform to the beginthreadex() specs. FossilOrigin-Name: 3144a16f91364b455e54e6030714e1c12ab0bdb7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 968ac9514b..7c55c884b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scouple\smore\sassert\sstatements. -D 2014-07-29T18:53:01.887 +C Update\sreturn\svalue\schecking\sto\sconform\sto\sthe\sbeginthreadex()\sspecs. +D 2014-07-29T19:00:43.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c c0c04b9c8beed10327412819126edbc6f3b3d4d7 +F src/threads.c d5812ef982cbd9f1a211142dba1a756a632adeea F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 -R f89aa137e4caaa8d5d9c03d775d53084 +P 4e816db235025c7998c649fddabfd807290a08b9 +R 5dbe393ae5ff45dc682e2cd252c64ea7 U mistachkin -Z 86b91bc53f3caa97895c33b5247f45da +Z dbee73cad348dd0ed17a62d1c858d9f3 diff --git a/manifest.uuid b/manifest.uuid index bb2eb8265b..383d189592 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e816db235025c7998c649fddabfd807290a08b9 \ No newline at end of file +3144a16f91364b455e54e6030714e1c12ab0bdb7 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 8bb24003b7..908ad784ae 100644 --- a/src/threads.c +++ b/src/threads.c @@ -146,7 +146,7 @@ int sqlite3ThreadCreate( p->xTask = xTask; p->pIn = pIn; p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); - if( p->tid==(uintptr_t)-1 ){ + if( p->tid==0 ){ memset(p, 0, sizeof(*p)); } } From 0479c6ab9b9dbd354f4ad0344d18b5356807d137 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 21:44:13 +0000 Subject: [PATCH 92/99] Disable an assert that is sometimes generated spuriously. FossilOrigin-Name: bd9ee0ea69181526cfc3cadac33a5ec5190112b0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7c55c884b5..1054831700 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sreturn\svalue\schecking\sto\sconform\sto\sthe\sbeginthreadex()\sspecs. -D 2014-07-29T19:00:43.216 +C Disable\san\sassert\sthat\sis\ssometimes\sgenerated\sspuriously. +D 2014-07-29T21:44:13.328 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c d5812ef982cbd9f1a211142dba1a756a632adeea +F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4e816db235025c7998c649fddabfd807290a08b9 -R 5dbe393ae5ff45dc682e2cd252c64ea7 +P 3144a16f91364b455e54e6030714e1c12ab0bdb7 +R 9e96844dee61c8b0e69a9d02798bc497 U mistachkin -Z dbee73cad348dd0ed17a62d1c858d9f3 +Z d96e323c9f24218fc7129fc10b94c004 diff --git a/manifest.uuid b/manifest.uuid index 383d189592..157c4b0987 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3144a16f91364b455e54e6030714e1c12ab0bdb7 \ No newline at end of file +bd9ee0ea69181526cfc3cadac33a5ec5190112b0 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 908ad784ae..213a129c99 100644 --- a/src/threads.c +++ b/src/threads.c @@ -119,7 +119,15 @@ static unsigned __stdcall sqlite3ThreadProc( SQLiteThread *p = (SQLiteThread *)pArg; assert( p!=0 ); +#if 0 + /* + ** This assert appears to trigger spuriously on certain + ** versions of Windows, possibly due to _beginthreadex() + ** and/or CreateThread() not fully setting their thread + ** ID parameter before starting the thread. + */ assert( p->id==GetCurrentThreadId() ); +#endif assert( p->xTask!=0 ); p->pResult = p->xTask(p->pIn); From c690461ea1bc3a4dab5a43f19fa6a46db5dd1973 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 17:21:37 +0000 Subject: [PATCH 93/99] Mark some invariants in the vdbesort.c logic when SQLITE_MAX_WORKER_THREADS==0. FossilOrigin-Name: 721cd965859c9ccc24d2a1d2851c914229e584b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 20 +++++++++++++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7a190a12d7..eead314fba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges,\sand\sespecially\sthe\sfix\sfor\sthe\sCREATE\sUNIQUE\sINDEX\nproblem\sof\sticket\s[9a6daf340df99ba9]. -D 2014-07-30T14:44:24.940 +C Mark\ssome\sinvariants\sin\sthe\svdbesort.c\slogic\swhen\sSQLITE_MAX_WORKER_THREADS==0. +D 2014-07-30T17:21:37.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c cab84b480d5c616e830d9b20c8903e42de3b60b3 +F src/vdbesort.c e37e494274fb9a05955b44eb61ead2230ae3e321 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1190,7 +1190,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bd9ee0ea69181526cfc3cadac33a5ec5190112b0 6b785e92f279cb65746834d5cd25594fd3333342 -R 5abc11219614f612d7ef9f82b784bc0c +P 5b50a8380b2b678c1646ff303e3696efc1d7d92c +R 79135dabbf9021221308446429439ade U drh -Z f20a2d1678a8f4c3f64fec3d49fad416 +Z af737cf3847cc90d4e2e364aefe53a30 diff --git a/manifest.uuid b/manifest.uuid index 96ada87e5a..a98bddf92c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b50a8380b2b678c1646ff303e3696efc1d7d92c \ No newline at end of file +721cd965859c9ccc24d2a1d2851c914229e584b3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 9110bd67b0..8ff68d54c8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1860,6 +1860,10 @@ static void vdbeMergeEngineCompare( /* ** Allowed values for the eMode parameter to vdbeMergeEngineInit() ** and vdbePmaReaderIncrMergeInit(). +** +** Only INCRINIT_NORMAL is valid in single-threaded builds (when +** SQLITE_MAX_WORKER_THREADS==0). The other values are only used +** when there exists one or more separate worker threads. */ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 @@ -1898,12 +1902,15 @@ static int vdbeMergeEngineInit( int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + /* Verify that the MergeEngine is assigned to a single thread */ assert( pMerger->pTask==0 ); // || pMerger->pTask==pTask ); pMerger->pTask = pTask; for(i=0; i0 && eMode==INCRINIT_ROOT ){ /* PmaReaders should be normally initialized in order, as if they are ** reading from the same temp file this makes for more linear file IO. ** However, in the INCRINIT_ROOT case, if PmaReader aReadr[nTask-1] is @@ -1962,6 +1969,10 @@ static int vdbeMergeEngineInit( static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; @@ -2005,7 +2016,9 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ } #endif - if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ + if( rc==SQLITE_OK + && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) + ){ rc = vdbePmaReaderNext(pReadr); } } @@ -2186,7 +2199,8 @@ static int vdbeSorterMergeTreeBuild( for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; - if( pTask->nPMA ){ + assert( pTask->nPMA>0 || SQLITE_MAX_WORKER_THREADS>0 ); + if( SQLITE_MAX_WORKER_THREADS==0 || pTask->nPMA ){ MergeEngine *pRoot = 0; /* Root node of tree for this task */ int nDepth = vdbeSorterTreeDepth(pTask->nPMA); i64 iReadOff = 0; From c0fea3cf53015606fbc1725937fd0b74fc523112 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 18:47:12 +0000 Subject: [PATCH 94/99] Add a new sqlite3FaultSim() call to vdbePmaReaderSeek() to facilitate tests of error handling in the sorter. FossilOrigin-Name: 655d8cfc752b3f5f63521a57f2155f8e14aaf7c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index eead314fba..e5ae89aca5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Mark\ssome\sinvariants\sin\sthe\svdbesort.c\slogic\swhen\sSQLITE_MAX_WORKER_THREADS==0. -D 2014-07-30T17:21:37.320 +C Add\sa\snew\ssqlite3FaultSim()\scall\sto\svdbePmaReaderSeek()\sto\sfacilitate\ntests\sof\serror\shandling\sin\sthe\ssorter. +D 2014-07-30T18:47:12.395 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e37e494274fb9a05955b44eb61ead2230ae3e321 +F src/vdbesort.c b9a830685826c057cfd41993902a5afc6fe436e1 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1190,7 +1190,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b50a8380b2b678c1646ff303e3696efc1d7d92c -R 79135dabbf9021221308446429439ade +P 721cd965859c9ccc24d2a1d2851c914229e584b3 +R 1fd8d4a346a6db9102e067303c7fe957 U drh -Z af737cf3847cc90d4e2e364aefe53a30 +Z a72347f7edf8a1215db60e10b80e325d diff --git a/manifest.uuid b/manifest.uuid index a98bddf92c..a8cbf441d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -721cd965859c9ccc24d2a1d2851c914229e584b3 \ No newline at end of file +655d8cfc752b3f5f63521a57f2155f8e14aaf7c4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8ff68d54c8..6915a2a9b9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -623,6 +623,7 @@ static int vdbePmaReaderSeek( assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); + if( sqlite3FaultSim(201) ) return SQLITE_IOERR_READ; if( pReadr->aMap ){ sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); pReadr->aMap = 0; From 034596153de60aca340675354c9d10959f187baa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 15:13:22 +0000 Subject: [PATCH 95/99] Query or change the maximum number of worker threads allowed on each database connection separately using the "PRAGMA threads" command. FossilOrigin-Name: 29c5e8a7c9d7ce349a1e1d72082d23450e877b45 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/pragma.c | 44 +++++++++++++++++++++++++++++++++++--------- src/shell.c | 4 ++-- src/sqliteInt.h | 1 + src/vdbesort.c | 4 +++- tool/mkpragmatab.tcl | 2 ++ 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d18353dedb..22dd6d848a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\srecent\sperformance\senhancements\simplemented\son\strunk\sinto\sthe\nthreads\sbranch. -D 2014-08-25T13:27:02.036 +C Query\sor\schange\sthe\smaximum\snumber\sof\sworker\sthreads\sallowed\son\seach\ndatabase\sconnection\sseparately\susing\sthe\s"PRAGMA\sthreads"\scommand. +D 2014-08-25T15:13:22.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,18 +217,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e +F src/pragma.c 4ed8bc86d1a9ee336e66bc441b30abb702f4de33 F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c f8b0b6c43bee15f4e239ead1c9c9e3009e507e39 -F src/shell.c 200772eebd7b0fcf2072fc3dbd95d694389d5efa +F src/shell.c 6dab215a30f7ca4d5fc31338c44b007bb6ef0dee F src/sqlite.h.in fef15a64d1358f5c365bd3f46f4c1915d5a5e5f0 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 80d0bb053ef728896142c4e808bfcdc49494e9be +F src/sqliteInt.h d8eb2d4d4ce26365dc9d49efee3bc880618e87c2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -292,7 +292,7 @@ F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4c9d686da474957d2e78834f13cc5f141fe6b87f -F src/vdbesort.c b9a830685826c057cfd41993902a5afc6fe436e1 +F src/vdbesort.c 3a76f51efdf0790fd3d26afabdd81e0a21f52ae7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1158,7 +1158,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 +F tool/mkpragmatab.tcl cce51d8f60c7f145d8fccabe6b5dfdedf31c5f5c F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 05807c4122505567ec64fb2d142077f48a0a10b1 54164ce47cfc3ad5dd8797114e4ba78811f23bef -R c157557aea90a1674d041ac0f1de2f68 +P dfdc900f5d1a31ee5c5f35a630c4a8253e69093b +R 4c161b4070e5a7022237a28172d6e331 U drh -Z 10ad80e0aa78a5fbf32d4d843030da4f +Z 1c68bfb86ec54e7d60991aea6df8a67a diff --git a/manifest.uuid b/manifest.uuid index 8fecff4bd8..645cc6429f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dfdc900f5d1a31ee5c5f35a630c4a8253e69093b \ No newline at end of file +29c5e8a7c9d7ce349a1e1d72082d23450e877b45 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 9ed5e13eb0..897d3b5d24 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -61,14 +61,15 @@ #define PragTyp_TABLE_INFO 30 #define PragTyp_TEMP_STORE 31 #define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_WAL_AUTOCHECKPOINT 33 -#define PragTyp_WAL_CHECKPOINT 34 -#define PragTyp_ACTIVATE_EXTENSIONS 35 -#define PragTyp_HEXKEY 36 -#define PragTyp_KEY 37 -#define PragTyp_REKEY 38 -#define PragTyp_LOCK_STATUS 39 -#define PragTyp_PARSER_TRACE 40 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ @@ -418,6 +419,10 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif + { /* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -465,7 +470,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 56 on by default, 69 total. */ +/* Number of pragmas: 57 on by default, 70 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -2273,6 +2278,27 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( sqlite3GlobalConfig.bCoreMutex + && zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + if( N>sqlite3GlobalConfig.nWorker ) N = sqlite3GlobalConfig.nWorker; + db->mxWorker = N&0xff; + } + returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases diff --git a/src/shell.c b/src/shell.c index f2a275fe39..0357c1a082 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3817,10 +3817,10 @@ static void main_init(ShellState *data) { data->showHeader = 0; sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 64); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 4); } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f3d49a7b10..ce123b395f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1012,6 +1012,7 @@ struct sqlite3 { u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + u8 mxWorker; /* Maximum number of worker threads */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbesort.c b/src/vdbesort.c index 6915a2a9b9..e71b19fc11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -795,8 +795,10 @@ int sqlite3VdbeSorterInit( int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 +#elif SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + int nWorker = MIN(SORTER_MAX_MERGE_COUNT-1, db->mxWorker); #else - int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); + int nWorker = db->mxWorker; #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 28a1e468b8..aa7c8078c5 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -294,6 +294,8 @@ set pragma_def { IF: defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) NAME: soft_heap_limit + + NAME: threads } fconfigure stdout -translation lf set name {} From 028696c4cc1f4bddcd376d3a290415f7eb67526e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 23:44:44 +0000 Subject: [PATCH 96/99] Remove the SQLITE_CONFIG_WORKER_THREADS configuration parameter. The number of worker threads in the sorter is now determined only by the PRAGMA threads=N setting. FossilOrigin-Name: e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 --- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- src/global.c | 1 - src/main.c | 9 --------- src/pragma.c | 2 +- src/shell.c | 1 - src/sqlite.h.in | 11 ----------- src/sqliteInt.h | 4 ++-- src/test_malloc.c | 26 -------------------------- src/vdbesort.c | 12 +++++++++--- test/sort.test | 6 ++---- test/sort2.test | 10 +--------- test/sort4.test | 11 +---------- test/sortfault.test | 6 +----- 14 files changed, 35 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index 84d67d6062..9edec3f675 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sCAST\soperator\senhancements\sfrom\strunk. -D 2014-08-25T22:43:17.516 +C Remove\sthe\sSQLITE_CONFIG_WORKER_THREADS\sconfiguration\sparameter.\s\sThe\snumber\nof\sworker\sthreads\sin\sthe\ssorter\sis\snow\sdetermined\sonly\sby\sthe\nPRAGMA\sthreads=N\ssetting. +D 2014-08-25T23:44:44.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/expr.c 358634f4ddeeb4e69643cb6db5819104a7834c60 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 -F src/global.c 77ec119d6f6453039c2820336af8e8f804f20acf +F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c ce41520e565eb8ef09824fa9778a72364291d371 +F src/main.c f1726e704941d365ce2846161e93ba689a245845 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -217,18 +217,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c 4ed8bc86d1a9ee336e66bc441b30abb702f4de33 +F src/pragma.c 33971fcaa7c13b84b1a0f2e813f4a3ab4d745ede F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c f8b0b6c43bee15f4e239ead1c9c9e3009e507e39 -F src/shell.c 6dab215a30f7ca4d5fc31338c44b007bb6ef0dee -F src/sqlite.h.in fef15a64d1358f5c365bd3f46f4c1915d5a5e5f0 +F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 +F src/sqlite.h.in aa2cc1405cb999c9d73484e0686f7b869b430ba3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h d8eb2d4d4ce26365dc9d49efee3bc880618e87c2 +F src/sqliteInt.h 43419afaed8cd3bf99df06d38952a52f827217b9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -257,7 +257,7 @@ F src/test_intarray.c db4614c2262a06abc4409dc048d59c580c38320f F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307 +F src/test_malloc.c 5368fb1de77246da1ae0ff59cba0d30cb0e5812f F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -292,7 +292,7 @@ F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4e08ea087aea367dae7c45129b75487e0056e819 -F src/vdbesort.c 3a76f51efdf0790fd3d26afabdd81e0a21f52ae7 +F src/vdbesort.c 50fe3442f41dbfe0b37d2e8b55e7460244015533 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -838,11 +838,11 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 -F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 +F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 +F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test 971452fd4e2928e6fc05c3868396ad7d5f9ce2ad -F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 +F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb +F test/sortfault.test b8e35177f97438b930ee87c9419ca2599e8073e1 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29c5e8a7c9d7ce349a1e1d72082d23450e877b45 af364cce9da0961593ef876b646197f82df08ad5 -R c7c8cecd9a8ffa9d596fa5137e696964 +P 6c8f86e4e08d5d57e21496277613e0f9dcc06514 +R 428258d1143faa4ab393e38d0d72251a U drh -Z 1d9d719c4b1a959a79787aab5421ad60 +Z 37ad2e5d491e7e49897b74763238faf5 diff --git a/manifest.uuid b/manifest.uuid index 25b89b4eae..a22d912847 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c8f86e4e08d5d57e21496277613e0f9dcc06514 \ No newline at end of file +e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 \ No newline at end of file diff --git a/src/global.c b/src/global.c index f152c3b5d0..2c14b58abd 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,6 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - SQLITE_DEFAULT_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/main.c b/src/main.c index 8a6456d685..98bd82a298 100644 --- a/src/main.c +++ b/src/main.c @@ -515,15 +515,6 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_WORKER_THREADS: { -#if SQLITE_MAX_WORKER_THREADS>0 - int n = va_arg(ap, int); - if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; - if( n>=0 ) sqlite3GlobalConfig.nWorker = n; -#endif - break; - } - default: { rc = SQLITE_ERROR; break; diff --git a/src/pragma.c b/src/pragma.c index 897d3b5d24..8421042e00 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2292,7 +2292,7 @@ void sqlite3Pragma( && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK && N>=0 ){ - if( N>sqlite3GlobalConfig.nWorker ) N = sqlite3GlobalConfig.nWorker; + if( N>SQLITE_MAX_WORKER_THREADS ) N = SQLITE_MAX_WORKER_THREADS; db->mxWorker = N&0xff; } returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); diff --git a/src/shell.c b/src/shell.c index 0357c1a082..9c235414af 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3818,7 +3818,6 @@ static void main_init(ShellState *data) { sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 64); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index be0ed74565..f8ea7ad623 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1718,16 +1718,6 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** -** -** [[SQLITE_CONFIG_WORKER_THREADS]] -**
    SQLITE_CONFIG_WORKER_THREADS -**
    ^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int. -** It is used to set the number of background worker threads that may be -** launched when sorting large amounts of data. A value of 0 means launch -** no background threads at all. The maximum number of background threads -** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS -** pre-processor option. -** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1752,7 +1742,6 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ -#define SQLITE_CONFIG_WORKER_THREADS 24 /* int nWorker */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ce123b395f..b48d5cf795 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -441,9 +441,10 @@ */ #if SQLITE_TEMP_STORE==3 # undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 #endif #ifndef SQLITE_MAX_WORKER_THREADS -# define SQLITE_MAX_WORKER_THREADS 0 +# define SQLITE_MAX_WORKER_THREADS 4 #endif #ifndef SQLITE_DEFAULT_WORKER_THREADS # define SQLITE_DEFAULT_WORKER_THREADS 0 @@ -2764,7 +2765,6 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ - int nWorker; /* Number of worker threads to use */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ diff --git a/src/test_malloc.c b/src/test_malloc.c index 6ac030f23c..900a8ac40c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1253,31 +1253,6 @@ static int test_config_cis( return TCL_OK; } -/* -** Usage: sqlite3_config_worker_threads N -*/ -static int test_config_worker_threads( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - int rc; - int nThread; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "N"); - return TCL_ERROR; - } - if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){ - return TCL_ERROR; - } - - rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - - return TCL_OK; -} /* ** Usage: sqlite3_dump_memsys3 FILENAME @@ -1532,7 +1507,6 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_config_cis", test_config_cis ,0 }, - { "sqlite3_config_worker_threads", test_config_worker_threads ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/vdbesort.c b/src/vdbesort.c index e71b19fc11..cdbf6e0252 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -795,10 +795,16 @@ int sqlite3VdbeSorterInit( int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 -#elif SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT - int nWorker = MIN(SORTER_MAX_MERGE_COUNT-1, db->mxWorker); #else - int nWorker = db->mxWorker; + int nWorker = sqlite3TempInMemory(db) ? 0 : db->mxWorker ; +#endif + + /* Do not allow the total number of threads (main thread + all workers) + ** to exceed the maximum merge count */ +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + if( nWorker>=SORTER_MAX_MERGE_COUNT ){ + nWorker = SORTER_MAX_MERGE_COUNT-1; + } #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); diff --git a/test/sort.test b/test/sort.test index e75740e9c3..1c89552bb1 100644 --- a/test/sort.test +++ b/test/sort.test @@ -544,7 +544,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { } { db close sqlite3_shutdown - sqlite3_config_worker_threads $nWorker if {$coremutex} { sqlite3_config multithread } else { @@ -556,7 +555,8 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { reset_db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit - execsql "PRAGMA temp_store = $tmpstore" + execsql "PRAGMA temp_store = $tmpstore; PRAGMA threads = $nWorker" + set ten [string repeat X 10300] set one [string repeat y 200] @@ -602,7 +602,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { db close sqlite3_shutdown -sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized @@ -637,4 +636,3 @@ do_execsql_test 17.1 { } {} finish_test - diff --git a/test/sort2.test b/test/sort2.test index e4e40dab74..29001f0099 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -22,9 +22,8 @@ foreach {tn script} { 1 { } 2 { catch { db close } - sqlite3_shutdown - sqlite3_config_worker_threads 7 reset_db + catch { db eval {PRAGMA threads=7} } } } { @@ -76,13 +75,6 @@ foreach {tn script} { } { 200000 100 200000 100 200000 100 200000 100 200000 100 } - - db close - sqlite3_shutdown - sqlite3_config_worker_threads 0 - sqlite3_initialize - } finish_test - diff --git a/test/sort4.test b/test/sort4.test index 4e8336cd84..01fcbfee95 100644 --- a/test/sort4.test +++ b/test/sort4.test @@ -19,11 +19,7 @@ source $testdir/tester.tcl set testprefix sort4 # Configure the sorter to use 3 background threads. -catch { db close } -sqlite3_shutdown -sqlite3_config_worker_threads 3 -sqlite3_initialize -reset_db +db eval {PRAGMA threads=3} # Minimum number of seconds to run for. If the value is 0, each test # is run exactly once. Otherwise, tests are repeated until the timeout @@ -190,9 +186,4 @@ for {set t 2} {1} {incr tn} { do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {} } -catch { db close } -sqlite3_shutdown -sqlite3_config_worker_threads 0 -sqlite3_initialize finish_test - diff --git a/test/sortfault.test b/test/sortfault.test index 4c199ab212..a1983ac1c0 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -30,9 +30,7 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { } { if {$sqlite_options(threadsafe)} { set threadsmode singlethread } - catch { db close } - sqlite3_shutdown - sqlite3_config_worker_threads $nWorker + db eval "PRAGMA threads=$nWorker" sqlite3_config $threadsmode if { $lookaside } { sqlite3_config_lookaside 100 500 @@ -110,7 +108,6 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { catch { db close } sqlite3_shutdown -sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized @@ -166,4 +163,3 @@ do_faultsim_test 5.1 -faults oom* -body { } finish_test - From 111544cbb4e1226ca80a1670f72075f07f949968 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 16:20:47 +0000 Subject: [PATCH 97/99] Add SQLITE_LIMIT_WORKER_THREADS for controlling the maximum number of worker threads. FossilOrigin-Name: 1b598c68f32db635d1cea1373bedc434aa60cf08 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 8 +++++++- src/pragma.c | 9 ++++----- src/sqlite.h.in | 5 +++++ src/sqliteInt.h | 3 +-- src/vdbesort.c | 11 ++++++++++- 7 files changed, 38 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index b491b9fbcb..6f1298ff91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\sperformance\senhancements\sfrom\strunk\sonto\sthe\sthreads\sbranch. -D 2014-08-29T14:40:07.113 +C Add\sSQLITE_LIMIT_WORKER_THREADS\sfor\scontrolling\sthe\smaximum\snumber\sof\nworker\sthreads. +D 2014-08-29T16:20:47.382 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c f1726e704941d365ce2846161e93ba689a245845 +F src/main.c e71d7c8b415e3b54d729aabb876f39f7aa35b000 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -217,7 +217,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c 33971fcaa7c13b84b1a0f2e813f4a3ab4d745ede +F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225 F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -225,10 +225,10 @@ F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 -F src/sqlite.h.in aa2cc1405cb999c9d73484e0686f7b869b430ba3 +F src/sqlite.h.in 74b42237f0d2b010779cc1b1a00190452b31a2ec F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 43419afaed8cd3bf99df06d38952a52f827217b9 +F src/sqliteInt.h e539938b3504520da5cefe225712452abea6795a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -292,7 +292,7 @@ F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 -F src/vdbesort.c 50fe3442f41dbfe0b37d2e8b55e7460244015533 +F src/vdbesort.c f92628f3d5d4432f751b15a5f39bacc3c6a64a03 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 2a74129a21f9745f1363f844807e2d10201a3f40 -R a4f057fd137875212defd2b87d648d95 +P 35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 +R 2f78b359e3468cdedd78b18f8317c2eb U drh -Z c730a4dfee48578be19b0ff5cd3b7719 +Z bfedd71a211c8cc0f7a49234bc820538 diff --git a/manifest.uuid b/manifest.uuid index 7f66b19a61..cd1149dbb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 \ No newline at end of file +1b598c68f32db635d1cea1373bedc434aa60cf08 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 98bd82a298..5681ba5abc 100644 --- a/src/main.c +++ b/src/main.c @@ -2078,6 +2078,7 @@ static const int aHardLimit[] = { SQLITE_MAX_LIKE_PATTERN_LENGTH, SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, }; /* @@ -2113,6 +2114,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_TRIGGER_DEPTH<1 # error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 #endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif /* @@ -2146,7 +2150,8 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ SQLITE_MAX_LIKE_PATTERN_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -2493,6 +2498,7 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = 0; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; diff --git a/src/pragma.c b/src/pragma.c index 8421042e00..12446125fb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2287,15 +2287,14 @@ void sqlite3Pragma( */ case PragTyp_THREADS: { sqlite3_int64 N; - if( sqlite3GlobalConfig.bCoreMutex - && zRight + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK && N>=0 ){ - if( N>SQLITE_MAX_WORKER_THREADS ) N = SQLITE_MAX_WORKER_THREADS; - db->mxWorker = N&0xff; + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); + returnSingleInt(pParse, "threads", + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f8ea7ad623..103bb0a80c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3073,6 +3073,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
    SQLITE_LIMIT_WORKER_THREADS
    +**
    The maximum number of separate worker threads that a single +** [database connection] may start to help it with a computation.
    )^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -3086,6 +3090,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 /* ** CAPI3REF: Compiling An SQL Statement diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b48d5cf795..aea497b7a9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -941,7 +941,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -1013,7 +1013,6 @@ struct sqlite3 { u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ - u8 mxWorker; /* Maximum number of worker threads */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbesort.c b/src/vdbesort.c index cdbf6e0252..158fa440fb 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -796,7 +796,16 @@ int sqlite3VdbeSorterInit( #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 #else - int nWorker = sqlite3TempInMemory(db) ? 0 : db->mxWorker ; + int nWorker; +#endif + + /* Initialize the upper limit on the number of worker threads */ +#if SQLITE_MAX_WORKER_THREADS>0 + if( sqlite3TempInMemory(db) || sqlite3GlobalConfig.bCoreMutex==0 ){ + nWorker = 0; + }else{ + nWorker = db->aLimit[SQLITE_LIMIT_WORKER_THREADS]; + } #endif /* Do not allow the total number of threads (main thread + all workers) From 43cbe1437891abfaf89ae72b85c7c3a0c6dbe45d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 18:06:33 +0000 Subject: [PATCH 98/99] Fix the speedtest1.c test program to set the worker thread count using the threads pragma. FossilOrigin-Name: 2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6f1298ff91..31a0c63cb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_LIMIT_WORKER_THREADS\sfor\scontrolling\sthe\smaximum\snumber\sof\nworker\sthreads. -D 2014-08-29T16:20:47.382 +C Fix\sthe\sspeedtest1.c\stest\sprogram\sto\sset\sthe\sworker\sthread\scount\susing\nthe\sthreads\spragma. +D 2014-08-29T18:06:33.187 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -851,7 +851,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c f452891e50571627f7060c0e1262359127055717 +F test/speedtest1.c 83f6b3318f7ee60e52b978b5a5e5dd7e83dfb7ee F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 -R 2f78b359e3468cdedd78b18f8317c2eb +P 1b598c68f32db635d1cea1373bedc434aa60cf08 +R 0f5325539e5516baa4a3f930643f73d7 U drh -Z bfedd71a211c8cc0f7a49234bc820538 +Z 8e5082b548f72bf30dfd81cfdd246c1c diff --git a/manifest.uuid b/manifest.uuid index cd1149dbb0..8313e46779 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b598c68f32db635d1cea1373bedc434aa60cf08 \ No newline at end of file +2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 5fb9917f9b..8589b16333 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -1278,11 +1278,6 @@ int main(int argc, char **argv){ rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); if( rc ) fatal_error("scratch configuration failed: %d\n", rc); } -#ifdef SQLITE_CONFIG_WORKER_THREADS - if( nThread>0 ){ - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); - } -#endif if( nLook>0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } @@ -1300,6 +1295,7 @@ int main(int argc, char **argv){ /* Set database connection options */ sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0); if( doTrace ) sqlite3_trace(g.db, traceCallback, 0); + speedtest1_exec("PRAGMA threads=%d", nThread); if( zKey ){ speedtest1_exec("PRAGMA key('%s')", zKey); } From 6b2129aaaf69c4318d4dc158b685e4522c4bf633 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 19:06:07 +0000 Subject: [PATCH 99/99] Disable worker threads when SQLITE_THREADSAFE=0. Set the default compile-time maximum number of worker threads to 8 and honor the SQLITE_DEFAULT_WORKER_THREADS compile-time constant (which defaults to 0). FossilOrigin-Name: 33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 2 +- src/sqliteInt.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 31a0c63cb4..6c7fe672c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sspeedtest1.c\stest\sprogram\sto\sset\sthe\sworker\sthread\scount\susing\nthe\sthreads\spragma. -D 2014-08-29T18:06:33.187 +C Disable\sworker\sthreads\swhen\sSQLITE_THREADSAFE=0.\s\sSet\sthe\sdefault\scompile-time\nmaximum\snumber\sof\sworker\sthreads\sto\s8\sand\shonor\sthe\nSQLITE_DEFAULT_WORKER_THREADS\scompile-time\sconstant\s(which\sdefaults\sto\s0). +D 2014-08-29T19:06:07.922 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c e71d7c8b415e3b54d729aabb876f39f7aa35b000 +F src/main.c d2ef03a45552e11813c68326d5edfda992e319d4 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -228,7 +228,7 @@ F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 F src/sqlite.h.in 74b42237f0d2b010779cc1b1a00190452b31a2ec F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h e539938b3504520da5cefe225712452abea6795a +F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1b598c68f32db635d1cea1373bedc434aa60cf08 -R 0f5325539e5516baa4a3f930643f73d7 +P 2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b +R 97462c8661f494fef9aea0705c0547b9 U drh -Z 8e5082b548f72bf30dfd81cfdd246c1c +Z 369e082ce9db2aa4e0cc673a60a91a43 diff --git a/manifest.uuid b/manifest.uuid index 8313e46779..680e153df5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b \ No newline at end of file +33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5681ba5abc..fd7151b174 100644 --- a/src/main.c +++ b/src/main.c @@ -2498,7 +2498,7 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = 0; + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aea497b7a9..7fd999d9ee 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -439,12 +439,12 @@ ** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it ** to zero. */ -#if SQLITE_TEMP_STORE==3 +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 # undef SQLITE_MAX_WORKER_THREADS # define SQLITE_MAX_WORKER_THREADS 0 #endif #ifndef SQLITE_MAX_WORKER_THREADS -# define SQLITE_MAX_WORKER_THREADS 4 +# define SQLITE_MAX_WORKER_THREADS 8 #endif #ifndef SQLITE_DEFAULT_WORKER_THREADS # define SQLITE_DEFAULT_WORKER_THREADS 0