From a7b3b635244620ce805245355309689724212eb1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 14 Feb 2014 23:35:49 +0000 Subject: [PATCH 001/177] Better support for MinGW 4.x. FossilOrigin-Name: e147230767383ed9f64085b613fd18508f74be21 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d8dd849567..8c86fcc99c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\scases\swhere\sit\sis\snecessary\sto\scheck\sfor\sNULL\safter\nthe\sloop\sterminating\scondition. -D 2014-02-14T20:59:53.587 +C Better\ssupport\sfor\sMinGW\s4.x. +D 2014-02-14T23:35:49.555 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 5b1f4e30c8332270e4c14254ef1fbcbd92a4a4c7 +F src/sqliteInt.h bc5e9c7e30d3d65ce2cd218b737f0057de0ce135 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1150,7 +1150,10 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b6bea903ac8e1717ed50b221d73bd0be061c7663 -R f976ccc3e6f99e20be380cc680402b19 -U drh -Z d8a7427791a9ddd9c5e417477ea3c5a6 +P 3c1ae447dec8fc2af1c5105134061717594ac0e0 +R ce6f8022fe93ee0d0d6cf8a6cc271c33 +T *branch * mingw4x +T *sym-mingw4x * +T -sym-trunk * +U mistachkin +Z c1377b93302be214ebcce2f123ef84cb diff --git a/manifest.uuid b/manifest.uuid index 3ff1fc2f3d..0a1ac78ab8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c1ae447dec8fc2af1c5105134061717594ac0e0 \ No newline at end of file +e147230767383ed9f64085b613fd18508f74be21 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8b9337a2b0..41e7e2d74d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -48,6 +48,38 @@ # define _LARGEFILE_SOURCE 1 #endif +/* +** For MinGW, check to see if we can include the header file containing its +** version information, among other things. Normally, this internal MinGW +** header file would [only] be included automatically by other MinGW header +** files; however, the contained version information is now required by this +** header file to work around binary compatibility issues (see below) and +** this is the only known way to reliably obtain it. This entire #if block +** would be completely unnecessary if there was any other way of detecting +** MinGW via their preprocessor (e.g. if they customized their GCC to define +** some MinGW-specific macros). When compiling for MinGW, either the +** _HAVE_MINGW_H or _HAVE__MINGW_H (note the extra underscore) macro must be +** defined; otherwise, detection of conditions specific to MinGW will be +** disabled. +*/ +#if defined(_HAVE_MINGW_H) +# include "mingw.h" +#elif defined(_HAVE__MINGW_H) +# include "_mingw.h" +#endif + +/* +** For MinGW version 4.x (and higher), check to see if the _USE_32BIT_TIME_T +** define is required to maintain binary compatibility with the MSVC runtime +** library in use (e.g. for Windows XP). +*/ +#if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ + defined(_WIN32) && !defined(_WIN64) && \ + defined(__MINGW32_MAJOR_VERSION) && __MINGW32_MAJOR_VERSION >= 4 && \ + defined(__MSVCRT__) +# define _USE_32BIT_TIME_T +#endif + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build From 65acf373c953f387d4a5166e7becee9135d88292 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 16 Feb 2014 19:20:00 +0000 Subject: [PATCH 002/177] Use the modern MinGW major version macro, not the deprecated one. FossilOrigin-Name: 057c16ee9786efcad6372a29b02d743ff3d628ff --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8c86fcc99c..2473432345 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\ssupport\sfor\sMinGW\s4.x. -D 2014-02-14T23:35:49.555 +C Use\sthe\smodern\sMinGW\smajor\sversion\smacro,\snot\sthe\sdeprecated\sone. +D 2014-02-16T19:20:00.825 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bc5e9c7e30d3d65ce2cd218b737f0057de0ce135 +F src/sqliteInt.h 168f311aa9887711ac5f31b52dc83f1ee33f7ab4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1150,10 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3c1ae447dec8fc2af1c5105134061717594ac0e0 -R ce6f8022fe93ee0d0d6cf8a6cc271c33 -T *branch * mingw4x -T *sym-mingw4x * -T -sym-trunk * +P e147230767383ed9f64085b613fd18508f74be21 +R 093f1edd2723941164ffa92a68b20dd8 U mistachkin -Z c1377b93302be214ebcce2f123ef84cb +Z f128626e0ed2b197fa14de6fa8f864a7 diff --git a/manifest.uuid b/manifest.uuid index 0a1ac78ab8..f4e941d42e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e147230767383ed9f64085b613fd18508f74be21 \ No newline at end of file +057c16ee9786efcad6372a29b02d743ff3d628ff \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 41e7e2d74d..0276d37045 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -75,7 +75,7 @@ */ #if !defined(_USE_32BIT_TIME_T) && !defined(_USE_64BIT_TIME_T) && \ defined(_WIN32) && !defined(_WIN64) && \ - defined(__MINGW32_MAJOR_VERSION) && __MINGW32_MAJOR_VERSION >= 4 && \ + defined(__MINGW_MAJOR_VERSION) && __MINGW_MAJOR_VERSION >= 4 && \ defined(__MSVCRT__) # define _USE_32BIT_TIME_T #endif From 5b4627e957d25d1ad4c57df4e940c9563c5a81bb Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 21 Feb 2014 05:44:18 +0000 Subject: [PATCH 003/177] Better loadable extension handling in the Win32 VFS when compiled for Cygwin. FossilOrigin-Name: 6676475c47558a52cb2632e92a541e13d143b2dc --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 419633cc87..4ed19202d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stext\sof\sa\scomment\sused\sto\sgenerate\sVDBE\sopcode\sdocumentation\sso\sthat\nit\somits\ssymbols\sthat\scan\sbe\smistaken\sfor\sa\shyperlink\sby\sthe\sparser. -D 2014-02-20T19:42:00.531 +C Better\sloadable\sextension\shandling\sin\sthe\sWin32\sVFS\swhen\scompiled\sfor\sCygwin. +D 2014-02-21T05:44:18.393 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 -F src/os_win.c d4284f003445054a26689f1264b1b9bf7261bd1b +F src/os_win.c 23bd567f897b8934096ee7ce51b21ba43aa94ec7 F src/pager.c 0ffa313a30ed6d061d9c6601b7b175cc50a1cab7 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 @@ -1151,7 +1151,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 824029090d1c4c4c608f67dd197961eba29c039d -R 819e2e4df777a2f59b5f5445b2831eb5 -U drh -Z c4d587ace814e560a6dc901fdec24d15 +P b46d4e8923e6e367412bba7aeac07039bbcbabd1 +R a7d693d1ea11a2ee85a54f6ea757a9d1 +T *branch * cygDlOpen +T *sym-cygDlOpen * +T -sym-trunk * +U mistachkin +Z 4b80816c39a01dfdffbdf857b2bcb9b9 diff --git a/manifest.uuid b/manifest.uuid index 32d13dfb30..b559183c43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b46d4e8923e6e367412bba7aeac07039bbcbabd1 \ No newline at end of file +6676475c47558a52cb2632e92a541e13d143b2dc \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d393c0d016..27f2337287 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5141,8 +5141,23 @@ static int winFullPathname( */ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; +#if defined(__CYGWIN__) + int nFull = pVfs->mxPathname+1; + char *zFull = sqlite3MallocZero( nFull ); + void *zConverted = 0; + if( zFull==0 ){ + return 0; + } + if( winFullPathname(pVfs, zFilename, nFull, zFull)!=SQLITE_OK ){ + sqlite3_free(zFull); + return 0; + } + zConverted = winConvertFromUtf8Filename(zFull); + sqlite3_free(zFull); +#else void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); +#endif if( zConverted==0 ){ return 0; } From 15ab9418d9b220a826c033847e074529c2b22dd0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Feb 2014 14:24:01 +0000 Subject: [PATCH 004/177] Remove an unused variable from the VDBE_PROFILE compile-time option. Keep the opcode count in an u32 instead of an int. FossilOrigin-Name: 4df0ac9023d9261145a4425a508ba009a10276fc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 6 ------ src/vdbe.h | 2 +- src/vdbeaux.c | 4 +++- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 419633cc87..349983f4f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\stext\sof\sa\scomment\sused\sto\sgenerate\sVDBE\sopcode\sdocumentation\sso\sthat\nit\somits\ssymbols\sthat\scan\sbe\smistaken\sfor\sa\shyperlink\sby\sthe\sparser. -D 2014-02-20T19:42:00.531 +C Remove\san\sunused\svariable\sfrom\sthe\sVDBE_PROFILE\scompile-time\soption.\s\sKeep\sthe\nopcode\scount\sin\san\su32\sinstead\sof\san\sint. +D 2014-02-24T14:24:01.038 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,11 +277,11 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c c5ff7b384ed108be747220d4c47fcea6a36b062b -F src/vdbe.h 6c703ccef97f4504bd0d79cc09180185a60ae8ad +F src/vdbe.c 6c8f28911e702151c8ef03c568db5a066d3a85d4 +F src/vdbe.h 147027d6e8e667a63e87177a38e2b42c71fdacf8 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 0e01d6fda149c689039caadb8c89b20abb58e21d +F src/vdbeaux.c 8b8eeb3cd89e4b3d4f40186344915b49b7c1c0f7 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 06603e8e9d2f3247b68c6bbe4bd37fb6721b5bda F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 824029090d1c4c4c608f67dd197961eba29c039d -R 819e2e4df777a2f59b5f5445b2831eb5 +P b46d4e8923e6e367412bba7aeac07039bbcbabd1 +R e5107e17f1549034e719556b6caa9337 U drh -Z c4d587ace814e560a6dc901fdec24d15 +Z 29723a7a1d524b99d64cf7ed14fa2052 diff --git a/manifest.uuid b/manifest.uuid index 32d13dfb30..2aabeadce3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b46d4e8923e6e367412bba7aeac07039bbcbabd1 \ No newline at end of file +4df0ac9023d9261145a4425a508ba009a10276fc \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 2253af4719..5ab0127658 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -492,7 +492,6 @@ int sqlite3VdbeExec( i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ #ifdef VDBE_PROFILE u64 start; /* CPU clock count at start of opcode */ - int origPc; /* Program counter at start of opcode */ #endif /*** INSERT STACK UNION HERE ***/ @@ -554,7 +553,6 @@ int sqlite3VdbeExec( assert( pc>=0 && pcnOp ); if( db->mallocFailed ) goto no_mem; #ifdef VDBE_PROFILE - origPc = pc; start = sqlite3Hwtime(); #endif nVmStep++; @@ -6275,10 +6273,6 @@ default: { /* This is really OP_Noop and OP_Explain */ u64 elapsed = sqlite3Hwtime() - start; pOp->cycles += elapsed; pOp->cnt++; -#if 0 - fprintf(stdout, "%10llu ", elapsed); - sqlite3VdbePrintOp(stdout, origPc, &aOp[origPc]); -#endif } #endif diff --git a/src/vdbe.h b/src/vdbe.h index 44cd9123aa..66c69eec23 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -65,7 +65,7 @@ struct VdbeOp { char *zComment; /* Comment to improve readability */ #endif #ifdef VDBE_PROFILE - int cnt; /* Number of times this instruction was executed */ + u32 cnt; /* Number of times this instruction was executed */ u64 cycles; /* Total time spent executing this instruction */ #endif #ifdef SQLITE_VDBE_COVERAGE diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 89bb8975a5..ef70f8ae3f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2580,11 +2580,13 @@ int sqlite3VdbeReset(Vdbe *p){ if( pc!='\n' ) fprintf(out, "\n"); } for(i=0; inOp; i++){ - fprintf(out, "%6d %10lld %8lld ", + char zHdr[100]; + sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", p->aOp[i].cnt, p->aOp[i].cycles, p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 ); + fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); } fclose(out); From 9d41bc105d1bc5da9431180f8295ce52bcc344e6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Feb 2014 19:07:51 +0000 Subject: [PATCH 005/177] In memsys5, initialize new allocations to non-zero bytes. Change the content of freed allocations to prevent use after free. These changes in SQLITE_DEBUG only. FossilOrigin-Name: ba5f0a5599dece6d8f3dfe652800c28875c74a24 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem5.c | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 349983f4f3..ea4b646be9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\svariable\sfrom\sthe\sVDBE_PROFILE\scompile-time\soption.\s\sKeep\sthe\nopcode\scount\sin\san\su32\sinstead\sof\san\sint. -D 2014-02-24T14:24:01.038 +C In\smemsys5,\sinitialize\snew\sallocations\sto\snon-zero\sbytes.\s\sChange\sthe\scontent\nof\sfreed\sallocations\sto\sprevent\suse\safter\sfree.\s\sThese\schanges\sin\sSQLITE_DEBUG\nonly. +D 2014-02-24T19:07:51.519 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 -F src/mem5.c 19d9271cb936742707b6118ed44d779657c7c511 +F src/mem5.c aeb019f271ea53de83d651ec526877e6ba863450 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b46d4e8923e6e367412bba7aeac07039bbcbabd1 -R e5107e17f1549034e719556b6caa9337 +P 4df0ac9023d9261145a4425a508ba009a10276fc +R d75bf2d1dc1afee51bd2a2b5f0c23dc4 U drh -Z 29723a7a1d524b99d64cf7ed14fa2052 +Z e260958c722bfd17d5ba07190b7e2529 diff --git a/manifest.uuid b/manifest.uuid index 2aabeadce3..99dcc1b1b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4df0ac9023d9261145a4425a508ba009a10276fc \ No newline at end of file +ba5f0a5599dece6d8f3dfe652800c28875c74a24 \ No newline at end of file diff --git a/src/mem5.c b/src/mem5.c index 4674ec68f5..5d75611a32 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -275,6 +275,12 @@ static void *memsys5MallocUnsafe(int nByte){ if( mem5.maxCount Date: Mon, 24 Feb 2014 19:49:36 +0000 Subject: [PATCH 006/177] Close some stray connections in walro.test. FossilOrigin-Name: d6746cabaa22429644c956a4b5fd91896376de2d --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/walro.test | 14 ++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ea4b646be9..916f53cc00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\smemsys5,\sinitialize\snew\sallocations\sto\snon-zero\sbytes.\s\sChange\sthe\scontent\nof\sfreed\sallocations\sto\sprevent\suse\safter\sfree.\s\sThese\schanges\sin\sSQLITE_DEBUG\nonly. -D 2014-02-24T19:07:51.519 +C Close\ssome\sstray\sconnections\sin\swalro.test. +D 2014-02-24T19:49:36.852 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1064,7 +1064,7 @@ F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 F test/walpersist.test 8c6b7e3ec1ba91b5e4dc4e0921d6d3f87cd356a6 -F test/walro.test 6cc247a0cc9b36aeea2057dd28a922a1cdfbd630 +F test/walro.test 34422d1d95aaff0388f0791ec20edb34e2a3ed57 F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4df0ac9023d9261145a4425a508ba009a10276fc -R d75bf2d1dc1afee51bd2a2b5f0c23dc4 -U drh -Z e260958c722bfd17d5ba07190b7e2529 +P ba5f0a5599dece6d8f3dfe652800c28875c74a24 +R 55b5056ebfab7e0aa95c7fc2968f106d +U dan +Z 43d53118dac35420ad2faa5a259188da diff --git a/manifest.uuid b/manifest.uuid index 99dcc1b1b7..62abb66e87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba5f0a5599dece6d8f3dfe652800c28875c74a24 \ No newline at end of file +d6746cabaa22429644c956a4b5fd91896376de2d \ No newline at end of file diff --git a/test/walro.test b/test/walro.test index f1ed275a1d..6d920b1e24 100644 --- a/test/walro.test +++ b/test/walro.test @@ -32,9 +32,6 @@ ifcapable !wal { } do_multiclient_test tn { - # Do not run tests with the connections in the same process. - # - if {$tn==2} continue # Close all connections and delete the database. # @@ -43,6 +40,10 @@ do_multiclient_test tn { code3 { db3 close } forcedelete test.db forcedelete walro + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue foreach c {code1 code2 code3} { $c { @@ -232,9 +233,6 @@ forcedelete test.db # database file while a checkpoint operation is ongoing. # do_multiclient_test tn { - # Do not run tests with the connections in the same process. - # - if {$tn==2} continue # Close all connections and delete the database. # @@ -243,6 +241,10 @@ do_multiclient_test tn { code3 { db3 close } forcedelete test.db forcedelete walro + + # Do not run tests with the connections in the same process. + # + if {$tn==2} continue foreach c {code1 code2 code3} { $c { From 1925a2e6f88d288b38474053356cb399df5d410f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 24 Feb 2014 21:20:25 +0000 Subject: [PATCH 007/177] Enhance testing support for the Win32 VFS dynamic extension loading facilities. FossilOrigin-Name: 53081e7202fb55f792a78e73627466eb0200f11c --- Makefile.msc | 11 +++++++++++ manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os_win.c | 13 ++++++++----- src/test_loadext.c | 6 ++++++ test/loadext.test | 21 +++++++++++++++++---- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 61e5102607..119468a92d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1319,6 +1319,10 @@ testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) $(TESTFIXTURE_SRC) \ /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +extensiontest: testfixture.exe testloadext.dll + .\testfixture.exe $(TOP)\test\loadext.test + .\testfixture.exe $(TOP)\test\loadext2.test + fulltest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\all.test @@ -1345,6 +1349,12 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) -DBUILD_sqlite -DTCLSH=2 -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +testloadext.lo: $(TOP)\src\test_loadext.c + $(LTCOMPILE) -c $(TOP)\src\test_loadext.c + +testloadext.dll: testloadext.lo + $(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /OUT:$@ testloadext.lo + showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) @@ -1371,6 +1381,7 @@ clean: -rmdir /Q/S tsrc del /Q .target_source del /Q tclsqlite3.exe tclsqlite3.exp + del /Q testloadext.dll testloadext.exp del /Q testfixture.exe testfixture.exp test.db del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c sqlite3-*.c diff --git a/manifest b/manifest index 916f53cc00..7b0e38f502 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Close\ssome\sstray\sconnections\sin\swalro.test. -D 2014-02-24T19:49:36.852 +C Enhance\stesting\ssupport\sfor\sthe\sWin32\sVFS\sdynamic\sextension\sloading\sfacilities. +D 2014-02-24T21:20:25.412 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 371a17ef9cd011b103cfa1964119a8528f5427c9 +F Makefile.msc 4b39e17eb244d2fd3fb825d896380b38b782e69a F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 0dc30ad5cf90736d5fd9e540c9f05c542658abe7 @@ -203,7 +203,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 -F src/os_win.c d4284f003445054a26689f1264b1b9bf7261bd1b +F src/os_win.c 166d2f195ea0b7b4ea948229f727f0a293dd3c65 F src/pager.c 0ffa313a30ed6d061d9c6601b7b175cc50a1cab7 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 @@ -249,7 +249,7 @@ F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 -F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e +F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 @@ -647,7 +647,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test e191e536d0fcd722a6b965e7cd1ee0bfd12a5991 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 -F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 +F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ba5f0a5599dece6d8f3dfe652800c28875c74a24 -R 55b5056ebfab7e0aa95c7fc2968f106d -U dan -Z 43d53118dac35420ad2faa5a259188da +P d6746cabaa22429644c956a4b5fd91896376de2d +R 92acab9fb5e448fe2bef6b9a7d9bb046 +U mistachkin +Z 2600b243b92aaa5b75c087148e5db2b4 diff --git a/manifest.uuid b/manifest.uuid index 62abb66e87..8331b56d75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6746cabaa22429644c956a4b5fd91896376de2d \ No newline at end of file +53081e7202fb55f792a78e73627466eb0200f11c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d393c0d016..ac5eb886e3 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5135,15 +5135,12 @@ static int winFullPathname( ** Interfaces for opening a shared library, finding entry points ** within the shared library, and closing the shared library. */ -/* -** Interfaces for opening a shared library, finding entry points -** within the shared library, and closing the shared library. -*/ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; void *zConverted = winConvertFromUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); if( zConverted==0 ){ + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)0)); return 0; } if( osIsNT() ){ @@ -5158,6 +5155,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ h = osLoadLibraryA((char*)zConverted); } #endif + OSTRACE(("DLOPEN name=%s, handle=%p\n", zFilename, (void*)h)); sqlite3_free(zConverted); return (void*)h; } @@ -5166,12 +5164,17 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); } static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ + FARPROC pProc; UNUSED_PARAMETER(pVfs); - return (void(*)(void))osGetProcAddressA((HANDLE)pH, zSym); + pProc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, proc=%p\n", + (void*)pH, zSym, (void*)pProc)); + return (void(*)(void))pProc; } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); osFreeLibrary((HANDLE)pHandle); + OSTRACE(("DLCLOSE handle=%p\n", (void*)pHandle)); } #else /* if SQLITE_OMIT_LOAD_EXTENSION is defined: */ #define winDlOpen 0 diff --git a/src/test_loadext.c b/src/test_loadext.c index 1137e3a9aa..5a1f46da9c 100644 --- a/src/test_loadext.c +++ b/src/test_loadext.c @@ -91,6 +91,9 @@ static void statusFunc( /* ** Extension load function. */ +#ifdef _WIN32 +__declspec(dllexport) +#endif int testloadext_init( sqlite3 *db, char **pzErrMsg, @@ -109,6 +112,9 @@ int testloadext_init( /* ** Another extension entry point. This one always fails. */ +#ifdef _WIN32 +__declspec(dllexport) +#endif int testbrokenext_init( sqlite3 *db, char **pzErrMsg, diff --git a/test/loadext.test b/test/loadext.test index 0d5b841980..7ba4c0cf77 100644 --- a/test/loadext.test +++ b/test/loadext.test @@ -66,6 +66,12 @@ if {$::tcl_platform(os) eq "Darwin"} { set dlerror_nosymbol {dlsym(XXX, %2$s): symbol not found} } +if {$::tcl_platform(platform) eq "windows"} { + set dlerror_nosuchfile {The specified module could not be found.*} + set dlerror_notadll {%%1 is not a valid Win32 application.*} + set dlerror_nosymbol {The specified procedure could not be found.*} +} + # Make sure the test extension actually exists. If it does not # exist, try to create it. If unable to create it, then skip this # test file. @@ -167,7 +173,7 @@ do_test loadext-2.3 { regsub {0x[1234567890abcdefABCDEF]*} $msg XXX msg } list $rc $msg -} [list 1 [format $dlerror_nosymbol $testextension icecream]] +} /[list 1 [format $dlerror_nosymbol $testextension icecream]]/ # Try to load an extension for which the entry point fails (returns non-zero) # @@ -267,10 +273,17 @@ do_malloc_test loadext-5 -tclprep { } -tclbody { if {[autoinstall_test_functions]==7} {error "out of memory"} } -do_malloc_test loadext-6 -tclbody { - db enable_load_extension 1 - sqlite3_load_extension db $::testextension testloadext_init + +# On Windows, this malloc test must be skipped because the winDlOpen +# function itself can fail due to "out of memory" conditions. +# +if {$::tcl_platform(platform) ne "windows"} { + do_malloc_test loadext-6 -tclbody { + db enable_load_extension 1 + sqlite3_load_extension db $::testextension testloadext_init + } } + autoinstall_test_functions finish_test From 151d05cd39f6693cd085c281bec6fd96c73bfe3c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 24 Feb 2014 21:42:33 +0000 Subject: [PATCH 008/177] Fix several small style/consistency issues with the previous check-in. FossilOrigin-Name: 2477b3a294beb35f806a38f3ebc008942ba01617 --- Makefile.msc | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 119468a92d..5d6460b7d5 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1321,7 +1321,6 @@ testfixture.exe: $(TESTFIXTURE_SRC) $(LIBRESOBJS) $(HDR) extensiontest: testfixture.exe testloadext.dll .\testfixture.exe $(TOP)\test\loadext.test - .\testfixture.exe $(TOP)\test\loadext2.test fulltest: testfixture.exe sqlite3.exe .\testfixture.exe $(TOP)\test\all.test @@ -1373,6 +1372,7 @@ clean: del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* del /Q mkkeywordhash.exe keywordhash.h + del /Q notasharedlib.* -rmdir /Q/S .deps -rmdir /Q/S .libs -rmdir /Q/S quota2a diff --git a/manifest b/manifest index 7b0e38f502..ee235b0695 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Enhance\stesting\ssupport\sfor\sthe\sWin32\sVFS\sdynamic\sextension\sloading\sfacilities. -D 2014-02-24T21:20:25.412 +C Fix\sseveral\ssmall\sstyle/consistency\sissues\swith\sthe\sprevious\scheck-in. +D 2014-02-24T21:42:33.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 4b39e17eb244d2fd3fb825d896380b38b782e69a +F Makefile.msc fdacba6fb574868c71fde6db6b77789a383a4c19 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 0dc30ad5cf90736d5fd9e540c9f05c542658abe7 @@ -203,7 +203,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 -F src/os_win.c 166d2f195ea0b7b4ea948229f727f0a293dd3c65 +F src/os_win.c c47f107fc5c9d3466c06ea4aa35822f3568e81ee F src/pager.c 0ffa313a30ed6d061d9c6601b7b175cc50a1cab7 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d6746cabaa22429644c956a4b5fd91896376de2d -R 92acab9fb5e448fe2bef6b9a7d9bb046 +P 53081e7202fb55f792a78e73627466eb0200f11c +R eaeb88d7c9d1a25aae3665432185c5f6 U mistachkin -Z 2600b243b92aaa5b75c087148e5db2b4 +Z 8d06aaa0b8bcae8ffbf3fa0332e7604d diff --git a/manifest.uuid b/manifest.uuid index 8331b56d75..9798e1319d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53081e7202fb55f792a78e73627466eb0200f11c \ No newline at end of file +2477b3a294beb35f806a38f3ebc008942ba01617 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index ac5eb886e3..55ade5c7e2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -5164,12 +5164,12 @@ static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); } static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ - FARPROC pProc; + FARPROC proc; UNUSED_PARAMETER(pVfs); - pProc = osGetProcAddressA((HANDLE)pH, zSym); - OSTRACE(("DLSYM handle=%p, symbol=%s, proc=%p\n", - (void*)pH, zSym, (void*)pProc)); - return (void(*)(void))pProc; + proc = osGetProcAddressA((HANDLE)pH, zSym); + OSTRACE(("DLSYM handle=%p, symbol=%s, address=%p\n", + (void*)pH, zSym, (void*)proc)); + return (void(*)(void))proc; } static void winDlClose(sqlite3_vfs *pVfs, void *pHandle){ UNUSED_PARAMETER(pVfs); From 5e990bea516af17f76e29853f2eb65cdd7bafe46 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Feb 2014 14:52:01 +0000 Subject: [PATCH 009/177] QNX requires that _FILE_OFFSET_BITS be set prior to any system #include. FossilOrigin-Name: 953cdd6adadfd46b51ad61d7939cecca154a02cb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ee235b0695..c264178bb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sseveral\ssmall\sstyle/consistency\sissues\swith\sthe\sprevious\scheck-in. -D 2014-02-24T21:42:33.898 +C QNX\srequires\sthat\s_FILE_OFFSET_BITS\sbe\sset\sprior\sto\sany\ssystem\s#include. +D 2014-02-25T14:52:01.414 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 616f8a225bfd6cf09f7f5906c7b91b29a3591b34 +F src/sqliteInt.h 6725dc99b3985a97bad315910e59c3f7f5058916 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 53081e7202fb55f792a78e73627466eb0200f11c -R eaeb88d7c9d1a25aae3665432185c5f6 -U mistachkin -Z 8d06aaa0b8bcae8ffbf3fa0332e7604d +P 2477b3a294beb35f806a38f3ebc008942ba01617 +R 2f8f4da586e3827cd7880a024d14180d +U drh +Z d8c76b3503c5028bde8839a0e5ab0156 diff --git a/manifest.uuid b/manifest.uuid index 9798e1319d..bb142df59c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2477b3a294beb35f806a38f3ebc008942ba01617 \ No newline at end of file +953cdd6adadfd46b51ad61d7939cecca154a02cb \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a6663ff679..72fe47be5a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -12,7 +12,6 @@ ** Internal interface definitions for SQLite. ** */ -#include "sqlite3.h" #ifndef _SQLITEINT_H_ #define _SQLITEINT_H_ @@ -48,6 +47,11 @@ # define _LARGEFILE_SOURCE 1 #endif +/* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear +** first in QNX. +*/ +#include "sqlite3.h" + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build From e8c1eba9a99a0251433aa58d0fd96155d47b18fe Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Feb 2014 18:12:58 +0000 Subject: [PATCH 010/177] Also adjust the order of files in the amalgamation to ensure that _FILE_OFFSET_BITS is defined before any #include, for QNX. FossilOrigin-Name: 23001a85cd334090cf6c70d4d7e722a01f4f6899 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/mksqlite3c.tcl | 2 -- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c264178bb7..f25d3185ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C QNX\srequires\sthat\s_FILE_OFFSET_BITS\sbe\sset\sprior\sto\sany\ssystem\s#include. -D 2014-02-25T14:52:01.414 +C Also\sadjust\sthe\sorder\sof\sfiles\sin\sthe\samalgamation\sto\sensure\sthat\n_FILE_OFFSET_BITS\sis\sdefined\sbefore\sany\s#include,\sfor\sQNX. +D 2014-02-25T18:12:58.878 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1120,7 +1120,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl e2ba20d3f690990079d17f3e5a7417dfb7ada543 +F tool/mksqlite3c.tcl ed5b4e9ca8bf209ff401059a16362f37710fd8b8 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/mkvsix.tcl 6477fb9dab838b7eea1eed50658ff1cda04850b5 @@ -1151,7 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2477b3a294beb35f806a38f3ebc008942ba01617 -R 2f8f4da586e3827cd7880a024d14180d +P 953cdd6adadfd46b51ad61d7939cecca154a02cb +R d169e5164f16cb0355a3972eba04326c U drh -Z d8c76b3503c5028bde8839a0e5ab0156 +Z 6c6f89e1dd41d2a146386cfa06ae53d9 diff --git a/manifest.uuid b/manifest.uuid index bb142df59c..a069c374c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -953cdd6adadfd46b51ad61d7939cecca154a02cb \ No newline at end of file +23001a85cd334090cf6c70d4d7e722a01f4f6899 \ No newline at end of file diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index de9b48b97b..7a4fe1b389 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -121,7 +121,6 @@ foreach hdr { set available_hdr($hdr) 1 } set available_hdr(sqliteInt.h) 0 -set available_hdr(sqlite3.h) 0 # 78 stars used for comment formatting. set s78 \ @@ -229,7 +228,6 @@ proc copy_file {filename} { # inlining opportunities. # foreach file { - sqlite3.h sqliteInt.h global.c From 1fed5dab0dedceca104c51670cf74114587b536f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Feb 2014 21:01:25 +0000 Subject: [PATCH 011/177] Attempt to speed up sqlite3VdbeRecordCompare() by various means. This code is in an interim state. FossilOrigin-Name: 85206e0bbac29adab52bef795f6d1479f2ae2c0e --- manifest | 31 ++-- manifest.uuid | 2 +- src/btree.c | 12 +- src/sqliteInt.h | 7 +- src/vdbe.c | 23 ++- src/vdbe.h | 3 + src/vdbeaux.c | 413 ++++++++++++++++++++++++++++++++++++++++++++--- src/vdbemem.c | 113 ------------- src/vdbesort.c | 2 +- test/pragma.test | 3 + 10 files changed, 440 insertions(+), 169 deletions(-) diff --git a/manifest b/manifest index f25d3185ff..2d70d7d908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Also\sadjust\sthe\sorder\sof\sfiles\sin\sthe\samalgamation\sto\sensure\sthat\n_FILE_OFFSET_BITS\sis\sdefined\sbefore\sany\s#include,\sfor\sQNX. -D 2014-02-25T18:12:58.878 +C Attempt\sto\sspeed\sup\ssqlite3VdbeRecordCompare()\sby\svarious\smeans.\sThis\scode\sis\sin\san\sinterim\sstate. +D 2014-02-25T21:01:25.824 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b945df4f0114b4cc71006acc2fbb1333fb33a200 +F src/btree.c f4d85c3e2e189a219965d6d4525330333735fd1d F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6725dc99b3985a97bad315910e59c3f7f5058916 +F src/sqliteInt.h 6d868994f476b616ddb1795a51aa83c331ef5a62 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,14 +277,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 6c8f28911e702151c8ef03c568db5a066d3a85d4 -F src/vdbe.h 147027d6e8e667a63e87177a38e2b42c71fdacf8 +F src/vdbe.c ab910206dd8c9c5c1455f82953934bdbfe0bcc2a +F src/vdbe.h 0758eff7f1bf939bcafa377b2fafba4f5be63007 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 8b8eeb3cd89e4b3d4f40186344915b49b7c1c0f7 +F src/vdbeaux.c 988269c675ebb75a3610479840b65bcb8ea92647 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 06603e8e9d2f3247b68c6bbe4bd37fb6721b5bda -F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 +F src/vdbemem.c 25cc487244bf6ad647105c5adbc3052403dfd143 +F src/vdbesort.c 5144d84bd7d0d2545af1c6322edbbf07f97e3892 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -735,7 +735,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 -F test/pragma.test e882183ecd21d064cec5c7aaea174fbd36293429 +F test/pragma.test c1bc3a29f664b57d0ae40b498ad31f370de8daa6 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 @@ -1151,7 +1151,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 953cdd6adadfd46b51ad61d7939cecca154a02cb -R d169e5164f16cb0355a3972eba04326c -U drh -Z 6c6f89e1dd41d2a146386cfa06ae53d9 +P 23001a85cd334090cf6c70d4d7e722a01f4f6899 +R 04ade79c9b23add6f2d6adf0c34db1e4 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z d3f147085a7b8a8eca4578239439871b diff --git a/manifest.uuid b/manifest.uuid index a069c374c5..dd611d410f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23001a85cd334090cf6c70d4d7e722a01f4f6899 \ No newline at end of file +85206e0bbac29adab52bef795f6d1479f2ae2c0e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4fd8e27bf4..6c5f02403e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4547,6 +4547,7 @@ int sqlite3BtreeMovetoUnpacked( int *pRes /* Write search results here */ ){ int rc; + int (*xRecordCompare)(int, const void*, UnpackedRecord*); assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -4568,6 +4569,11 @@ int sqlite3BtreeMovetoUnpacked( } } + if( pIdxKey ){ + xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1); + } + rc = moveToRoot(pCur); if( rc ){ return rc; @@ -4652,14 +4658,14 @@ int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = sqlite3VdbeRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -4680,7 +4686,7 @@ int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + c = xRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } if( c<0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 72fe47be5a..76dea413e0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1590,15 +1590,10 @@ struct KeyInfo { struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ - u8 flags; /* Boolean settings. UNPACKED_... below */ + char default_rc; /* Comparison result if keys are equal */ Mem *aMem; /* Values */ }; -/* -** Allowed values of UnpackedRecord.flags -*/ -#define UNPACKED_INCRKEY 0x01 /* Make this key an epsilon larger */ -#define UNPACKED_PREFIX_MATCH 0x02 /* A prefix match is considered OK */ /* ** Each SQL index is represented in memory by an diff --git a/src/vdbe.c b/src/vdbe.c index 5ab0127658..097f182a23 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3555,16 +3555,16 @@ case OP_SeekGT: { /* jump, in3 */ /* The next line of code computes as follows, only faster: ** if( oc==OP_SeekGT || oc==OP_SeekLE ){ - ** r.flags = UNPACKED_INCRKEY; + ** r.default_rc = -1; ** }else{ - ** r.flags = 0; + ** r.default_rc = +1; ** } */ - r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLT))); - assert( oc!=OP_SeekGT || r.flags==UNPACKED_INCRKEY ); - assert( oc!=OP_SeekLE || r.flags==UNPACKED_INCRKEY ); - assert( oc!=OP_SeekGE || r.flags==0 ); - assert( oc!=OP_SeekLT || r.flags==0 ); + r.default_rc = ((1 & (oc - OP_SeekLT)) ? -1 : +1); + assert( oc!=OP_SeekGT || r.default_rc==-1 ); + assert( oc!=OP_SeekLE || r.default_rc==-1 ); + assert( oc!=OP_SeekGE || r.default_rc==+1 ); + assert( oc!=OP_SeekLT || r.default_rc==+1 ); r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG @@ -3722,7 +3722,6 @@ case OP_Found: { /* jump, in3 */ if( ii ) REGISTER_TRACE(pOp->p3+ii, &r.aMem[ii]); #endif } - r.flags = UNPACKED_PREFIX_MATCH; pIdxKey = &r; }else{ pIdxKey = sqlite3VdbeAllocUnpackedRecord( @@ -3732,8 +3731,8 @@ case OP_Found: { /* jump, in3 */ assert( pIn3->flags & MEM_Blob ); assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); - pIdxKey->flags |= UNPACKED_PREFIX_MATCH; } + pIdxKey->default_rc = 0; if( pOp->opcode==OP_NoConflict ){ /* For the OP_NoConflict opcode, take the jump if any of the ** input fields are NULL, since any key with a NULL will not @@ -4622,7 +4621,7 @@ case OP_IdxDelete: { assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; - r.flags = UNPACKED_PREFIX_MATCH; + r.default_rc = 0; r.aMem = &aMem[pOp->p2]; #ifdef SQLITE_DEBUG { int i; for(i=0; ip4.i; if( pOp->opcodeopcode==OP_IdxLE || pOp->opcode==OP_IdxGT ); - r.flags = UNPACKED_INCRKEY | UNPACKED_PREFIX_MATCH; + r.default_rc = -1; }else{ assert( pOp->opcode==OP_IdxGE || pOp->opcode==OP_IdxLT ); - r.flags = UNPACKED_PREFIX_MATCH; + r.default_rc = 0; } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG diff --git a/src/vdbe.h b/src/vdbe.h index 66c69eec23..af07831faa 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -214,6 +214,9 @@ void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); +typedef int (*RecordCompare)(int,const void*, UnpackedRecord*); +RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *); + #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ef70f8ae3f..e817ccb23f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3102,7 +3102,7 @@ void sqlite3VdbeRecordUnpack( u32 szHdr; Mem *pMem = p->aMem; - p->flags = 0; + p->default_rc = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); idx = getVarint32(aKey, szHdr); d = szHdr; @@ -3123,6 +3123,215 @@ void sqlite3VdbeRecordUnpack( p->nField = u; } +static int vdbeRecordCompareString( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey1 = (const u8*)pKey1; + int szHdr; + int serial_type; + int res; + + szHdr = aKey1[0]; + getVarint32(&aKey1[1], serial_type); + + if( serial_type<12 ){ + res = -1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ + res = +1; /* (pKey1/nKey1) is a blob */ + }else{ + int nCmp; + int nStr; + aKey1 = &aKey1[szHdr]; + + nStr = (serial_type-12) / 2; + if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(aKey1, pPKey2->aMem[0].z, nCmp); + + if( res==0 ){ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ) res = pPKey2->default_rc; + } + } + + assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) + || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) + || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) + ); + return res; +} + +static int vdbeRecordCompareInt( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey1 = (const u8*)pKey1; + int szHdr; + int serial_type; + int res; + + szHdr = aKey1[0]; + getVarint32(&aKey1[1], serial_type); + + if( serial_type==0 ){ + res = -1; /* NULL values are smaller than integers */ + }else if( serial_type>=12 ){ + res = +1; /* text/blob values are greater */ + }else{ + Mem mem; + sqlite3VdbeSerialGet(&aKey1[szHdr], serial_type, &mem); + if( mem.flags & MEM_Int ){ + i64 v = pPKey2->aMem[0].u.i; + if( v>mem.u.i ){ + res = -1; + }else if( vdefault_rc; + } + }else{ + double v = (double)pPKey2->aMem[0].u.i; + if( v>mem.r ){ + res = -1; + }else if( vdefault_rc; + } + } + } + + assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) + || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) + || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) + ); + return res; +} + +static int vdbeCompareMemString( + const Mem *pMem1, + const Mem *pMem2, + const CollSeq *pColl +){ + if( pMem1->enc==pColl->enc ){ + /* The strings are already in the correct encoding. Call the + ** comparison function directly */ + return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); + }else{ + int rc; + const void *v1, *v2; + int n1, n2; + Mem c1; + Mem c2; + memset(&c1, 0, sizeof(c1)); + memset(&c2, 0, sizeof(c2)); + sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); + sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); + v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); + n1 = v1==0 ? 0 : c1.n; + v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); + n2 = v2==0 ? 0 : c2.n; + rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); + sqlite3VdbeMemRelease(&c1); + sqlite3VdbeMemRelease(&c2); + return rc; + } +} + +/* +** Compare the values contained by the two memory cells, returning +** negative, zero or positive if pMem1 is less than, equal to, or greater +** than pMem2. Sorting order is NULL's first, followed by numbers (integers +** and reals) sorted numerically, followed by text ordered by the collating +** sequence pColl and finally blob's ordered by memcmp(). +** +** Two NULL values are considered equal by this function. +*/ +int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ + int rc; + int f1, f2; + int combined_flags; + + f1 = pMem1->flags; + f2 = pMem2->flags; + combined_flags = f1|f2; + assert( (combined_flags & MEM_RowSet)==0 ); + + /* If one value is NULL, it is less than the other. If both values + ** are NULL, return 0. + */ + if( combined_flags&MEM_Null ){ + return (f2&MEM_Null) - (f1&MEM_Null); + } + + /* If one value is a number and the other is not, the number is less. + ** If both are numbers, compare as reals if one is a real, or as integers + ** if both values are integers. + */ + if( combined_flags&(MEM_Int|MEM_Real) ){ + double r1, r2; + if( (f1 & f2 & MEM_Int)!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return 1; + return 0; + } + if( (f1&MEM_Real)!=0 ){ + r1 = pMem1->r; + }else if( (f1&MEM_Int)!=0 ){ + r1 = (double)pMem1->u.i; + }else{ + return 1; + } + if( (f2&MEM_Real)!=0 ){ + r2 = pMem2->r; + }else if( (f2&MEM_Int)!=0 ){ + r2 = (double)pMem2->u.i; + }else{ + return -1; + } + if( r1r2 ) return 1; + return 0; + } + + /* If one value is a string and the other is a blob, the string is less. + ** If both are strings, compare using the collating functions. + */ + if( combined_flags&MEM_Str ){ + if( (f1 & MEM_Str)==0 ){ + return 1; + } + if( (f2 & MEM_Str)==0 ){ + return -1; + } + + assert( pMem1->enc==pMem2->enc ); + assert( pMem1->enc==SQLITE_UTF8 || + pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); + + /* The collation sequence must be defined at this point, even if + ** the user deletes the collation sequence after the vdbe program is + ** compiled (this was not always the case). + */ + assert( !pColl || pColl->xCmp ); + + if( pColl ){ + return vdbeCompareMemString(pMem1, pMem2, pColl); + } + /* If a NULL pointer was passed as the collate function, fall through + ** to the blob case and use memcmp(). */ + } + + /* Both values must be blobs. Compare using memcmp(). */ + rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); + if( rc==0 ){ + rc = pMem1->n - pMem2->n; + } + return rc; +} + + /* ** This function compares the two table rows or index records ** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero @@ -3140,7 +3349,7 @@ void sqlite3VdbeRecordUnpack( ** equal, then the keys are considered to be equal and ** the parts beyond the common prefix are ignored. */ -int sqlite3VdbeRecordCompare( +static int vdbeRecordComparePrev( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2 /* Right key */ ){ @@ -3216,25 +3425,192 @@ int sqlite3VdbeRecordCompare( assert( mem1.zMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. If the UNPACKED_INCRKEY - ** flag is set, then break the tie by treating key2 as larger. - ** If the UPACKED_PREFIX_MATCH flag is set, then keys with common prefixes - ** are considered to be equal. Otherwise, the longer key is the - ** larger. As it happens, the pPKey2 will always be the longer - ** if there is a difference. - */ - assert( rc==0 ); - if( pPKey2->flags & UNPACKED_INCRKEY ){ - rc = -1; - }else if( pPKey2->flags & UNPACKED_PREFIX_MATCH ){ - /* Leave rc==0 */ - }else if( idx1default_rc; } +int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + u32 d1; /* Offset into aKey[] of next data element */ + u32 idx1; /* Offset into aKey[] of next header element */ + u32 szHdr1; /* Number of bytes in header */ + int i = 0; + int rc = 0; + KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + Mem mem1; + +#ifdef SQLITE_DEBUG + int expected = vdbeRecordComparePrev(nKey1, pKey1, pPKey2); + static int nCall = 0; + nCall++; +#endif + + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField + || CORRUPT_DB ); + assert( pPKey2->pKeyInfo->aSortOrder!=0 ); + assert( pPKey2->pKeyInfo->nField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + Mem *pRhs = &pPKey2->aMem[i]; + u32 serial_type; + + /* RHS is an integer */ + if( pRhs->flags & MEM_Int ){ + serial_type = aKey1[idx1]; + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( serial_type==7 ){ + double rhs = (double)pRhs->u.i; + if( mem1.rrhs ){ + rc = +1; + } + }else{ + i64 rhs = pRhs->u.i; + if( mem1.u.irhs ){ + rc = +1; + } + } + } + } + + /* RHS is real */ + else if( pRhs->flags & MEM_Real ){ + serial_type = aKey1[idx1]; + if( serial_type>=12 ){ + rc = +1; + }else if( serial_type==0 ){ + rc = -1; + }else{ + double rhs = pRhs->r; + double lhs; + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + if( serial_type==7 ){ + lhs = mem1.r; + }else{ + lhs = mem1.u.i; + } + if( lhsrhs ){ + rc = +1; + } + } + } + + /* RHS is a string */ + else if( pRhs->flags & MEM_Str ){ + getVarint32(&aKey1[idx1], serial_type); + if( serial_type<12 ){ + rc = -1; + }else if( !(serial_type & 0x01) ){ + rc = +1; + }else{ + mem1.n = (serial_type - 12) / 2; + if( (d1+mem1.n) > nKey1 ){ + rc = 1; /* Corruption */ + }else if( pKeyInfo->aColl[i] ){ + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + mem1.flags = MEM_Str; + mem1.z = &aKey1[d1]; + rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + }else{ + int nCmp = MIN(mem1.n, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = mem1.n - pRhs->n; + } + } + } + + /* RHS is a blob */ + else if( pRhs->flags & MEM_Blob ){ + getVarint32(&aKey1[idx1], serial_type); + if( serial_type<12 || (serial_type & 0x01) ){ + rc = -1; + }else{ + int nStr = (serial_type - 12) / 2; + if( (d1+nStr) > nKey1 ){ + rc = 1; /* Corruption */ + }else{ + int nCmp = MIN(nStr, pRhs->n); + rc = memcmp(&aKey1[d1], pRhs->z, nCmp); + if( rc==0 ) rc = nStr - pRhs->n; + } + } + } + + /* RHS is null */ + else{ + serial_type = aKey1[idx1]; + rc = (serial_type!=0); + } + + if( rc!=0 ){ + assert( mem1.zMalloc==0 ); /* See comment below */ + if( pKeyInfo->aSortOrder[i] ){ + rc = -rc; +#if 0 + assert( (rc>0 && (rc^(int)0x80000000)<0) + || (rc<0 && (rc^(int)0x80000000)>0) ); + assert( sizeof(int)==4 ); + rc ^= (int)0x80000000; /* similar in spirit to: "rc = -rc;" */ + assert( rc!=0 ); +#endif + } + assert( (rc<0 && expected<0) || (rc>0 && expected>0) || CORRUPT_DB ); + return rc; + } + + i++; + d1 += sqlite3VdbeSerialTypeLen(serial_type); + idx1 += sqlite3VarintLen(serial_type); + }while( idx1nField && d1<=nKey1 ); + + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). + */ + assert( mem1.zMalloc==0 ); + + /* rc==0 here means that one or both of the keys ran out of fields and + ** all the fields up to that point were equal. Return the the default_rc + ** value. */ + assert( pPKey2->default_rc==expected ); + return pPKey2->default_rc; +} + +RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ + if( p->nField==1 && p->pKeyInfo->aSortOrder[0]==0 ){ + int flags = p->aMem[0].flags; + if( (flags & MEM_Int) ){ + return vdbeRecordCompareInt; + }else if( (p->aMem[0].flags&(MEM_Int|MEM_Real|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ + return vdbeRecordCompareString; + } + } + return sqlite3VdbeRecordCompare; +} + + /* ** pCur points at an index entry created using the OP_MakeRecord opcode. ** Read the rowid (the last field in the record) and store it in *rowid. @@ -3347,7 +3723,6 @@ int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - assert( pUnpacked->flags & UNPACKED_PREFIX_MATCH ); *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); sqlite3VdbeMemRelease(&m); return SQLITE_OK; diff --git a/src/vdbemem.c b/src/vdbemem.c index bcf9586b3f..4936f40016 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -754,119 +754,6 @@ int sqlite3VdbeMemSetStr( return SQLITE_OK; } -/* -** Compare the values contained by the two memory cells, returning -** negative, zero or positive if pMem1 is less than, equal to, or greater -** than pMem2. Sorting order is NULL's first, followed by numbers (integers -** and reals) sorted numerically, followed by text ordered by the collating -** sequence pColl and finally blob's ordered by memcmp(). -** -** Two NULL values are considered equal by this function. -*/ -int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; - int f1, f2; - int combined_flags; - - f1 = pMem1->flags; - f2 = pMem2->flags; - combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); - - /* If one value is NULL, it is less than the other. If both values - ** are NULL, return 0. - */ - if( combined_flags&MEM_Null ){ - return (f2&MEM_Null) - (f1&MEM_Null); - } - - /* If one value is a number and the other is not, the number is less. - ** If both are numbers, compare as reals if one is a real, or as integers - ** if both values are integers. - */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - double r1, r2; - if( (f1 & f2 & MEM_Int)!=0 ){ - if( pMem1->u.i < pMem2->u.i ) return -1; - if( pMem1->u.i > pMem2->u.i ) return 1; - return 0; - } - if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->r; - }else if( (f1&MEM_Int)!=0 ){ - r1 = (double)pMem1->u.i; - }else{ - return 1; - } - if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->r; - }else if( (f2&MEM_Int)!=0 ){ - r2 = (double)pMem2->u.i; - }else{ - return -1; - } - if( r1r2 ) return 1; - return 0; - } - - /* If one value is a string and the other is a blob, the string is less. - ** If both are strings, compare using the collating functions. - */ - if( combined_flags&MEM_Str ){ - if( (f1 & MEM_Str)==0 ){ - return 1; - } - if( (f2 & MEM_Str)==0 ){ - return -1; - } - - assert( pMem1->enc==pMem2->enc ); - assert( pMem1->enc==SQLITE_UTF8 || - pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); - - /* The collation sequence must be defined at this point, even if - ** the user deletes the collation sequence after the vdbe program is - ** compiled (this was not always the case). - */ - assert( !pColl || pColl->xCmp ); - - if( pColl ){ - if( pMem1->enc==pColl->enc ){ - /* The strings are already in the correct encoding. Call the - ** comparison function directly */ - return pColl->xCmp(pColl->pUser,pMem1->n,pMem1->z,pMem2->n,pMem2->z); - }else{ - const void *v1, *v2; - int n1, n2; - Mem c1; - Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); - sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); - sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); - v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - n1 = v1==0 ? 0 : c1.n; - v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - n2 = v2==0 ? 0 : c2.n; - rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); - sqlite3VdbeMemRelease(&c1); - sqlite3VdbeMemRelease(&c2); - return rc; - } - } - /* If a NULL pointer was passed as the collate function, fall through - ** to the blob case and use memcmp(). */ - } - - /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; - } - return rc; -} - /* ** Move data out of a btree key or data field and into a Mem structure. ** The data or key is taken from the entry that pCur is currently pointing diff --git a/src/vdbesort.c b/src/vdbesort.c index 3e4cad5b4a..c4d15d6872 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -409,7 +409,7 @@ static void vdbeSorterCompare( return; } } - r2->flags |= UNPACKED_PREFIX_MATCH; + assert( r2->default_rc==0 ); } *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); diff --git a/test/pragma.test b/test/pragma.test index 1043170378..fcbb1808fa 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1575,6 +1575,8 @@ do_test pragma-20.8 { forcedelete data_dir } ;# endif windows +database_may_be_corrupt + do_test 21.1 { # Create a corrupt database in testerr.db. And a non-corrupt at test.db. # @@ -1680,4 +1682,5 @@ do_test 23.5 { } } {0 0 t1 y {} {NO ACTION} {NO ACTION} NONE} +database_never_corrupt finish_test From 0baa035a1e3339d8bb0aea2205a2dee4f56fcc9e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Feb 2014 21:55:16 +0000 Subject: [PATCH 012/177] Do not allow temporary registers to be in use across an OP_Yield within a co-routine. Fix for ticket [8c63ff0eca81a9132d8]. FossilOrigin-Name: 97a8c9733cba97c78e979dfd5c66610c23e90288 --- manifest | 15 +++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 1 - src/where.c | 8 +++---- test/tkt-8c63ff0ec.test | 48 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 test/tkt-8c63ff0ec.test diff --git a/manifest b/manifest index f25d3185ff..e66acbb7e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Also\sadjust\sthe\sorder\sof\sfiles\sin\sthe\samalgamation\sto\sensure\sthat\n_FILE_OFFSET_BITS\sis\sdefined\sbefore\sany\s#include,\sfor\sQNX. -D 2014-02-25T18:12:58.878 +C Do\snot\sallow\stemporary\sregisters\sto\sbe\sin\suse\sacross\san\sOP_Yield\swithin\sa\nco-routine.\s\sFix\sfor\sticket\s[8c63ff0eca81a9132d8]. +D 2014-02-25T21:55:16.402 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6725dc99b3985a97bad315910e59c3f7f5058916 +F src/sqliteInt.h 46dfbe0b58282421188a6c25b6c0c0fae18e0134 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -290,7 +290,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d622974f30d3347c7b71bfe49ce1f1e9b6570980 +F src/where.c 0b5a4d45fdf05ad541893d986abd2f660f6e0094 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -881,6 +881,7 @@ F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 +F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 @@ -1151,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 953cdd6adadfd46b51ad61d7939cecca154a02cb -R d169e5164f16cb0355a3972eba04326c +P 23001a85cd334090cf6c70d4d7e722a01f4f6899 +R 53d871612a374a058c13125cf02ae108 U drh -Z 6c6f89e1dd41d2a146386cfa06ae53d9 +Z 196bbfe69fc9298304e0610559f3cd1e diff --git a/manifest.uuid b/manifest.uuid index a069c374c5..a96614dc89 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23001a85cd334090cf6c70d4d7e722a01f4f6899 \ No newline at end of file +97a8c9733cba97c78e979dfd5c66610c23e90288 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 72fe47be5a..c7f2941d5d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2358,7 +2358,6 @@ struct Parse { u8 checkSchema; /* Causes schema cookie check after an error */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ - u8 nTempInUse; /* Number of aTempReg[] currently checked out */ u8 nColCache; /* Number of entries in aColCache[] */ u8 iColCache; /* Next entry in aColCache[] to replace */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ diff --git a/src/where.c b/src/where.c index d606ba4c18..f5d44a3877 100644 --- a/src/where.c +++ b/src/where.c @@ -2853,13 +2853,14 @@ static Bitmask codeOneLoopStart( ** construct. */ assert( pLoop->u.btree.nEq==1 ); - iReleaseReg = sqlite3GetTempReg(pParse); pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); + iReleaseReg = ++pParse->nMem; iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); + if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); @@ -2948,7 +2949,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = start; assert( pLevel->p5==0 ); if( testOp!=OP_Noop ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); @@ -3181,7 +3182,7 @@ static Bitmask codeOneLoopStart( if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse); + iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg); /* Deferred seek */ @@ -3529,7 +3530,6 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } } - sqlite3ReleaseTempReg(pParse, iReleaseReg); return pLevel->notReady; } diff --git a/test/tkt-8c63ff0ec.test b/test/tkt-8c63ff0ec.test new file mode 100644 index 0000000000..d4aaefd74a --- /dev/null +++ b/test/tkt-8c63ff0ec.test @@ -0,0 +1,48 @@ +# 2014-02-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases to show that ticket [8c63ff0eca81a9132d8d67b31cd6ae9712a2cc6f] +# "Incorrect query result on a UNION ALL" which was caused by using the same +# temporary register in concurrent co-routines, as been fixed. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix tkt-8c63ff0ec + +do_execsql_test 1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d, e); + INSERT INTO t1 VALUES(1,20,30,40,50),(3,60,70,80,90); + CREATE TABLE t2(x INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES(2); + CREATE TABLE t3(z); + INSERT INTO t3 VALUES(2),(2),(2),(2); + + SELECT a, b+c FROM t1 + UNION ALL + SELECT x, 5 FROM t2 JOIN t3 ON z=x WHERE x=2 + ORDER BY a; +} {1 50 2 5 2 5 2 5 2 5 3 130} +do_execsql_test 1.2 { + SELECT a, b+c+d FROM t1 + UNION ALL + SELECT x, 5 FROM t2 JOIN t3 ON z=x WHERE x=2 + ORDER BY a; +} {1 90 2 5 2 5 2 5 2 5 3 210} +do_execsql_test 1.3 { + SELECT a, b+c+d+e FROM t1 + UNION ALL + SELECT x, 5 FROM t2 JOIN t3 ON z=x WHERE x=2 + ORDER BY a; +} {1 140 2 5 2 5 2 5 2 5 3 300} + +finish_test From 434a93147cfa23b7a9bd49fef8f1f1b59c9da4d3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Feb 2014 02:26:09 +0000 Subject: [PATCH 013/177] Improved handling of constants and especially constant functions in the ORDER BY clause of a query. Do not optimize out "ORDER BY random()". Fix for ticket [65bdeb9739605cc2296]. FossilOrigin-Name: dca1945aeb3fb005263f9be00ee8e72b966ae303 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 21 ++++----------------- test/where2.test | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index e66acbb7e2..1acd1fc253 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\stemporary\sregisters\sto\sbe\sin\suse\sacross\san\sOP_Yield\swithin\sa\nco-routine.\s\sFix\sfor\sticket\s[8c63ff0eca81a9132d8]. -D 2014-02-25T21:55:16.402 +C Improved\shandling\sof\sconstants\sand\sespecially\sconstant\sfunctions\sin\sthe\nORDER\sBY\sclause\sof\sa\squery.\s\sDo\snot\soptimize\sout\s"ORDER\sBY\srandom()".\nFix\sfor\sticket\s[65bdeb9739605cc2296]. +D 2014-02-26T02:26:09.921 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0b5a4d45fdf05ad541893d986abd2f660f6e0094 +F src/where.c 6042e1a377cf7dc72c10493269ed75e276275cd8 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1070,7 +1070,7 @@ F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b -F test/where2.test ed6baa9420a109d8be683dbef5d153d186f3690b +F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 23001a85cd334090cf6c70d4d7e722a01f4f6899 -R 53d871612a374a058c13125cf02ae108 +P 97a8c9733cba97c78e979dfd5c66610c23e90288 +R a3177866e2afe2abf843086af2a9a3a2 U drh -Z 196bbfe69fc9298304e0610559f3cd1e +Z c50f86b62c93dcd00c53d2ca75883787 diff --git a/manifest.uuid b/manifest.uuid index a96614dc89..68c025b08a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97a8c9733cba97c78e979dfd5c66610c23e90288 \ No newline at end of file +dca1945aeb3fb005263f9be00ee8e72b966ae303 \ No newline at end of file diff --git a/src/where.c b/src/where.c index f5d44a3877..a5dd7b59e7 100644 --- a/src/where.c +++ b/src/where.c @@ -4901,9 +4901,12 @@ static int wherePathSatisfiesOrderBy( orderDistinctMask |= pLoop->maskSelf; for(i=0; ia[i].pExpr; - if( (exprTableUsage(&pWInfo->sMaskSet, p)&~orderDistinctMask)==0 ){ + mTerm = exprTableUsage(&pWInfo->sMaskSet,p); + if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } } @@ -5526,22 +5529,6 @@ WhereInfo *sqlite3WhereBegin( goto whereBeginError; } - /* If the ORDER BY (or GROUP BY) clause contains references to general - ** expressions, then we won't be able to satisfy it using indices, so - ** go ahead and disable it now. - */ - if( pOrderBy && (wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ - for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOrderBy->a[ii].pExpr); - if( pExpr->op!=TK_COLUMN ){ - pWInfo->pOrderBy = pOrderBy = 0; - break; - }else if( pExpr->iColumn<0 ){ - break; - } - } - } - if( wctrlFlags & WHERE_WANT_DISTINCT ){ if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ diff --git a/test/where2.test b/test/where2.test index 45ea486d95..d9b2b23de9 100644 --- a/test/where2.test +++ b/test/where2.test @@ -121,6 +121,42 @@ do_test where2-2.3 { } } {85 6 7396 7402 nosort t1 *} +# Ticket [65bdeb9739605cc22966f49208452996ff29a640] 2014-02-26 +# Make sure "ORDER BY random" does not gets optimized out. +# +do_test where2-2.4 { + db eval { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b DEFAULT 1); + WITH RECURSIVE + cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<50) + INSERT INTO x1 SELECT x, 1 FROM cnt; + CREATE TABLE x2(x INTEGER PRIMARY KEY); + INSERT INTO x2 VALUES(1); + } + set sql {SELECT * FROM x1, x2 WHERE x=1 ORDER BY random()} + set out1 [db eval $sql] + set out2 [db eval $sql] + set out3 [db eval $sql] + expr {$out1!=$out2 && $out2!=$out3} +} {1} +do_execsql_test where2-2.5 { + -- random() is not optimized out + EXPLAIN SELECT * FROM x1, x2 WHERE x=1 ORDER BY random(); +} {/ random/} +do_execsql_test where2-2.5b { + -- random() is not optimized out + EXPLAIN SELECT * FROM x1, x2 WHERE x=1 ORDER BY random(); +} {/ SorterOpen /} +do_execsql_test where2-2.6 { + -- other constant functions are optimized out + EXPLAIN SELECT * FROM x1, x2 WHERE x=1 ORDER BY abs(5); +} {~/ abs/} +do_execsql_test where2-2.6b { + -- other constant functions are optimized out + EXPLAIN SELECT * FROM x1, x2 WHERE x=1 ORDER BY abs(5); +} {~/ SorterOpen /} + + # Efficient handling of forward and reverse table scans. # From 3852b68edd2d378f9f4a4b2377654ecbb6d423c1 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Feb 2014 13:53:34 +0000 Subject: [PATCH 014/177] In the command-line shell for CSV import, if the lines are \r\n terminated and the last field is blank, make sure an empty string and not a "\r" string is imported. FossilOrigin-Name: 9c2e7612cd137895e13ee872e668ce1216719d85 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 2 +- test/shell5.test | 19 +++++++++++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1acd1fc253..b398e0004c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\shandling\sof\sconstants\sand\sespecially\sconstant\sfunctions\sin\sthe\nORDER\sBY\sclause\sof\sa\squery.\s\sDo\snot\soptimize\sout\s"ORDER\sBY\srandom()".\nFix\sfor\sticket\s[65bdeb9739605cc2296]. -D 2014-02-26T02:26:09.921 +C In\sthe\scommand-line\sshell\sfor\sCSV\simport,\sif\sthe\slines\sare\s\\r\\n\sterminated\nand\sthe\slast\sfield\sis\sblank,\smake\ssure\san\sempty\sstring\sand\snot\sa\s"\\r"\sstring\nis\simported. +D 2014-02-26T13:53:34.384 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 28bff39f9bc5ec618b0719fe3f7b4be9f88b6f02 -F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 +F src/shell.c ca2b066ab31793532530e0c2fcf79e99628f3b2b F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -804,7 +804,7 @@ F test/shell1.test f2a1d471e5cd2b42f7a65b166dc1ace2b8d11583 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 -F test/shell5.test cee83b4385f842fec1f2a0bec9ea811f35386edf +F test/shell5.test bb755ea9144b8078a752fc56223582627070b5f1 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 97a8c9733cba97c78e979dfd5c66610c23e90288 -R a3177866e2afe2abf843086af2a9a3a2 +P dca1945aeb3fb005263f9be00ee8e72b966ae303 +R 4bbcd8ca02f23a972c97ceb5ec0d3c0c U drh -Z c50f86b62c93dcd00c53d2ca75883787 +Z 998297c355a76316139497aa81dafb02 diff --git a/manifest.uuid b/manifest.uuid index 68c025b08a..b18a32bd5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dca1945aeb3fb005263f9be00ee8e72b966ae303 \ No newline at end of file +9c2e7612cd137895e13ee872e668ce1216719d85 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 266bb8f90f..7bb8da3e0f 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1893,7 +1893,7 @@ static char *csv_read_one_field(CSVReader *p){ } if( c=='\n' ){ p->nLine++; - if( p->n>1 && p->z[p->n-1]=='\r' ) p->n--; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; } p->cTerm = c; } diff --git a/test/shell5.test b/test/shell5.test index ce05a303ca..4c38b75e6f 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -285,6 +285,25 @@ do_test shell5-1.10 { db eval {SELECT hex(c) FROM t1 ORDER BY rowid} } {636F6C756D6E33 783320220D0A64617461222033 783320220A64617461222033} +# Blank last column with \r\n line endings. +do_test shell5-1.11 { + set out [open shell5.csv w] + fconfigure $out -translation binary + puts $out "column1,column2,column3\r" + puts $out "a,b, \r" + puts $out "x,y,\r" + puts $out "p,q,r\r" + close $out + catch {db close} + forcedelete test.db + catchcmd test.db {.mode csv +.import shell5.csv t1 + } + sqlite3 db test.db + db eval {SELECT *, '|' FROM t1} +} {a b { } | x y {} | p q r |} + + db close finish_test From 7e4efaecad8b403574647d08d2f47f0cc60679ef Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Feb 2014 21:35:31 +0000 Subject: [PATCH 015/177] More efficient removal of duplicates in recursive queries using the UNION compound operator. FossilOrigin-Name: 06c2db875e6cabd81c2a153b2fa4aba03eaed268 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b398e0004c..9c72c3971c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell\sfor\sCSV\simport,\sif\sthe\slines\sare\s\\r\\n\sterminated\nand\sthe\slast\sfield\sis\sblank,\smake\ssure\san\sempty\sstring\sand\snot\sa\s"\\r"\sstring\nis\simported. -D 2014-02-26T13:53:34.384 +C More\sefficient\sremoval\sof\sduplicates\sin\srecursive\squeries\susing\sthe\sUNION\ncompound\soperator. +D 2014-02-26T21:35:31.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 28bff39f9bc5ec618b0719fe3f7b4be9f88b6f02 +F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 F src/shell.c ca2b066ab31793532530e0c2fcf79e99628f3b2b F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dca1945aeb3fb005263f9be00ee8e72b966ae303 -R 4bbcd8ca02f23a972c97ceb5ec0d3c0c +P 9c2e7612cd137895e13ee872e668ce1216719d85 +R 8af1befb78d550449f6afe7ccc2f524b U drh -Z 998297c355a76316139497aa81dafb02 +Z 54acbfe19bff5d9f12c833489ea306bb diff --git a/manifest.uuid b/manifest.uuid index b18a32bd5f..b0059e4d9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c2e7612cd137895e13ee872e668ce1216719d85 \ No newline at end of file +06c2db875e6cabd81c2a153b2fa4aba03eaed268 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 93947d56b2..a8cb341c65 100644 --- a/src/select.c +++ b/src/select.c @@ -818,14 +818,16 @@ static void selectInnerLoop( r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempRange(pParse, nKey+2); r3 = r2+nKey+1; - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); if( eDest==SRT_DistQueue ){ /* If the destination is DistQueue, then cursor (iParm+1) is open ** on a second ephemeral index that holds all values every previously - ** added to the queue. Only add this new value if it has never before - ** been added */ - addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + ** added to the queue. */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + regResult, nResultCol); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); } From c7181907688fa0d2a2bcabff7f392757e6efab96 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Feb 2014 15:04:13 +0000 Subject: [PATCH 016/177] Add #defines to the command-line shell source code (shell.c) so that it can be imported directly into other projects (ex: Fossil) and used without editing. FossilOrigin-Name: a6690400235705ecc0d1a60dacff6ad5fb1f944a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9c72c3971c..7ec2f4b525 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sefficient\sremoval\sof\sduplicates\sin\srecursive\squeries\susing\sthe\sUNION\ncompound\soperator. -D 2014-02-26T21:35:31.484 +C Add\s#defines\sto\sthe\scommand-line\sshell\ssource\scode\s(shell.c)\sso\sthat\sit\ncan\sbe\simported\sdirectly\sinto\sother\sprojects\s(ex:\sFossil)\sand\sused\swithout\s\nediting. +D 2014-02-27T15:04:13.436 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 -F src/shell.c ca2b066ab31793532530e0c2fcf79e99628f3b2b +F src/shell.c 720e79facdae4787e7bfa577dae6b37a685244b0 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9c2e7612cd137895e13ee872e668ce1216719d85 -R 8af1befb78d550449f6afe7ccc2f524b +P 06c2db875e6cabd81c2a153b2fa4aba03eaed268 +R b72dc063fb220f3231791c5c73300f82 U drh -Z 54acbfe19bff5d9f12c833489ea306bb +Z d3f85704b6a67858d1c2e6d5e8f9f735 diff --git a/manifest.uuid b/manifest.uuid index b0059e4d9c..03c67c2593 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06c2db875e6cabd81c2a153b2fa4aba03eaed268 \ No newline at end of file +a6690400235705ecc0d1a60dacff6ad5fb1f944a \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7bb8da3e0f..3cd7aaf892 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1008,7 +1008,7 @@ static int run_table_dump_query( int nResult; int i; const char *z; - rc = sqlite3_prepare(p->db, zSelect, -1, &pSelect, 0); + rc = sqlite3_prepare_v2(p->db, zSelect, -1, &pSelect, 0); if( rc!=SQLITE_OK || !pSelect ){ fprintf(p->out, "/**** ERROR: (%d) %s *****/\n", rc, sqlite3_errmsg(p->db)); if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; @@ -1456,7 +1456,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ zTableInfo = appendText(zTableInfo, zTable, '"'); zTableInfo = appendText(zTableInfo, ");", 0); - rc = sqlite3_prepare(p->db, zTableInfo, -1, &pTableInfo, 0); + rc = sqlite3_prepare_v2(p->db, zTableInfo, -1, &pTableInfo, 0); free(zTableInfo); if( rc!=SQLITE_OK || !pTableInfo ){ return 1; @@ -2399,7 +2399,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ return 1; } nByte = strlen30(zSql); - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); if( rc && sqlite3_strglob("no such table: *", sqlite3_errmsg(db))==0 ){ char *zCreate = sqlite3_mprintf("CREATE TABLE %s", zTable); char cSep = '('; @@ -2425,7 +2425,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ xCloser(sCsv.in); return 1; } - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); } sqlite3_free(zSql); if( rc ){ @@ -2452,7 +2452,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } zSql[j++] = ')'; zSql[j] = 0; - rc = sqlite3_prepare(p->db, zSql, -1, &pStmt, 0); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc ){ fprintf(stderr, "Error: %s\n", sqlite3_errmsg(db)); @@ -3548,11 +3548,13 @@ int main(int argc, char **argv){ int rc = 0; int warnInmemoryDb = 0; +#if !defined(USE_SYSTEM_SQLITE) || USE_SYSTEM_SQLITE!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); exit(1); } +#endif Argv0 = argv[0]; main_init(&data); stdin_is_interactive = isatty(0); @@ -3645,6 +3647,11 @@ int main(int argc, char **argv){ #else fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; +#endif +#ifdef SQLITE_SHELL_DBNAME_PROC + { extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; } #endif } data.out = stdout; From 69b30abd598ed827cddeede9d133c83e579517c3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Feb 2014 15:11:52 +0000 Subject: [PATCH 017/177] A better way of handling the USE_SYSTEM_SQLITE #define in shell.c. FossilOrigin-Name: 51ce713c6ee91bdf0126155334dcc800b3daa509 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7ec2f4b525..6892cecea2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#defines\sto\sthe\scommand-line\sshell\ssource\scode\s(shell.c)\sso\sthat\sit\ncan\sbe\simported\sdirectly\sinto\sother\sprojects\s(ex:\sFossil)\sand\sused\swithout\s\nediting. -D 2014-02-27T15:04:13.436 +C A\sbetter\sway\sof\shandling\sthe\sUSE_SYSTEM_SQLITE\s#define\sin\sshell.c. +D 2014-02-27T15:11:52.919 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 -F src/shell.c 720e79facdae4787e7bfa577dae6b37a685244b0 +F src/shell.c bf75ce6bea4c8f56c1b46bee201c25acddffb675 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 06c2db875e6cabd81c2a153b2fa4aba03eaed268 -R b72dc063fb220f3231791c5c73300f82 +P a6690400235705ecc0d1a60dacff6ad5fb1f944a +R d9e4cc8c2428822c2a58511d3bd2892d U drh -Z d3f85704b6a67858d1c2e6d5e8f9f735 +Z 2ca6d024109a270000c3879d5e05ce0a diff --git a/manifest.uuid b/manifest.uuid index 03c67c2593..5a0d8578c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a6690400235705ecc0d1a60dacff6ad5fb1f944a \ No newline at end of file +51ce713c6ee91bdf0126155334dcc800b3daa509 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3cd7aaf892..6815c489e6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3548,7 +3548,7 @@ int main(int argc, char **argv){ int rc = 0; int warnInmemoryDb = 0; -#if !defined(USE_SYSTEM_SQLITE) || USE_SYSTEM_SQLITE!=1 +#if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ fprintf(stderr, "SQLite header and source version mismatch\n%s\n%s\n", sqlite3_sourceid(), SQLITE_SOURCE_ID); From 3b9330f83ce327bd34acb6e48a910fcc41f2dd92 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Feb 2014 20:44:18 +0000 Subject: [PATCH 018/177] Further changes to sqlite3VdbeRecordCompare(). FossilOrigin-Name: 570893740067a7caa952f259fa078cdf67017d71 --- manifest | 23 +-- manifest.uuid | 2 +- src/btree.c | 13 +- src/sqliteInt.h | 2 + src/vdbe.h | 5 +- src/vdbeaux.c | 495 +++++++++++++++++++++++++++++------------------- src/vdbesort.c | 5 + 7 files changed, 333 insertions(+), 212 deletions(-) diff --git a/manifest b/manifest index 2d70d7d908..e7df2f4996 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sspeed\sup\ssqlite3VdbeRecordCompare()\sby\svarious\smeans.\sThis\scode\sis\sin\san\sinterim\sstate. -D 2014-02-25T21:01:25.824 +C Further\schanges\sto\ssqlite3VdbeRecordCompare(). +D 2014-02-27T20:44:18.479 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c f4d85c3e2e189a219965d6d4525330333735fd1d +F src/btree.c 77f175987c80ebec063f8653cb7d300776411413 F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 @@ -221,7 +221,7 @@ F src/shell.c 3dd86bf73ccd079f0e32ef5069600586085e8239 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6d868994f476b616ddb1795a51aa83c331ef5a62 +F src/sqliteInt.h 7b42e02c1ca4599b5420d44cb549460b2348139a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -278,13 +278,13 @@ F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c ab910206dd8c9c5c1455f82953934bdbfe0bcc2a -F src/vdbe.h 0758eff7f1bf939bcafa377b2fafba4f5be63007 +F src/vdbe.h 6833579fc0fbdc1c933e34519064841abda5b9b3 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 988269c675ebb75a3610479840b65bcb8ea92647 +F src/vdbeaux.c 80e5315957377554c9011858f5afde61afedc181 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 25cc487244bf6ad647105c5adbc3052403dfd143 -F src/vdbesort.c 5144d84bd7d0d2545af1c6322edbbf07f97e3892 +F src/vdbesort.c 72290f12428973c2c6b9d4f95ad0a7c8181e1280 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1151,10 +1151,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 23001a85cd334090cf6c70d4d7e722a01f4f6899 -R 04ade79c9b23add6f2d6adf0c34db1e4 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P 85206e0bbac29adab52bef795f6d1479f2ae2c0e +R eb22cd7a0c8c7d3bc8c75c472c26dd6a U dan -Z d3f147085a7b8a8eca4578239439871b +Z 91158d7f280c804b2fe508b4511e22e9 diff --git a/manifest.uuid b/manifest.uuid index dd611d410f..927ab5603a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -85206e0bbac29adab52bef795f6d1479f2ae2c0e \ No newline at end of file +570893740067a7caa952f259fa078cdf67017d71 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6c5f02403e..c0b04cc08c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4547,7 +4547,7 @@ int sqlite3BtreeMovetoUnpacked( int *pRes /* Write search results here */ ){ int rc; - int (*xRecordCompare)(int, const void*, UnpackedRecord*); + RecordCompare xRecordCompare; assert( cursorHoldsMutex(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); @@ -4571,7 +4571,10 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1); + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); } rc = moveToRoot(pCur); @@ -4658,14 +4661,14 @@ int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[1], pCell[1], 1, pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[2], pCell[2], 1, pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -4686,7 +4689,7 @@ int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey); + c = xRecordCompare(nCell, pCellKey, ((u8*)pCellKey)[0], 1, pIdxKey); sqlite3_free(pCellKey); } if( c<0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 76dea413e0..b19a0bd453 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1592,6 +1592,8 @@ struct UnpackedRecord { u16 nField; /* Number of entries in apMem[] */ char default_rc; /* Comparison result if keys are equal */ Mem *aMem; /* Values */ + int r1; + int r2; }; diff --git a/src/vdbe.h b/src/vdbe.h index af07831faa..c08512e7aa 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -214,8 +214,9 @@ void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*, UnpackedRecord*); -RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *); +typedef int (*RecordCompare)(int,const void*,int,u32,UnpackedRecord*); +RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); +RecordCompare sqlite3VdbeFindSorterCompare(KeyInfo*); #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e817ccb23f..8ca605ebe6 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3123,90 +3123,102 @@ void sqlite3VdbeRecordUnpack( p->nField = u; } -static int vdbeRecordCompareString( +/* +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. +** +** Key1 and Key2 do not have to contain the same number of fields. +** The key with fewer fields is usually compares less than the +** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set +** and the common prefixes are equal, then key1 is less than key2. +** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are +** equal, then the keys are considered to be equal and +** the parts beyond the common prefix are ignored. +*/ +static int vdbeRecordComparePrev( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2 /* Right key */ ){ - const u8 *aKey1 = (const u8*)pKey1; - int szHdr; - int serial_type; - int res; + u32 d1; /* Offset into aKey[] of next data element */ + u32 idx1; /* Offset into aKey[] of next header element */ + u32 szHdr1; /* Number of bytes in header */ + int i = 0; + int rc = 0; + const unsigned char *aKey1 = (const unsigned char *)pKey1; + KeyInfo *pKeyInfo; + Mem mem1; - szHdr = aKey1[0]; - getVarint32(&aKey1[1], serial_type); + pKeyInfo = pPKey2->pKeyInfo; + mem1.enc = pKeyInfo->enc; + mem1.db = pKeyInfo->db; + /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ - if( serial_type<12 ){ - res = -1; /* (pKey1/nKey1) is a number or a null */ - }else if( !(serial_type & 0x01) ){ - res = +1; /* (pKey1/nKey1) is a blob */ - }else{ - int nCmp; - int nStr; - aKey1 = &aKey1[szHdr]; + /* Compilers may complain that mem1.u.i is potentially uninitialized. + ** We could initialize it, as shown here, to silence those complaints. + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing + ** the unnecessary initialization has a measurable negative performance + ** impact, since this routine is a very high runner. And so, we choose + ** to ignore the compiler warnings and leave this variable uninitialized. + */ + /* mem1.u.i = 0; // not needed, here to silence compiler warning */ + + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); + assert( pKeyInfo->aSortOrder!=0 ); + assert( pKeyInfo->nField>0 ); + assert( idx1<=szHdr1 || CORRUPT_DB ); + do{ + u32 serial_type1; - nStr = (serial_type-12) / 2; - if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ - nCmp = MIN( pPKey2->aMem[0].n, nStr ); - res = memcmp(aKey1, pPKey2->aMem[0].z, nCmp); + /* Read the serial types for the next element in each key. */ + idx1 += getVarint32( aKey1+idx1, serial_type1 ); - if( res==0 ){ - res = nStr - pPKey2->aMem[0].n; - if( res==0 ) res = pPKey2->default_rc; + /* Verify that there is enough key space remaining to avoid + ** a buffer overread. The "d1+serial_type1+2" subexpression will + ** always be greater than or equal to the amount of required key space. + ** Use that approximation to avoid the more expensive call to + ** sqlite3VdbeSerialTypeLen() in the common case. + */ + if( d1+serial_type1+2>(u32)nKey1 + && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 + ){ + break; } - } - assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) - || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) - || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) - ); - return res; -} + /* Extract the values to be compared. + */ + d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); -static int vdbeRecordCompareInt( - int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ -){ - const u8 *aKey1 = (const u8*)pKey1; - int szHdr; - int serial_type; - int res; - - szHdr = aKey1[0]; - getVarint32(&aKey1[1], serial_type); - - if( serial_type==0 ){ - res = -1; /* NULL values are smaller than integers */ - }else if( serial_type>=12 ){ - res = +1; /* text/blob values are greater */ - }else{ - Mem mem; - sqlite3VdbeSerialGet(&aKey1[szHdr], serial_type, &mem); - if( mem.flags & MEM_Int ){ - i64 v = pPKey2->aMem[0].u.i; - if( v>mem.u.i ){ - res = -1; - }else if( vdefault_rc; - } - }else{ - double v = (double)pPKey2->aMem[0].u.i; - if( v>mem.r ){ - res = -1; - }else if( vdefault_rc; + /* Do the comparison + */ + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); + if( rc!=0 ){ + assert( mem1.zMalloc==0 ); /* See comment below */ + if( pKeyInfo->aSortOrder[i] ){ + rc = -rc; /* Invert the result for DESC sort order. */ } + return rc; } - } + i++; + }while( idx1nField ); - assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) - || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) - || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) - ); - return res; + /* No memory allocation is ever used on mem1. Prove this using + ** the following assert(). If the assert() fails, it indicates a + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). + */ + assert( mem1.zMalloc==0 ); + + /* rc==0 here means that one of the keys ran out of fields and + ** all the fields up to that point were equal. Return the the default_rc + ** value. */ + return pPKey2->default_rc; } static int vdbeCompareMemString( @@ -3332,114 +3344,43 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ } -/* -** This function compares the two table rows or index records -** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero -** or positive integer if key1 is less than, equal to or -** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 -** key must be a parsed key such as obtained from -** sqlite3VdbeParseRecord. -** -** Key1 and Key2 do not have to contain the same number of fields. -** The key with fewer fields is usually compares less than the -** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set -** and the common prefixes are equal, then key1 is less than key2. -** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are -** equal, then the keys are considered to be equal and -** the parts beyond the common prefix are ignored. -*/ -static int vdbeRecordComparePrev( - int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ -){ - u32 d1; /* Offset into aKey[] of next data element */ - u32 idx1; /* Offset into aKey[] of next header element */ - u32 szHdr1; /* Number of bytes in header */ - int i = 0; - int rc = 0; - const unsigned char *aKey1 = (const unsigned char *)pKey1; - KeyInfo *pKeyInfo; - Mem mem1; +static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + switch( serial_type ){ + case 1: + return (char)aKey[0]; + case 2: + return ((char)aKey[0] << 8) | aKey[1]; + case 3: + return ((char)aKey[0] << 16) | (aKey[1] << 8) | aKey[2]; + case 4: + return ((char)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]; - pKeyInfo = pPKey2->pKeyInfo; - mem1.enc = pKeyInfo->enc; - mem1.db = pKeyInfo->db; - /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ - - /* Compilers may complain that mem1.u.i is potentially uninitialized. - ** We could initialize it, as shown here, to silence those complaints. - ** But in fact, mem1.u.i will never actually be used uninitialized, and doing - ** the unnecessary initialization has a measurable negative performance - ** impact, since this routine is a very high runner. And so, we choose - ** to ignore the compiler warnings and leave this variable uninitialized. - */ - /* mem1.u.i = 0; // not needed, here to silence compiler warning */ - - idx1 = getVarint32(aKey1, szHdr1); - d1 = szHdr1; - assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); - assert( pKeyInfo->aSortOrder!=0 ); - assert( pKeyInfo->nField>0 ); - assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ - u32 serial_type1; - - /* Read the serial types for the next element in each key. */ - idx1 += getVarint32( aKey1+idx1, serial_type1 ); - - /* Verify that there is enough key space remaining to avoid - ** a buffer overread. The "d1+serial_type1+2" subexpression will - ** always be greater than or equal to the amount of required key space. - ** Use that approximation to avoid the more expensive call to - ** sqlite3VdbeSerialTypeLen() in the common case. - */ - if( d1+serial_type1+2>(u32)nKey1 - && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 - ){ - break; + case 5: { + i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; + u32 lsw = (aKey[4] << 8) | aKey[5]; + return (i64)( msw << 16 | (u64)lsw ); } - /* Extract the values to be compared. - */ - d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); - - /* Do the comparison - */ - rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); - if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ - if( pKeyInfo->aSortOrder[i] ){ - rc = -rc; /* Invert the result for DESC sort order. */ - } - return rc; + case 6: { + i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; + u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; + return (i64)( msw << 32 | (u64)lsw ); } - i++; - }while( idx1nField ); + } - /* No memory allocation is ever used on mem1. Prove this using - ** the following assert(). If the assert() fails, it indicates a - ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). - */ - assert( mem1.zMalloc==0 ); - - /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc - ** value. */ - return pPKey2->default_rc; + return (serial_type - 8); } - -int sqlite3VdbeRecordCompare( +static int vdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ + int szHdr1, /* Size of record header in bytes */ + u32 idx1, /* Offset of first type in header */ + UnpackedRecord *const pPKey2 /* Right key */ ){ - u32 d1; /* Offset into aKey[] of next data element */ - u32 idx1; /* Offset into aKey[] of next header element */ - u32 szHdr1; /* Number of bytes in header */ + u32 d1 = szHdr1; /* Offset into aKey[] of next data element */ int i = 0; int rc = 0; + Mem *pRhs = pPKey2->aMem; KeyInfo *pKeyInfo = pPKey2->pKeyInfo; const unsigned char *aKey1 = (const unsigned char *)pKey1; Mem mem1; @@ -3450,17 +3391,25 @@ int sqlite3VdbeRecordCompare( nCall++; #endif + /* If idx==0, then the caller has already determined that the first two + ** elements in the keys are equal. Fix the various stack variables so + ** that this routine begins comparing at the second field. */ + if( idx1==0 ){ + u32 s1; + assert( sqlite3VarintLen(szHdr1)==1 ); + idx1 = 1 + getVarint32(&aKey1[1], s1); + d1 += sqlite3VdbeSerialTypeLen(s1); + i = 1; + pRhs++; + } + VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ - - idx1 = getVarint32(aKey1, szHdr1); - d1 = szHdr1; assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pPKey2->pKeyInfo->aSortOrder!=0 ); assert( pPKey2->pKeyInfo->nField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ - Mem *pRhs = &pPKey2->aMem[i]; u32 serial_type; /* RHS is an integer */ @@ -3470,22 +3419,21 @@ int sqlite3VdbeRecordCompare( rc = +1; }else if( serial_type==0 ){ rc = -1; - }else{ + }else if( serial_type==7 ){ + double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - if( serial_type==7 ){ - double rhs = (double)pRhs->u.i; - if( mem1.rrhs ){ - rc = +1; - } - }else{ - i64 rhs = pRhs->u.i; - if( mem1.u.irhs ){ - rc = +1; - } + if( mem1.rrhs ){ + rc = +1; + } + }else{ + i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); + i64 rhs = pRhs->u.i; + if( lhsrhs ){ + rc = +1; } } } @@ -3579,6 +3527,7 @@ int sqlite3VdbeRecordCompare( } i++; + pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); idx1 += sqlite3VarintLen(serial_type); }while( idx1nField && d1<=nKey1 ); @@ -3596,20 +3545,184 @@ int sqlite3VdbeRecordCompare( return pPKey2->default_rc; } +static int vdbeRecordCompareInt( + int nKey1, const void *pKey1, /* Left key */ + int szHdr, + u32 idx1, + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey = &((const u8*)pKey1)[szHdr]; + int serial_type = ((const u8*)pKey1)[1]; + int res; + i64 v = pPKey2->aMem[0].u.i; + i64 lhs; + + switch( serial_type ){ + case 1: + lhs = (char)(aKey[0]); + break; + case 2: + lhs = 256*(signed char)aKey[0] + aKey[1]; + break; + case 3: + lhs = 65536*(char)aKey[0] | (aKey[1]<<8) | aKey[2]; + break; + case 4: + lhs = (int)(((u32)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]); + break; + + case 5: { + i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; + u32 lsw = (aKey[4] << 8) | aKey[5]; + lhs = (i64)( msw << 16 | (u64)lsw ); + break; + } + + case 6: { + i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; + u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; + lhs = (i64)( msw << 32 | (u64)lsw ); + break; + } + + case 8: + lhs = 0; + break; + + case 9: + lhs = 1; + break; + + default: + return vdbeRecordCompare(nKey1, pKey1, szHdr, 1, pPKey2); + } + + if( v>lhs ){ + res = pPKey2->r1; + }else if( vr2; + }else if( pPKey2->nField>1 ){ + res = vdbeRecordCompare(nKey1, pKey1, szHdr, 0, pPKey2); + }else{ + res = pPKey2->default_rc; + } + + assert( (res==0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + ); + return res; +} + +static int vdbeRecordCompareString( + int nKey1, const void *pKey1, /* Left key */ + int szHdr, + u32 idx1, + UnpackedRecord *pPKey2 /* Right key */ +){ + const u8 *aKey1 = (const u8*)pKey1; + int serial_type; + int res; + + getVarint32(&aKey1[1], serial_type); + + if( serial_type<12 ){ + res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ + }else if( !(serial_type & 0x01) ){ + res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ + }else{ + int nCmp; + int nStr; + aKey1 = &aKey1[szHdr]; + + nStr = (serial_type-12) / 2; + if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ + nCmp = MIN( pPKey2->aMem[0].n, nStr ); + res = memcmp(aKey1, pPKey2->aMem[0].z, nCmp); + + if( res==0 ){ + res = nStr - pPKey2->aMem[0].n; + if( res==0 ){ + if( pPKey2->nField>1 ){ + res = vdbeRecordCompare(nKey1, pKey1, szHdr, 0, pPKey2); + }else{ + res = pPKey2->default_rc; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + }else if( res>0 ){ + res = pPKey2->r2; + }else{ + res = pPKey2->r1; + } + } + + assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) + || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) + || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) + || CORRUPT_DB + ); + return res; +} + + +int vdbeRecordCompareLargeHeader( + int nKey1, const void *pKey1, /* Left key */ + int dummy1, u32 dummy2, /* Unused arguments */ + UnpackedRecord *pPKey2 /* Right key */ +){ + int szHdr; + u32 idx1; + idx1 = getVarint32(((u8*)pKey1), szHdr); + return vdbeRecordCompare(nKey1, pKey1, szHdr, idx1, pPKey2); +} + RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ - if( p->nField==1 && p->pKeyInfo->aSortOrder[0]==0 ){ + if( (p->pKeyInfo->nField + p->pKeyInfo->nXField) > 10 ){ + return vdbeRecordCompareLargeHeader; + }else{ int flags = p->aMem[0].flags; + if( p->pKeyInfo->aSortOrder[0] ){ + p->r1 = 1; + p->r2 = -1; + }else{ + p->r1 = -1; + p->r2 = 1; + } if( (flags & MEM_Int) ){ return vdbeRecordCompareInt; - }else if( (p->aMem[0].flags&(MEM_Int|MEM_Real|MEM_Null|MEM_Blob))==0 - && p->pKeyInfo->aColl[0]==0 + } + if( (flags & (MEM_Int|MEM_Real|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 ){ return vdbeRecordCompareString; } } - return sqlite3VdbeRecordCompare; + + return vdbeRecordCompare; } +RecordCompare sqlite3VdbeFindSorterCompare(KeyInfo *pKeyInfo){ + if( (pKeyInfo->nField + pKeyInfo->nXField) > 10 ){ + return vdbeRecordCompareLargeHeader; + } + return vdbeRecordCompare; +} + +int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + int szHdr; + u32 idx1; + + idx1 = getVarint32(((u8*)pKey1), szHdr); + return vdbeRecordCompare(nKey1, pKey1, szHdr, idx1, pPKey2); +} /* ** pCur points at an index entry created using the OP_MakeRecord opcode. diff --git a/src/vdbesort.c b/src/vdbesort.c index c4d15d6872..be5a6064c2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,6 +105,7 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ + RecordCompare xRecordCompare; /* Record compare function */ }; /* @@ -412,7 +413,10 @@ static void vdbeSorterCompare( assert( r2->default_rc==0 ); } +#if 0 *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); +#endif + *pRes = pSorter->xRecordCompare(nKey1, pKey1, *((u8*)pKey1), 1, r2); } /* @@ -488,6 +492,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ if( mxCachemxPmaSize = mxCache * pgsz; } + pSorter->xRecordCompare = sqlite3VdbeFindSorterCompare(pCsr->pKeyInfo); return SQLITE_OK; } From 063d4a041a5f1bf8938c7a01c2c4c656bf9384ce Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Feb 2014 09:48:30 +0000 Subject: [PATCH 019/177] Minor tweak to vdbeRecordCompareInt(). FossilOrigin-Name: 284bde0ee20261737446eb8f5b6b36ad9bc3f355 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 14 ++++++++++++++ 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ab9697fb73..3b948c33cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\slatest\strunk\schanges. -D 2014-02-27T20:52:26.303 +C Minor\stweak\sto\svdbeRecordCompareInt(). +D 2014-02-28T09:48:30.694 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c ab910206dd8c9c5c1455f82953934bdbfe0bcc2a F src/vdbe.h 6833579fc0fbdc1c933e34519064841abda5b9b3 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 80e5315957377554c9011858f5afde61afedc181 +F src/vdbeaux.c fc95358566a454fadcf17858d582ff38c2e55c54 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 25cc487244bf6ad647105c5adbc3052403dfd143 F src/vdbesort.c 72290f12428973c2c6b9d4f95ad0a7c8181e1280 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 570893740067a7caa952f259fa078cdf67017d71 51ce713c6ee91bdf0126155334dcc800b3daa509 -R f4fa5a4e2d9c34dae5f26cd35d7bbdb3 +P 8f30b09518d23c3f6cecd244a66ef918fdb55323 +R c980d8afe2372c0277fee962182b32ac U dan -Z 489718005e4064c97910e87ca8ac5405 +Z 7196a891773623dba164b983a9b4c516 diff --git a/manifest.uuid b/manifest.uuid index ab94dbb62a..52a4739a0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f30b09518d23c3f6cecd244a66ef918fdb55323 \ No newline at end of file +284bde0ee20261737446eb8f5b6b36ad9bc3f355 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8ca605ebe6..8468d13b96 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3558,6 +3558,7 @@ static int vdbeRecordCompareInt( i64 lhs; switch( serial_type ){ + case 1: lhs = (char)(aKey[0]); break; @@ -3593,6 +3594,15 @@ static int vdbeRecordCompareInt( lhs = 1; break; + /* This case could be removed without changing the results of running + ** this code. Including it causes gcc to generate a faster switch + ** statement (since the range of switch targets now starts at zero and + ** is contiguous)) but does not cause any duplicate code to be generated + ** (as gcc is clever enough to combine the two like cases). Other + ** compilers might be similar. */ + case 0: case 7: + return vdbeRecordCompare(nKey1, pKey1, szHdr, 1, pPKey2); + default: return vdbeRecordCompare(nKey1, pKey1, szHdr, 1, pPKey2); } @@ -3602,8 +3612,12 @@ static int vdbeRecordCompareInt( }else if( vr2; }else if( pPKey2->nField>1 ){ + /* The first fields of the two keys are equal. Compare the trailing + ** fields. */ res = vdbeRecordCompare(nKey1, pKey1, szHdr, 0, pPKey2); }else{ + /* The first fields of the two keys are equal and there are no trailing + ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; } From 597515d70c009a74c54dd8a82acb2bfc66cb2664 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 28 Feb 2014 18:39:51 +0000 Subject: [PATCH 020/177] Update some test cases that deal with corrupt databases. FossilOrigin-Name: 3a09f5605ac7c6e503eb10acfdc607010414d917 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/vdbeaux.c | 2 +- src/vdbemem.c | 1 - test/analyze9.test | 3 +++ test/corruptG.test | 2 +- test/pragma.test | 15 ++++++++++----- 7 files changed, 26 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 3b948c33cf..f20f7a0353 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\stweak\sto\svdbeRecordCompareInt(). -D 2014-02-28T09:48:30.694 +C Update\ssome\stest\scases\sthat\sdeal\swith\scorrupt\sdatabases. +D 2014-02-28T18:39:51.462 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,9 +281,9 @@ F src/vdbe.c ab910206dd8c9c5c1455f82953934bdbfe0bcc2a F src/vdbe.h 6833579fc0fbdc1c933e34519064841abda5b9b3 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c fc95358566a454fadcf17858d582ff38c2e55c54 +F src/vdbeaux.c aad5345869c110f45074e2c1ca230e623b1ba182 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 25cc487244bf6ad647105c5adbc3052403dfd143 +F src/vdbemem.c 93fc3f13ebe6809ebaac8d0a17c812ec053ba233 F src/vdbesort.c 72290f12428973c2c6b9d4f95ad0a7c8181e1280 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -310,7 +310,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 339e87723cd4dc158dc5e9095acd8df9e87faf79 +F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -402,7 +402,7 @@ F test/corruptC.test 02405cf7ed0c1e989060e1aab6d02ffbc3906fbb F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 -F test/corruptG.test c150f156dace653c00a121ad0f5772a0568c41ba +F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 9d8186f6f8751efdfd445d8546fd98f073499039 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 @@ -735,7 +735,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 -F test/pragma.test c1bc3a29f664b57d0ae40b498ad31f370de8daa6 +F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8f30b09518d23c3f6cecd244a66ef918fdb55323 -R c980d8afe2372c0277fee962182b32ac +P 284bde0ee20261737446eb8f5b6b36ad9bc3f355 +R 75b55639e7bdd048d8263d0acfa3cf8a U dan -Z 7196a891773623dba164b983a9b4c516 +Z d116a418c5c97c83b8ebfe7a3e655c3c diff --git a/manifest.uuid b/manifest.uuid index 52a4739a0c..9cb218f1b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -284bde0ee20261737446eb8f5b6b36ad9bc3f355 \ No newline at end of file +3a09f5605ac7c6e503eb10acfdc607010414d917 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8468d13b96..007f4e650d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3597,7 +3597,7 @@ static int vdbeRecordCompareInt( /* This case could be removed without changing the results of running ** this code. Including it causes gcc to generate a faster switch ** statement (since the range of switch targets now starts at zero and - ** is contiguous)) but does not cause any duplicate code to be generated + ** is contiguous) but does not cause any duplicate code to be generated ** (as gcc is clever enough to combine the two like cases). Other ** compilers might be similar. */ case 0: case 7: diff --git a/src/vdbemem.c b/src/vdbemem.c index 4936f40016..bc703e8eeb 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -913,7 +913,6 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ if( pRec->pKeyInfo ){ assert( pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField==nCol ); assert( pRec->pKeyInfo->enc==ENC(db) ); - pRec->flags = UNPACKED_PREFIX_MATCH; pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); for(i=0; iaMem[i].flags = MEM_Null; diff --git a/test/analyze9.test b/test/analyze9.test index 0fce55ac4b..820bcdb0e7 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -326,6 +326,7 @@ do_execsql_test 6.2 { reset_db sqlite3_db_config_lookaside db 0 0 0 +database_may_be_corrupt do_execsql_test 7.1 { CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a, b); @@ -366,6 +367,8 @@ do_execsql_test 7.5 { SELECT * FROM t1 WHERE a = 5; } {5 5} +database_never_corrupt + #------------------------------------------------------------------------- # reset_db diff --git a/test/corruptG.test b/test/corruptG.test index 11253fd12c..1ec5d6f6a9 100644 --- a/test/corruptG.test +++ b/test/corruptG.test @@ -76,6 +76,6 @@ do_test 2.1 { # incorrect answer from a corrupt database file, that is OK. If the # result below changes, that just means that "undefined behavior" has # changed. -} {0 52} +} {/0 .*/} finish_test diff --git a/test/pragma.test b/test/pragma.test index fcbb1808fa..8f54e695d7 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1602,11 +1602,16 @@ Multiple uses for byte 672 of page 15} set auxerr {*** in database aux *** Multiple uses for byte 672 of page 15} +set mainerr {/{\*\*\* in database main \*\*\* +Multiple uses for byte 672 of page 15}.*/} +set auxerr {/{\*\*\* in database aux \*\*\* +Multiple uses for byte 672 of page 15}.*/} + do_test 22.2 { catch { db close } sqlite3 db testerr.db execsql { PRAGMA integrity_check } -} [list $mainerr] +} $mainerr do_test 22.3.1 { catch { db close } @@ -1615,13 +1620,13 @@ do_test 22.3.1 { ATTACH 'testerr.db' AS 'aux'; PRAGMA integrity_check; } -} [list $auxerr] +} $auxerr do_test 22.3.2 { execsql { PRAGMA main.integrity_check; } } {ok} do_test 22.3.3 { execsql { PRAGMA aux.integrity_check; } -} [list $auxerr] +} $auxerr do_test 22.4.1 { catch { db close } @@ -1630,10 +1635,10 @@ do_test 22.4.1 { ATTACH 'test.db' AS 'aux'; PRAGMA integrity_check; } -} [list $mainerr] +} $mainerr do_test 22.4.2 { execsql { PRAGMA main.integrity_check; } -} [list $mainerr] +} $mainerr do_test 22.4.3 { execsql { PRAGMA aux.integrity_check; } } {ok} From 07901eb8c3ab341c5fe5119069e39ebf91117b39 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Feb 2014 19:37:45 +0000 Subject: [PATCH 021/177] Provide an #ifdef in shell.c to work around issues when cross-compiling from Ubuntu to windows. FossilOrigin-Name: 0a3579d9b9d2a60bb85a9811bc7936edb88debae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6892cecea2..e1c9e01261 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\sway\sof\shandling\sthe\sUSE_SYSTEM_SQLITE\s#define\sin\sshell.c. -D 2014-02-27T15:11:52.919 +C Provide\san\s#ifdef\sin\sshell.c\sto\swork\saround\sissues\swhen\scross-compiling\nfrom\sUbuntu\sto\swindows. +D 2014-02-28T19:37:45.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 -F src/shell.c bf75ce6bea4c8f56c1b46bee201c25acddffb675 +F src/shell.c e6d573326f12901df3801777831b48bd80c11dab F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a6690400235705ecc0d1a60dacff6ad5fb1f944a -R d9e4cc8c2428822c2a58511d3bd2892d +P 51ce713c6ee91bdf0126155334dcc800b3daa509 +R f77dc22964f582c1f952cbee70cf06fb U drh -Z 2ca6d024109a270000c3879d5e05ce0a +Z 5ee6ba31533a91dba4172ad5a5e2ece1 diff --git a/manifest.uuid b/manifest.uuid index 5a0d8578c1..297558a6f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51ce713c6ee91bdf0126155334dcc800b3daa509 \ No newline at end of file +0a3579d9b9d2a60bb85a9811bc7936edb88debae \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 6815c489e6..0709f6c41e 100644 --- a/src/shell.c +++ b/src/shell.c @@ -65,7 +65,9 @@ #if defined(_WIN32) || defined(WIN32) # include #define isatty(h) _isatty(h) -#define access(f,m) _access((f),(m)) +#ifndef access +# define access(f,m) _access((f),(m)) +#endif #undef popen #define popen _popen #undef pclose From efbf3b1a1066b541123e065c9a1469462b1bec27 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Feb 2014 20:47:24 +0000 Subject: [PATCH 022/177] In the command-line shell add the (undocumented and unsupported) ".eqp" command and -eqp command-line option, to cause EXPLAIN QUERY PLAN to be run on each SQL statement as it is evaluated. Intended use is for analysis of the query planner. FossilOrigin-Name: e6ecf7337658624d664e1e71ba3fc527fd6578c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 25 +++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e1c9e01261..3686379a18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\san\s#ifdef\sin\sshell.c\sto\swork\saround\sissues\swhen\scross-compiling\nfrom\sUbuntu\sto\swindows. -D 2014-02-28T19:37:45.291 +C In\sthe\scommand-line\sshell\sadd\sthe\s(undocumented\sand\sunsupported)\s".eqp"\ncommand\sand\s-eqp\scommand-line\soption,\sto\scause\sEXPLAIN\sQUERY\sPLAN\sto\sbe\nrun\son\seach\sSQL\sstatement\sas\sit\sis\sevaluated.\s\sIntended\suse\sis\sfor\sanalysis\nof\sthe\squery\splanner. +D 2014-02-28T20:47:24.221 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 -F src/shell.c e6d573326f12901df3801777831b48bd80c11dab +F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 51ce713c6ee91bdf0126155334dcc800b3daa509 -R f77dc22964f582c1f952cbee70cf06fb +P 0a3579d9b9d2a60bb85a9811bc7936edb88debae +R 195e6e751f71ffa9e9cfd28d6f8e532d U drh -Z 5ee6ba31533a91dba4172ad5a5e2ece1 +Z d48791979092ee78fb00c8a21c663b3b diff --git a/manifest.uuid b/manifest.uuid index 297558a6f1..6108cf5bed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a3579d9b9d2a60bb85a9811bc7936edb88debae \ No newline at end of file +e6ecf7337658624d664e1e71ba3fc527fd6578c1 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 0709f6c41e..7ecedfd3c1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -446,6 +446,7 @@ struct previous_mode_data { struct callback_data { sqlite3 *db; /* The database */ int echoOn; /* True to echo input commands */ + int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL statement */ int statsOn; /* True to display memory stats before each finalize */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -1302,6 +1303,23 @@ static int shell_exec( fprintf(pArg->out, "%s\n", zStmtSql ? zStmtSql : zSql); } + /* Show the EXPLAIN QUERY PLAN if .eqp is on */ + if( pArg && pArg->autoEQP ){ + sqlite3_stmt *pExplain; + char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", sqlite3_sql(pStmt)); + rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); + if( rc==SQLITE_OK ){ + while( sqlite3_step(pExplain)==SQLITE_ROW ){ + fprintf(pArg->out,"--EQP-- %d,", sqlite3_column_int(pExplain, 0)); + fprintf(pArg->out,"%d,", sqlite3_column_int(pExplain, 1)); + fprintf(pArg->out,"%d,", sqlite3_column_int(pExplain, 2)); + fprintf(pArg->out,"%s\n", sqlite3_column_text(pExplain, 3)); + } + } + sqlite3_finalize(pExplain); + sqlite3_free(zEQP); + } + /* Output TESTCTRL_EXPLAIN text of requested */ if( pArg && pArg->mode==MODE_Explain ){ const char *zExplain = 0; @@ -2301,6 +2319,10 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->echoOn = booleanValue(azArg[1]); }else + if( c=='e' && strncmp(azArg[0], "eqp", n)==0 && nArg>1 && nArg<3 ){ + p->autoEQP = booleanValue(azArg[1]); + }else + if( c=='e' && strncmp(azArg[0], "exit", n)==0 ){ if( nArg>1 && (rc = (int)integerValue(azArg[1]))!=0 ) exit(rc); rc = 2; @@ -2873,6 +2895,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='s' && strncmp(azArg[0], "show", n)==0 && nArg==1 ){ int i; fprintf(p->out,"%9.9s: %s\n","echo", p->echoOn ? "on" : "off"); + fprintf(p->out,"%9.9s: %s\n","eqp", p->autoEQP ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","explain", p->explainPrev.valid ? "on" :"off"); fprintf(p->out,"%9.9s: %s\n","headers", p->showHeader ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","mode", modeDescr[p->mode]); @@ -3710,6 +3733,8 @@ int main(int argc, char **argv){ data.showHeader = 0; }else if( strcmp(z,"-echo")==0 ){ data.echoOn = 1; + }else if( strcmp(z,"-eqp")==0 ){ + data.autoEQP = 1; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; }else if( strcmp(z,"-bail")==0 ){ From defbd9dade8a135ebc79c14efafe1a22c0f99c94 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 Mar 2014 14:45:18 +0000 Subject: [PATCH 023/177] Add extra assert() statements trying to catch a Mem object in an inconsistent state. FossilOrigin-Name: 4aeb3ae435c78070232fef21a147fde4e1c5cd31 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 4 ++++ src/vdbeInt.h | 18 ++++++++++++++++++ src/vdbeaux.c | 1 + src/vdbemem.c | 8 ++------ 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 3686379a18..8fbb26ce51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell\sadd\sthe\s(undocumented\sand\sunsupported)\s".eqp"\ncommand\sand\s-eqp\scommand-line\soption,\sto\scause\sEXPLAIN\sQUERY\sPLAN\sto\sbe\nrun\son\seach\sSQL\sstatement\sas\sit\sis\sevaluated.\s\sIntended\suse\sis\sfor\sanalysis\nof\sthe\squery\splanner. -D 2014-02-28T20:47:24.221 +C Add\sextra\sassert()\sstatements\strying\sto\scatch\sa\sMem\sobject\sin\san\sinconsistent\nstate. +D 2014-03-01T14:45:18.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,13 +277,13 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 6c8f28911e702151c8ef03c568db5a066d3a85d4 +F src/vdbe.c e9eda47b8d8410d35840589daf8f3177efa04478 F src/vdbe.h 147027d6e8e667a63e87177a38e2b42c71fdacf8 -F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 +F src/vdbeInt.h fa0cadb90633d260f0c7d21f68a2ec5b1c6900e2 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 8b8eeb3cd89e4b3d4f40186344915b49b7c1c0f7 +F src/vdbeaux.c 087e7c75643157a903c9794294f4ff45cec28e7e F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 06603e8e9d2f3247b68c6bbe4bd37fb6721b5bda +F src/vdbemem.c 7db453561091f0ed8fc7c96ee635cc612334025f F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1152,7 +1152,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0a3579d9b9d2a60bb85a9811bc7936edb88debae -R 195e6e751f71ffa9e9cfd28d6f8e532d +P e6ecf7337658624d664e1e71ba3fc527fd6578c1 +R ac78e315b388dc3aeb5c2c8983219f64 +T *branch * enhanced-mem-check +T *sym-enhanced-mem-check * +T -sym-trunk * U drh -Z d48791979092ee78fb00c8a21c663b3b +Z 099f8256c2a0fcf00ebef2162229fe0b diff --git a/manifest.uuid b/manifest.uuid index 6108cf5bed..aa3e8c1f3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6ecf7337658624d664e1e71ba3fc527fd6578c1 \ No newline at end of file +4aeb3ae435c78070232fef21a147fde4e1c5cd31 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5ab0127658..cca69092a2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -600,18 +600,21 @@ int sqlite3VdbeExec( assert( pOp->p1>0 ); assert( pOp->p1<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); + assert( memSanity1(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); + assert( memSanity1(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); + assert( memSanity1(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ @@ -2460,6 +2463,7 @@ case OP_Column: { */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); + assert( memSanity1(pDest) ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 88c072adec..56ea39ffc3 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -232,6 +232,24 @@ struct Mem { #define memIsValid(M) ((M)->flags & MEM_Undefined)==0 #endif +/* +** A sanity check on a Mem object and especially the Mem.z field. +** This check says that no more than one of the following may be true: +** (1) Mem.z comes from Mem.zMalloc +** (2) Mem.z has a destructor Mem.xDel +** (3) Mem.z is an ephemeral string +** (4) Mem.z is a static string +** +** Use only inside of an assert() as follows: assert( memSanify(pMem) ); +*/ +#ifdef SQLITE_DEBUG +#define memSanity1(p) \ + ((((p)->zMalloc && (p)->zMalloc==(p)->z) ? 1 : 0) + \ + ((((p)->flags&MEM_Dyn)&&(p)->xDel) ? 1 : 0) + \ + (((p)->flags&MEM_Ephem) ? 1 : 0) + \ + (((p)->flags&MEM_Static) ? 1 : 0) <= 1 ) +#endif + /* ** Each auxilliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ef70f8ae3f..c22003f461 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1220,6 +1220,7 @@ static void releaseMemArray(Mem *p, int N){ } for(pEnd=&p[N]; pz is discarded. */ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ - assert( 1 >= - ((pMem->zMalloc && pMem->zMalloc==pMem->z) ? 1 : 0) + - (((pMem->flags&MEM_Dyn)&&pMem->xDel) ? 1 : 0) + - ((pMem->flags&MEM_Ephem) ? 1 : 0) + - ((pMem->flags&MEM_Static) ? 1 : 0) - ); + assert( memSanity1(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -292,6 +287,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** (Mem.memType==MEM_Str). */ void sqlite3VdbeMemRelease(Mem *p){ + assert( memSanity1(p) ); VdbeMemRelease(p); if( p->zMalloc ){ sqlite3DbFree(p->db, p->zMalloc); From 75fd054201c0ba287252bd53b6fd60067fc2c409 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 Mar 2014 16:24:44 +0000 Subject: [PATCH 024/177] Factor the Mem invariant checker into a separate procedure (rather than a macro) so that it can be more easily extended. FossilOrigin-Name: 354699d50e7d251504a7b3d6fbec9e5bcf99557f --- manifest | 21 +++++++++------------ manifest.uuid | 2 +- src/vdbe.c | 8 ++++---- src/vdbeInt.h | 19 +------------------ src/vdbeaux.c | 2 +- src/vdbemem.c | 22 ++++++++++++++++++++-- 6 files changed, 36 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 8fbb26ce51..75bec54679 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\sassert()\sstatements\strying\sto\scatch\sa\sMem\sobject\sin\san\sinconsistent\nstate. -D 2014-03-01T14:45:18.939 +C Factor\sthe\sMem\sinvariant\schecker\sinto\sa\sseparate\sprocedure\s(rather\sthan\sa\nmacro)\sso\sthat\sit\scan\sbe\smore\seasily\sextended. +D 2014-03-01T16:24:44.768 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,13 +277,13 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c e9eda47b8d8410d35840589daf8f3177efa04478 +F src/vdbe.c 9abaa1ebb8b98e33121ea1aade1022fdc195b464 F src/vdbe.h 147027d6e8e667a63e87177a38e2b42c71fdacf8 -F src/vdbeInt.h fa0cadb90633d260f0c7d21f68a2ec5b1c6900e2 +F src/vdbeInt.h a82d5ab4fd8138bdff7de4a267ea5c04e4cbb370 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 087e7c75643157a903c9794294f4ff45cec28e7e +F src/vdbeaux.c 2f97cd756e2704b06006b398ce468f0bcc006055 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 7db453561091f0ed8fc7c96ee635cc612334025f +F src/vdbemem.c 8c84ec056c7934cef359f4d49fa32b856eeffd88 F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1152,10 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6ecf7337658624d664e1e71ba3fc527fd6578c1 -R ac78e315b388dc3aeb5c2c8983219f64 -T *branch * enhanced-mem-check -T *sym-enhanced-mem-check * -T -sym-trunk * +P 4aeb3ae435c78070232fef21a147fde4e1c5cd31 +R c42d4082e634170a16f76850fb72106d U drh -Z 099f8256c2a0fcf00ebef2162229fe0b +Z e1bae2e998058d57d79a76345aaf3403 diff --git a/manifest.uuid b/manifest.uuid index aa3e8c1f3c..7b0fa50ebd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4aeb3ae435c78070232fef21a147fde4e1c5cd31 \ No newline at end of file +354699d50e7d251504a7b3d6fbec9e5bcf99557f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index cca69092a2..c8e837b81b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -600,21 +600,21 @@ int sqlite3VdbeExec( assert( pOp->p1>0 ); assert( pOp->p1<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); - assert( memSanity1(&aMem[pOp->p1]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); assert( pOp->p2<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); - assert( memSanity1(&aMem[pOp->p2]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); assert( pOp->p3<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); - assert( memSanity1(&aMem[pOp->p3]) ); + assert( sqlite3VdbeCheckMemInvariants(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ @@ -2463,7 +2463,7 @@ case OP_Column: { */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); - assert( memSanity1(pDest) ); + assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 56ea39ffc3..047a4f2697 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -232,24 +232,6 @@ struct Mem { #define memIsValid(M) ((M)->flags & MEM_Undefined)==0 #endif -/* -** A sanity check on a Mem object and especially the Mem.z field. -** This check says that no more than one of the following may be true: -** (1) Mem.z comes from Mem.zMalloc -** (2) Mem.z has a destructor Mem.xDel -** (3) Mem.z is an ephemeral string -** (4) Mem.z is a static string -** -** Use only inside of an assert() as follows: assert( memSanify(pMem) ); -*/ -#ifdef SQLITE_DEBUG -#define memSanity1(p) \ - ((((p)->zMalloc && (p)->zMalloc==(p)->z) ? 1 : 0) + \ - ((((p)->flags&MEM_Dyn)&&(p)->xDel) ? 1 : 0) + \ - (((p)->flags&MEM_Ephem) ? 1 : 0) + \ - (((p)->flags&MEM_Static) ? 1 : 0) <= 1 ) -#endif - /* ** Each auxilliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance @@ -474,6 +456,7 @@ int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); #ifdef SQLITE_DEBUG void sqlite3VdbeMemAboutToChange(Vdbe*,Mem*); +int sqlite3VdbeCheckMemInvariants(Mem*); #endif #ifndef SQLITE_OMIT_FOREIGN_KEY diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c22003f461..4a541e0d48 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1220,7 +1220,7 @@ static void releaseMemArray(Mem *p, int N){ } for(pEnd=&p[N]; pzMalloc && (p)->zMalloc==(p)->z) ? 1 : 0) + + ((((p)->flags&MEM_Dyn)&&(p)->xDel) ? 1 : 0) + + (((p)->flags&MEM_Ephem) ? 1 : 0) + + (((p)->flags&MEM_Static) ? 1 : 0) <= 1 ); + return 1; +} +#endif + + /* ** If pMem is an object with a valid string representation, this routine ** ensures the internal encoding for the string representation is @@ -67,7 +85,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** in pMem->z is discarded. */ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ - assert( memSanity1(pMem) ); + assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -287,7 +305,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** (Mem.memType==MEM_Str). */ void sqlite3VdbeMemRelease(Mem *p){ - assert( memSanity1(p) ); + assert( sqlite3VdbeCheckMemInvariants(p) ); VdbeMemRelease(p); if( p->zMalloc ){ sqlite3DbFree(p->db, p->zMalloc); From c91b2fd310e4138b1a36c3f3c418a21a6c2ae895 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 Mar 2014 18:13:23 +0000 Subject: [PATCH 025/177] Change the MEM_Dyn flag so that it means that Mem.xDel exists and must be used to free the string or blob. Add tighter invariant checks on Mem. FossilOrigin-Name: 44e1c33767cae3bf2cbd2238831fe67197009b43 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/utf.c | 3 +-- src/vdbe.c | 23 ++++++++++++----------- src/vdbeInt.h | 2 +- src/vdbeaux.c | 6 +++--- src/vdbemem.c | 43 +++++++++++++++++++++++++++++++------------ 7 files changed, 59 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 75bec54679..5a221fb21d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sthe\sMem\sinvariant\schecker\sinto\sa\sseparate\sprocedure\s(rather\sthan\sa\nmacro)\sso\sthat\sit\scan\sbe\smore\seasily\sextended. -D 2014-03-01T16:24:44.768 +C Change\sthe\sMEM_Dyn\sflag\sso\sthat\sit\smeans\sthat\sMem.xDel\sexists\sand\smust\sbe\nused\sto\sfree\sthe\sstring\sor\sblob.\s\sAdd\stighter\sinvariant\schecks\son\sMem. +D 2014-03-01T18:13:23.744 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -274,16 +274,16 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c a80036fcbd992729adc7cd34a875d59a71fa10cc F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 -F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 +F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 9abaa1ebb8b98e33121ea1aade1022fdc195b464 +F src/vdbe.c cbed4957185bfee7e585dce208d8ec4cd87179c2 F src/vdbe.h 147027d6e8e667a63e87177a38e2b42c71fdacf8 -F src/vdbeInt.h a82d5ab4fd8138bdff7de4a267ea5c04e4cbb370 +F src/vdbeInt.h d55cab859abb2c6656911497ae74eba9dcf34e28 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 2f97cd756e2704b06006b398ce468f0bcc006055 +F src/vdbeaux.c 89a0ad3b2ecdbe71cf827a1f99d0ae51eccd6cce F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 8c84ec056c7934cef359f4d49fa32b856eeffd88 +F src/vdbemem.c 868a498a670c08344a594a9d5903a3ff330916a7 F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4aeb3ae435c78070232fef21a147fde4e1c5cd31 -R c42d4082e634170a16f76850fb72106d +P 354699d50e7d251504a7b3d6fbec9e5bcf99557f +R 0950c7cf1a1151e6071aa7a5599d1f4e U drh -Z e1bae2e998058d57d79a76345aaf3403 +Z c9678592d0817ad1269cd44af9e6bf96 diff --git a/manifest.uuid b/manifest.uuid index 7b0fa50ebd..764bf41b7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -354699d50e7d251504a7b3d6fbec9e5bcf99557f \ No newline at end of file +44e1c33767cae3bf2cbd2238831fe67197009b43 \ No newline at end of file diff --git a/src/utf.c b/src/utf.c index ecb3ea03b8..96b679fff4 100644 --- a/src/utf.c +++ b/src/utf.c @@ -317,7 +317,7 @@ int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ sqlite3VdbeMemRelease(pMem); pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); pMem->enc = desiredEnc; - pMem->flags |= (MEM_Term|MEM_Dyn); + pMem->flags |= (MEM_Term); pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; @@ -445,7 +445,6 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){ } assert( (m.flags & MEM_Term)!=0 || db->mallocFailed ); assert( (m.flags & MEM_Str)!=0 || db->mallocFailed ); - assert( (m.flags & MEM_Dyn)!=0 || db->mallocFailed ); assert( m.z || db->mallocFailed ); return m.z; } diff --git a/src/vdbe.c b/src/vdbe.c index c8e837b81b..8e5a3da00f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -152,7 +152,7 @@ int sqlite3_found_count = 0; ** ** This routine converts an ephemeral string into a dynamically allocated ** string that the register itself controls. In other words, it -** converts an MEM_Ephem string into an MEM_Dyn string. +** converts an MEM_Ephem string into a string with P.z==P.zMalloc. */ #define Deephemeralize(P) \ if( ((P)->flags&MEM_Ephem)!=0 \ @@ -716,7 +716,7 @@ check_for_interrupt: case OP_Gosub: { /* jump */ assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); + assert( VdbeMemDynamic(pIn1)==0 ); memAboutToChange(p, pIn1); pIn1->flags = MEM_Int; pIn1->u.i = pc; @@ -789,7 +789,7 @@ case OP_EndCoroutine: { /* in1 */ case OP_Yield: { /* in1, jump */ int pcDest; pIn1 = &aMem[pOp->p1]; - assert( (pIn1->flags & MEM_Dyn)==0 ); + assert( VdbeMemDynamic(pIn1)==0 ); pIn1->flags = MEM_Int; pcDest = (int)pIn1->u.i; pIn1->u.i = pc; @@ -962,10 +962,9 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ if( rc==SQLITE_TOOBIG ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->zMalloc==pOut->z ); - assert( pOut->flags & MEM_Dyn ); + assert( VdbeMemDynamic(pOut)==0 ); pOut->zMalloc = 0; pOut->flags |= MEM_Static; - pOut->flags &= ~MEM_Dyn; if( pOp->p4type==P4_DYNAMIC ){ sqlite3DbFree(db, pOp->p4.z); } @@ -1101,14 +1100,16 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); + VdbeMemRelease(pOut); zMalloc = pOut->zMalloc; - pOut->zMalloc = 0; - sqlite3VdbeMemMove(pOut, pIn1); + memcpy(pOut, pIn1, sizeof(Mem)); #ifdef SQLITE_DEBUG if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ pOut->pScopyFrom += p1 - pOp->p2; } #endif + pIn1->flags = MEM_Undefined; + pIn1->xDel = 0; pIn1->zMalloc = zMalloc; REGISTER_TRACE(p2++, pOut); pIn1++; @@ -1285,10 +1286,10 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } - MemSetTypeFlag(pOut, MEM_Str); if( sqlite3VdbeMemGrow(pOut, (int)nByte+2, pOut==pIn2) ){ goto no_mem; } + MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); } @@ -2501,8 +2502,8 @@ case OP_Column: { ** This prevents a memory copy. */ if( sMem.zMalloc ){ assert( sMem.z==sMem.zMalloc ); - assert( !(pDest->flags & MEM_Dyn) ); - assert( !(pDest->flags & (MEM_Blob|MEM_Str)) || pDest->z==sMem.z ); + assert( VdbeMemDynamic(pDest)==0 ); + assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); pDest->flags &= ~(MEM_Ephem|MEM_Static); pDest->flags |= MEM_Term; pDest->z = sMem.z; @@ -2685,7 +2686,7 @@ case OP_MakeRecord: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; - pOut->flags = MEM_Blob | MEM_Dyn; + pOut->flags = MEM_Blob; pOut->xDel = 0; if( nZero ){ pOut->u.nZero = nZero; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 047a4f2697..c16baff685 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -208,7 +208,7 @@ struct Mem { ** string is \000 or \u0000 terminated */ #define MEM_Term 0x0200 /* String rep is nul terminated */ -#define MEM_Dyn 0x0400 /* Need to call sqliteFree() on Mem.z */ +#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ #define MEM_Static 0x0800 /* Mem.z points to a static string */ #define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ #define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 4a541e0d48..39be964093 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1414,7 +1414,7 @@ int sqlite3VdbeList( assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; zP4 = displayP4(pOp, pMem->z, 32); if( zP4!=pMem->z ){ sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); @@ -1431,7 +1431,7 @@ int sqlite3VdbeList( assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; pMem->n = 2; sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ pMem->memType = MEM_Str; @@ -1443,7 +1443,7 @@ int sqlite3VdbeList( assert( p->db->mallocFailed ); return SQLITE_ERROR; } - pMem->flags = MEM_Dyn|MEM_Str|MEM_Term; + pMem->flags = MEM_Str|MEM_Term; pMem->n = displayComment(pOp, zP4, pMem->z, 500); pMem->memType = MEM_Str; pMem->enc = SQLITE_UTF8; diff --git a/src/vdbemem.c b/src/vdbemem.c index 532cd58d9e..3c9412a700 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -26,11 +26,29 @@ ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ int sqlite3VdbeCheckMemInvariants(Mem *p){ - assert( - (((p)->zMalloc && (p)->zMalloc==(p)->z) ? 1 : 0) + - ((((p)->flags&MEM_Dyn)&&(p)->xDel) ? 1 : 0) + - (((p)->flags&MEM_Ephem) ? 1 : 0) + - (((p)->flags&MEM_Static) ? 1 : 0) <= 1 ); + /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor + ** function for Mem.z + */ + assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); + + /* If p holds a string or blob, the Mem.z must point to exactly + ** one of the following: + ** + ** (1) Memory in Mem.zMalloc and managed by the Mem object + ** (2) Memory to be freed using Mem.xDel + ** (3) An ephermal string or blob + ** (4) A static string or blob + */ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + assert( + ((p->z==p->zMalloc)? 1 : 0) + + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 + ); + } + return 1; } #endif @@ -103,7 +121,7 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - sqlite3VdbeMemRelease(pMem); + VdbeMemRelease(pMem); pMem->flags = MEM_Null; return SQLITE_NOMEM; } @@ -112,13 +130,13 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } - if( (pMem->flags&MEM_Dyn)!=0 && pMem->xDel ){ - assert( pMem->xDel!=SQLITE_DYNAMIC ); + if( (pMem->flags&MEM_Dyn)!=0 ){ + assert( pMem->xDel!=0 && pMem->xDel!=SQLITE_DYNAMIC ); pMem->xDel((void *)(pMem->z)); } pMem->z = pMem->zMalloc; - pMem->flags &= ~(MEM_Ephem|MEM_Static); + pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); pMem->xDel = 0; return SQLITE_OK; } @@ -287,9 +305,9 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ sqlite3VdbeMemFinalize(p, p->u.pDef); assert( (p->flags & MEM_Agg)==0 ); sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn && p->xDel ){ + }else if( p->flags&MEM_Dyn ){ assert( (p->flags&MEM_RowSet)==0 ); - assert( p->xDel!=SQLITE_DYNAMIC ); + assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); p->xDel = 0; }else if( p->flags&MEM_RowSet ){ @@ -643,6 +661,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ VdbeMemRelease(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; + pTo->xDel = 0; if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ @@ -922,7 +941,7 @@ int sqlite3VdbeMemFromBtree( pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - pMem->flags = MEM_Blob|MEM_Dyn|MEM_Term; + pMem->flags = MEM_Blob|MEM_Term; pMem->enc = 0; pMem->memType = MEM_Blob; if( key ){ From 3833e934d3b219dfb05d9a38333d1c8d7bfe4f82 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 Mar 2014 19:44:56 +0000 Subject: [PATCH 026/177] Remove the vdbeRecordCompareLargeHeader function. Fix some other details. FossilOrigin-Name: 3861e853105cb8da344c7eebd2e455622b26395e --- manifest | 22 ++--- manifest.uuid | 2 +- src/btree.c | 6 +- src/sqliteInt.h | 7 +- src/vdbe.h | 5 +- src/vdbeaux.c | 226 ++++++++++++++++++++++++------------------------ src/vdbesort.c | 7 +- src/where.c | 8 +- 8 files changed, 142 insertions(+), 141 deletions(-) diff --git a/manifest b/manifest index f20f7a0353..08b453f312 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\stest\scases\sthat\sdeal\swith\scorrupt\sdatabases. -D 2014-02-28T18:39:51.462 +C Remove\sthe\svdbeRecordCompareLargeHeader\sfunction.\sFix\ssome\sother\sdetails. +D 2014-03-01T19:44:56.574 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 77f175987c80ebec063f8653cb7d300776411413 +F src/btree.c d288e668614449571ec269535dc4aaf216a23db2 F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 @@ -221,7 +221,7 @@ F src/shell.c bf75ce6bea4c8f56c1b46bee201c25acddffb675 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 843e23d43ee2ab9369e68eb778815ba02835da59 +F src/sqliteInt.h b06500d391d4b7bf4c69fc110e37dd45719b760c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -278,19 +278,19 @@ F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c ab910206dd8c9c5c1455f82953934bdbfe0bcc2a -F src/vdbe.h 6833579fc0fbdc1c933e34519064841abda5b9b3 +F src/vdbe.h c6dc01f85cf3bdcc992d244aeff952c033fd5ad0 F src/vdbeInt.h 5286af9067cabdb8ba57b87c0c988a931be6c6c8 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c aad5345869c110f45074e2c1ca230e623b1ba182 +F src/vdbeaux.c da0979ce81ecdf3d018be9ee1e8998110181fb12 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 93fc3f13ebe6809ebaac8d0a17c812ec053ba233 -F src/vdbesort.c 72290f12428973c2c6b9d4f95ad0a7c8181e1280 +F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 6042e1a377cf7dc72c10493269ed75e276275cd8 +F src/where.c 36ef94b653a10944b39e34938ee4c758f3f42879 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 284bde0ee20261737446eb8f5b6b36ad9bc3f355 -R 75b55639e7bdd048d8263d0acfa3cf8a +P 3a09f5605ac7c6e503eb10acfdc607010414d917 +R b23b5e438c1d698174ad52d7cfba370e U dan -Z d116a418c5c97c83b8ebfe7a3e655c3c +Z 35d9cecacf0cc917a2c2ed9a75b36eab diff --git a/manifest.uuid b/manifest.uuid index 9cb218f1b4..54a81caa10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a09f5605ac7c6e503eb10acfdc607010414d917 \ No newline at end of file +3861e853105cb8da344c7eebd2e455622b26395e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c0b04cc08c..4d42969be8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4661,14 +4661,14 @@ int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pCell[1], 1, pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pCell[2], 1, pIdxKey); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -4689,7 +4689,7 @@ int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, ((u8*)pCellKey)[0], 1, pIdxKey); + c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } if( c<0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index eece50f7a1..a368b96c63 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1586,14 +1586,17 @@ struct KeyInfo { ** ** This structure holds a record that has already been disassembled ** into its constituent fields. +** +** The r1 and r2 member variables are only used by the optimized comparison +** functions vdbeRecordCompareInt() and vdbeRecordCompareString(). */ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ char default_rc; /* Comparison result if keys are equal */ Mem *aMem; /* Values */ - int r1; - int r2; + int r1; /* Value to return if (lhs > rhs) */ + int r2; /* Value to return if (rhs < lhs) */ }; diff --git a/src/vdbe.h b/src/vdbe.h index c08512e7aa..389c1d5ffb 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -211,12 +211,11 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,int,u32,UnpackedRecord*); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); -RecordCompare sqlite3VdbeFindSorterCompare(KeyInfo*); #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 007f4e650d..f15f5e95be 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3123,24 +3123,16 @@ void sqlite3VdbeRecordUnpack( p->nField = u; } +#if SQLITE_DEBUG /* -** This function compares the two table rows or index records -** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero -** or positive integer if key1 is less than, equal to or -** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 -** key must be a parsed key such as obtained from -** sqlite3VdbeParseRecord. -** -** Key1 and Key2 do not have to contain the same number of fields. -** The key with fewer fields is usually compares less than the -** longer key. However if the UNPACKED_INCRKEY flags in pPKey2 is set -** and the common prefixes are equal, then key1 is less than key2. -** Or if the UNPACKED_MATCH_PREFIX flag is set and the prefixes are -** equal, then the keys are considered to be equal and -** the parts beyond the common prefix are ignored. +** This function compares two index or table record keys in the same way +** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), +** this function deserializes and compares values using the +** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used +** in assert() statements to ensure that the optimized code in +** sqlite3VdbeRecordCompare() returns results with these two primitives. */ -static int vdbeRecordComparePrev( +static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2 /* Right key */ ){ @@ -3220,10 +3212,17 @@ static int vdbeRecordComparePrev( ** value. */ return pPKey2->default_rc; } +#endif +/* +** Both *pMem1 and *pMem2 contain string values. Compare the two values +** using the collation sequence pColl. As usual, return a negative , zero +** or positive value if *pMem1 is less than, equal to or greater than +** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". +*/ static int vdbeCompareMemString( - const Mem *pMem1, - const Mem *pMem2, + const Mem *pMem1, + const Mem *pMem2, const CollSeq *pColl ){ if( pMem1->enc==pColl->enc ){ @@ -3344,8 +3343,17 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ } +/* +** The first argument passed to this function is a serial-type that +** corresponds to an integer - all values between 1 and 9 inclusive +** except 7. The second points to a buffer containing an integer value +** serialized according to serial_type. This function deserializes +** and returns the value. +*/ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + assert( CORRUPT_DB || (serial_type>=1 && serial_type<=9 && serial_type!=7) ); switch( serial_type ){ + case 0: case 1: return (char)aKey[0]; case 2: @@ -3354,13 +3362,11 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ return ((char)aKey[0] << 16) | (aKey[1] << 8) | aKey[2]; case 4: return ((char)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]; - case 5: { i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; u32 lsw = (aKey[4] << 8) | aKey[5]; return (i64)( msw << 16 | (u64)lsw ); } - case 6: { i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; @@ -3371,36 +3377,51 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ return (serial_type - 8); } -static int vdbeRecordCompare( - int nKey1, const void *pKey1, /* Left key */ - int szHdr1, /* Size of record header in bytes */ - u32 idx1, /* Offset of first type in header */ - UnpackedRecord *const pPKey2 /* Right key */ +/* +** This function compares the two table rows or index records +** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero +** or positive integer if key1 is less than, equal to or +** greater than key2. The {nKey1, pKey1} key must be a blob +** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** key must be a parsed key such as obtained from +** sqlite3VdbeParseRecord. +** +** If argument bSkip is non-zero, it is assumed that the caller has already +** determined that the first fields of the keys are equal. +** +** Key1 and Key2 do not have to contain the same number of fields. If all +** fields that appear in both keys are equal, then pPKey2->default_rc is +** returned. +*/ +int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *const pPKey2, /* Right key */ + int bSkip /* If true, skip the first field */ ){ - u32 d1 = szHdr1; /* Offset into aKey[] of next data element */ - int i = 0; - int rc = 0; - Mem *pRhs = pPKey2->aMem; + u32 d1; /* Offset into aKey[] of next data element */ + int i; /* Index of next field to compare */ + int szHdr1; /* Size of record header in bytes */ + u32 idx1; /* Offset of first type in header */ + int rc = 0; /* Return value */ + Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ KeyInfo *pKeyInfo = pPKey2->pKeyInfo; const unsigned char *aKey1 = (const unsigned char *)pKey1; Mem mem1; -#ifdef SQLITE_DEBUG - int expected = vdbeRecordComparePrev(nKey1, pKey1, pPKey2); - static int nCall = 0; - nCall++; -#endif - - /* If idx==0, then the caller has already determined that the first two - ** elements in the keys are equal. Fix the various stack variables so + /* If bSkip is true, then the caller has already determined that the first + ** two elements in the keys are equal. Fix the various stack variables so ** that this routine begins comparing at the second field. */ - if( idx1==0 ){ + if( bSkip ){ u32 s1; - assert( sqlite3VarintLen(szHdr1)==1 ); idx1 = 1 + getVarint32(&aKey1[1], s1); - d1 += sqlite3VdbeSerialTypeLen(s1); + szHdr1 = aKey1[0]; + d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); i = 1; pRhs++; + }else{ + idx1 = getVarint32(aKey1, szHdr1); + d1 = szHdr1; + i = 0; } VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ @@ -3511,18 +3532,14 @@ static int vdbeRecordCompare( } if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ if( pKeyInfo->aSortOrder[i] ){ rc = -rc; -#if 0 - assert( (rc>0 && (rc^(int)0x80000000)<0) - || (rc<0 && (rc^(int)0x80000000)>0) ); - assert( sizeof(int)==4 ); - rc ^= (int)0x80000000; /* similar in spirit to: "rc = -rc;" */ - assert( rc!=0 ); -#endif } - assert( (rc<0 && expected<0) || (rc>0 && expected>0) || CORRUPT_DB ); + assert( CORRUPT_DB + || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + ); + assert( mem1.zMalloc==0 ); /* See comment below */ return rc; } @@ -3534,31 +3551,38 @@ static int vdbeRecordCompare( /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a - ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). - */ + ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ assert( mem1.zMalloc==0 ); /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( pPKey2->default_rc==expected ); + assert( CORRUPT_DB + || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + ); return pPKey2->default_rc; } +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is an integer, and (b) the +** size-of-header varint at the start of (pKey1/nKey1) fits in a single +** byte (i.e. is less than 128). +*/ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - int szHdr, - u32 idx1, - UnpackedRecord *pPKey2 /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip /* Ignored */ ){ - const u8 *aKey = &((const u8*)pKey1)[szHdr]; + const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1]; int serial_type = ((const u8*)pKey1)[1]; int res; i64 v = pPKey2->aMem[0].u.i; i64 lhs; - switch( serial_type ){ + assert( bSkip==0 ); + switch( serial_type ){ case 1: lhs = (char)(aKey[0]); break; @@ -3571,25 +3595,21 @@ static int vdbeRecordCompareInt( case 4: lhs = (int)(((u32)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]); break; - case 5: { i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; u32 lsw = (aKey[4] << 8) | aKey[5]; lhs = (i64)( msw << 16 | (u64)lsw ); break; } - case 6: { i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; lhs = (i64)( msw << 32 | (u64)lsw ); break; } - case 8: lhs = 0; break; - case 9: lhs = 1; break; @@ -3601,10 +3621,10 @@ static int vdbeRecordCompareInt( ** (as gcc is clever enough to combine the two like cases). Other ** compilers might be similar. */ case 0: case 7: - return vdbeRecordCompare(nKey1, pKey1, szHdr, 1, pPKey2); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); default: - return vdbeRecordCompare(nKey1, pKey1, szHdr, 1, pPKey2); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); } if( v>lhs ){ @@ -3614,31 +3634,37 @@ static int vdbeRecordCompareInt( }else if( pPKey2->nField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = vdbeRecordCompare(nKey1, pKey1, szHdr, 0, pPKey2); + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordComparePrev(nKey1, pKey1, pPKey2)>0) + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || CORRUPT_DB ); return res; } +/* +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is a string, that (b) the first field +** uses the collation sequence BINARY and (c) that the size-of-header varint +** at the start of (pKey1/nKey1) fits in a single byte. +*/ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - int szHdr, - u32 idx1, - UnpackedRecord *pPKey2 /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ + int bSkip ){ const u8 *aKey1 = (const u8*)pKey1; int serial_type; int res; + assert( bSkip==0 ); getVarint32(&aKey1[1], serial_type); if( serial_type<12 ){ @@ -3648,18 +3674,18 @@ static int vdbeRecordCompareString( }else{ int nCmp; int nStr; - aKey1 = &aKey1[szHdr]; + int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ nCmp = MIN( pPKey2->aMem[0].n, nStr ); - res = memcmp(aKey1, pPKey2->aMem[0].z, nCmp); + res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); if( res==0 ){ res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = vdbeRecordCompare(nKey1, pKey1, szHdr, 0, pPKey2); + res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; } @@ -3675,30 +3701,26 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)==0) - || (res<0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)<0) - || (res>0 && sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2)>0) + assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) + || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || CORRUPT_DB ); return res; } - -int vdbeRecordCompareLargeHeader( - int nKey1, const void *pKey1, /* Left key */ - int dummy1, u32 dummy2, /* Unused arguments */ - UnpackedRecord *pPKey2 /* Right key */ -){ - int szHdr; - u32 idx1; - idx1 = getVarint32(((u8*)pKey1), szHdr); - return vdbeRecordCompare(nKey1, pKey1, szHdr, idx1, pPKey2); -} - +/* +** Return a pointer to an sqlite3VdbeRecordCompare() compatible function +** suitable for comparing serialized records to the unpacked record passed +** as the only argument. +*/ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ - if( (p->pKeyInfo->nField + p->pKeyInfo->nXField) > 10 ){ - return vdbeRecordCompareLargeHeader; - }else{ + /* As the varints that make up a record header are all 5 bytes in size + ** or less, if the binary keys being compared have 25 or fewer fields + ** then it is guaranteed that the varint at the start of every record + ** (the record-header size in bytes) fits in a single byte. If this + ** is not the case, then sqlite3VdbeRecordCompare() must be used. */ + if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=25 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortOrder[0] ){ p->r1 = 1; @@ -3717,25 +3739,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ } } - return vdbeRecordCompare; -} - -RecordCompare sqlite3VdbeFindSorterCompare(KeyInfo *pKeyInfo){ - if( (pKeyInfo->nField + pKeyInfo->nXField) > 10 ){ - return vdbeRecordCompareLargeHeader; - } - return vdbeRecordCompare; -} - -int sqlite3VdbeRecordCompare( - int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ -){ - int szHdr; - u32 idx1; - - idx1 = getVarint32(((u8*)pKey1), szHdr); - return vdbeRecordCompare(nKey1, pKey1, szHdr, idx1, pPKey2); + return sqlite3VdbeRecordCompare; } /* @@ -3850,7 +3854,7 @@ int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } diff --git a/src/vdbesort.c b/src/vdbesort.c index be5a6064c2..b9ed97e8b3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,7 +105,6 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ - RecordCompare xRecordCompare; /* Record compare function */ }; /* @@ -413,10 +412,7 @@ static void vdbeSorterCompare( assert( r2->default_rc==0 ); } -#if 0 - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); -#endif - *pRes = pSorter->xRecordCompare(nKey1, pKey1, *((u8*)pKey1), 1, r2); + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } /* @@ -492,7 +488,6 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ if( mxCachemxPmaSize = mxCache * pgsz; } - pSorter->xRecordCompare = sqlite3VdbeFindSorterCompare(pCsr->pKeyInfo); return SQLITE_OK; } diff --git a/src/where.c b/src/where.c index a5dd7b59e7..4ce0cb7ded 100644 --- a/src/where.c +++ b/src/where.c @@ -1913,7 +1913,7 @@ static void whereKeyStats( assert( pRec->nField>0 && iColnSampleCol ); do{ iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); if( res<0 ){ iMin = iTest+1; }else{ @@ -1928,16 +1928,16 @@ static void whereKeyStats( if( res==0 ){ /* If (res==0) is true, then sample $i must be equal to pRec */ assert( inSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) || pParse->db->mallocFailed ); }else{ /* Otherwise, pRec must be smaller than sample $i and larger than ** sample ($i-1). */ assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 || pParse->db->mallocFailed ); assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 || pParse->db->mallocFailed ); } #endif /* ifdef SQLITE_DEBUG */ From fbf0f0e1f636718d7c6541b5e2d88a4710bc98fc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Mar 2014 14:20:30 +0000 Subject: [PATCH 027/177] Fix a segfault that can occur following an OOM error. FossilOrigin-Name: 7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e3e4bab9d6..86c4f75cc8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sMEM_Dyn\sflag\sso\sthat\sit\smeans\sthat\sMem.xDel\sexists\sand\smust\s\nbe\sused\sto\sfree\sthe\sstring\sor\sblob.\sAdd\stighter\sinvariant\schecks\son\sMem. -D 2014-03-03T00:12:39.825 +C Fix\sa\ssegfault\sthat\scan\soccur\sfollowing\san\sOOM\serror. +D 2014-03-03T14:20:30.542 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 6042e1a377cf7dc72c10493269ed75e276275cd8 +F src/where.c e4b57d81d2a5fece518c1419f047f0803483f90c F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1152,8 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6ecf7337658624d664e1e71ba3fc527fd6578c1 44e1c33767cae3bf2cbd2238831fe67197009b43 -R 0950c7cf1a1151e6071aa7a5599d1f4e -T +closed 44e1c33767cae3bf2cbd2238831fe67197009b43 -U drh -Z 4d8fd5c0a86772bcec65a15ca628db9f +P e3f6c61e0333d4d237dcd2ba2e404f0a91ee5069 +R eaec45cec313455580b422d7721ccfeb +U dan +Z 0f4ad0e1bfa9fac79dc39c1726a86e9b diff --git a/manifest.uuid b/manifest.uuid index eadf23b042..e37ef738a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3f6c61e0333d4d237dcd2ba2e404f0a91ee5069 \ No newline at end of file +7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a5dd7b59e7..f13d2b5044 100644 --- a/src/where.c +++ b/src/where.c @@ -5870,7 +5870,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ ** the co-routine into OP_SCopy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->viaCoroutine ){ + if( pTabItem->viaCoroutine && !db->mallocFailed ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; pOp = sqlite3VdbeGetOp(v, k); From fcb44a89f226010a40265101128cc13c4fae5dfa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Mar 2014 15:13:27 +0000 Subject: [PATCH 028/177] Fix a harmless compiler warning. FossilOrigin-Name: d7f6837e6997a4d5c0fd2d08b52887d691dd6f97 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2b023bb25c..956052609f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2014-03-03T14:45:06.920 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-03-03T15:13:27.429 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h c6dc01f85cf3bdcc992d244aeff952c033fd5ad0 F src/vdbeInt.h d55cab859abb2c6656911497ae74eba9dcf34e28 F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c f0532f7ce4fc3f4d9811d0bc29c6ce56d448596e +F src/vdbeaux.c e4a0f89d15097debae8117ba17e4a640a0d3116f F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9c1747b5ded995e77fdab2fa9d334ec3c17c7c14 7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 -R 4cede51faff0bb760b6f55494711c674 -U dan -Z 2c68a923f498f2b3622ae1d55f13bc1f +P e00ed717fbe456010ec6c746490d7c3e9d94ec5a +R 8e3a8f60ae350e66db9c9ecb1f09657d +U drh +Z 6dbc33a5cff12ba0585c075afb66f191 diff --git a/manifest.uuid b/manifest.uuid index b161fe627f..206d58eaa6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e00ed717fbe456010ec6c746490d7c3e9d94ec5a \ No newline at end of file +d7f6837e6997a4d5c0fd2d08b52887d691dd6f97 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d13049a1a9..10623ecbc0 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3499,7 +3499,7 @@ int sqlite3VdbeRecordCompare( mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; - mem1.z = &aKey1[d1]; + mem1.z = (char*)&aKey1[d1]; rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); }else{ int nCmp = MIN(mem1.n, pRhs->n); From dea63f24e183ef65702bfb86dccc82fd7cb95d7b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Mar 2014 16:48:47 +0000 Subject: [PATCH 029/177] Change the "explain_i" tcl test command so that xterm color codes are only added if the output is actually a terminal. FossilOrigin-Name: 559835e54e0715f2b4d207d056199a8268af739e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/tester.tcl | 15 +++++++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 86c4f75cc8..edfa442779 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sthat\scan\soccur\sfollowing\san\sOOM\serror. -D 2014-03-03T14:20:30.542 +C Change\sthe\s"explain_i"\stcl\stest\scommand\sso\sthat\sxterm\scolor\scodes\sare\sonly\sadded\sif\sthe\soutput\sis\sactually\sa\sterminal. +D 2014-03-03T16:48:47.992 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -841,7 +841,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl 9bd04481b8b0ef1f2049ad01f28e175ee9a14f7b +F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1152,7 +1152,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e3f6c61e0333d4d237dcd2ba2e404f0a91ee5069 -R eaec45cec313455580b422d7721ccfeb +P 7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 +R d76d9cf70a7a6abc4c8a0d1e487a1b2f U dan -Z 0f4ad0e1bfa9fac79dc39c1726a86e9b +Z 3cca059248ddd2dc717b9671dde6b1cc diff --git a/manifest.uuid b/manifest.uuid index e37ef738a9..0198683302 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 \ No newline at end of file +559835e54e0715f2b4d207d056199a8268af739e \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index fefd5e8a8f..1c4e93937c 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1062,10 +1062,17 @@ proc explain_i {sql {db db}} { # Blue: Opcodes that reposition or seek a cursor. # Green: The ResultRow opcode. # - set R "\033\[31;1m" ;# Red fg - set G "\033\[32;1m" ;# Green fg - set B "\033\[34;1m" ;# Red fg - set D "\033\[39;0m" ;# Default fg + if { [catch {fconfigure stdout -mode}]==0 } { + set R "\033\[31;1m" ;# Red fg + set G "\033\[32;1m" ;# Green fg + set B "\033\[34;1m" ;# Red fg + set D "\033\[39;0m" ;# Default fg + } else { + set R "" + set G "" + set B "" + set D "" + } foreach opcode { Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind NoConflict Next Prev VNext VPrev VFilter From 6295524e3ddfbcb77bd7a4eed20702c5443464d3 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Mar 2014 17:36:39 +0000 Subject: [PATCH 030/177] Change an OP_SCopy into an OP_Copy in a case where the destination might be used after the source has changed. FossilOrigin-Name: c0fa0c0e2de50d7eda19ab8862496b18eff93538 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/select.c | 2 +- test/selectF.test | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 9 deletions(-) create mode 100644 test/selectF.test diff --git a/manifest b/manifest index edfa442779..4cca5ef21a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s"explain_i"\stcl\stest\scommand\sso\sthat\sxterm\scolor\scodes\sare\sonly\sadded\sif\sthe\soutput\sis\sactually\sa\sterminal. -D 2014-03-03T16:48:47.992 +C Change\san\sOP_SCopy\sinto\san\sOP_Copy\sin\sa\scase\swhere\sthe\sdestination\smight\sbe\nused\safter\sthe\ssource\shas\schanged. +D 2014-03-03T17:36:39.940 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 35b07a47fdee9a98e03c4ffb3801026c72114cb7 +F src/select.c 780bbf39e401091845ba745a38326eabe5d44293 F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -788,6 +788,7 @@ F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 F test/selectE.test fc02a1eb04c8eb537091482644b7d778ae8759b7 +F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 @@ -1152,7 +1153,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7fdd378d54754a3ffdc01c6c0a66cf6d5899a495 -R d76d9cf70a7a6abc4c8a0d1e487a1b2f -U dan -Z 3cca059248ddd2dc717b9671dde6b1cc +P 559835e54e0715f2b4d207d056199a8268af739e +R 0f407d04c62b2be5f5455aa20c5e914d +U drh +Z c144dcb8dd86ca2b393613351986e190 diff --git a/manifest.uuid b/manifest.uuid index 0198683302..097e35067a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -559835e54e0715f2b4d207d056199a8268af739e \ No newline at end of file +c0fa0c0e2de50d7eda19ab8862496b18eff93538 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a8cb341c65..9430888c4a 100644 --- a/src/select.c +++ b/src/select.c @@ -609,7 +609,7 @@ static void selectInnerLoop( ** values returned by the SELECT are not required. */ sqlite3ExprCodeExprList(pParse, pEList, regResult, - (eDest==SRT_Output)?SQLITE_ECEL_DUP:0); + (eDest==SRT_Output||eDest==SRT_Coroutine)?SQLITE_ECEL_DUP:0); } /* If the DISTINCT keyword was present on the SELECT statement diff --git a/test/selectF.test b/test/selectF.test new file mode 100644 index 0000000000..3fb226e012 --- /dev/null +++ b/test/selectF.test @@ -0,0 +1,49 @@ +# 2014-03-03 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file verifies that an OP_Copy operation is used instead of OP_SCopy +# in a compound select in a case where the source register might be changed +# before the copy is used. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix selectF + +do_execsql_test 1 { + BEGIN TRANSACTION; + CREATE TABLE t1(a, b, c); + INSERT INTO "t1" VALUES(1,'one','I'); + CREATE TABLE t2(d, e, f); + INSERT INTO "t2" VALUES(5,'ten','XX'); + INSERT INTO "t2" VALUES(6,NULL,NULL); + + CREATE INDEX i1 ON t1(b, a); + COMMIT; +} + +#explain_i { +# SELECT * FROM t2 +# UNION ALL +# SELECT * FROM t1 WHERE a<5 +# ORDER BY 2, 1 +#} + +do_execsql_test 2 { + SELECT * FROM t2 + UNION ALL + SELECT * FROM t1 WHERE a<5 + ORDER BY 2, 1 +} {6 {} {} 1 one I 5 ten XX} + + + +finish_test From 295aedf097e947c5975cc1a79e16f779e7992f31 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Mar 2014 18:25:24 +0000 Subject: [PATCH 031/177] Fix compiler warnings. FossilOrigin-Name: ba8993727e5752cbc92c7d78437e9e0510c899f7 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.h | 4 ++-- src/vdbeInt.h | 2 +- src/vdbeaux.c | 24 +++++++++++++----------- 5 files changed, 26 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 15339f6098..d6d61e95b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2014-03-03T17:48:03.279 +C Fix\scompiler\swarnings. +D 2014-03-03T18:25:24.475 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,10 +278,10 @@ F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 -F src/vdbe.h c6dc01f85cf3bdcc992d244aeff952c033fd5ad0 -F src/vdbeInt.h d55cab859abb2c6656911497ae74eba9dcf34e28 +F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 +F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c e4a0f89d15097debae8117ba17e4a640a0d3116f +F src/vdbeaux.c f6d49b824fb9b4025098d73ba63a579bceab46c1 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1153,7 +1153,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d7f6837e6997a4d5c0fd2d08b52887d691dd6f97 c0fa0c0e2de50d7eda19ab8862496b18eff93538 -R 9a9d986bbf5df1b40c16a7d9f71c4a7c -U dan -Z 5c6a6f0f42fa42f03fa0c54709a35c4c +P 1d60356462f111ed147aa865dd17a13511db6ab7 +R 21f5b7d1de45908a8d9a08e2efe5ba51 +U drh +Z bf94921d4ecf2467a56de90bb29e8bf1 diff --git a/manifest.uuid b/manifest.uuid index a61bcd33e6..19e8b9d202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d60356462f111ed147aa865dd17a13511db6ab7 \ No newline at end of file +ba8993727e5752cbc92c7d78437e9e0510c899f7 \ No newline at end of file diff --git a/src/vdbe.h b/src/vdbe.h index 389c1d5ffb..66ca602ab7 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -211,10 +211,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,const UnpackedRecord*,int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,const UnpackedRecord*,int); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/vdbeInt.h b/src/vdbeInt.h index c16baff685..33c06142f6 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -391,7 +391,7 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); +int sqlite3VdbeIdxKeyCompare(VdbeCursor*,const UnpackedRecord*,int*); int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3VdbeExec(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 10623ecbc0..0a2f11ab2b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3135,7 +3135,7 @@ void sqlite3VdbeRecordUnpack( */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2 /* Right key */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -3396,7 +3396,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *const pPKey2, /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ @@ -3474,7 +3474,7 @@ int sqlite3VdbeRecordCompare( if( serial_type==7 ){ lhs = mem1.r; }else{ - lhs = mem1.u.i; + lhs = (double)mem1.u.i; } if( lhs nKey1 ){ + if( (d1+mem1.n) > (unsigned)nKey1 ){ rc = 1; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; @@ -3516,7 +3516,7 @@ int sqlite3VdbeRecordCompare( rc = -1; }else{ int nStr = (serial_type - 12) / 2; - if( (d1+nStr) > nKey1 ){ + if( (d1+nStr) > (unsigned)nKey1 ){ rc = 1; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); @@ -3548,7 +3548,7 @@ int sqlite3VdbeRecordCompare( pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); idx1 += sqlite3VarintLen(serial_type); - }while( idx1nField && d1<=nKey1 ); + }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -3572,7 +3572,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1]; @@ -3580,6 +3580,7 @@ static int vdbeRecordCompareInt( int res; i64 v = pPKey2->aMem[0].u.i; i64 lhs; + UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); @@ -3658,12 +3659,13 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ int bSkip ){ const u8 *aKey1 = (const u8*)pKey1; int serial_type; int res; + UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); getVarint32(&aKey1[1], serial_type); @@ -3832,9 +3834,9 @@ idx_rowid_corruption: ** of the keys prior to the final rowid, not the entire key. */ int sqlite3VdbeIdxKeyCompare( - VdbeCursor *pC, /* The cursor to compare against */ - UnpackedRecord *pUnpacked, /* Unpacked version of key to compare against */ - int *res /* Write the comparison result here */ + VdbeCursor *pC, /* The cursor to compare against */ + const UnpackedRecord *pUnpacked, /* Unpacked version of key */ + int *res /* Write the comparison result here */ ){ i64 nCellKey = 0; int rc; From 5668969a69c34859d6c670218c9cebb52f6d7f55 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Mar 2014 19:29:28 +0000 Subject: [PATCH 032/177] Fix a typo in a comment. No changes to code. FossilOrigin-Name: 1f4991ab1676bf85599fc32068ceeddeb05f648c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d6d61e95b6..4302dac32e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings. -D 2014-03-03T18:25:24.475 +C Fix\sa\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2014-03-03T19:29:28.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c f6d49b824fb9b4025098d73ba63a579bceab46c1 +F src/vdbeaux.c b817480055ea37f04cbda6524cbcda56d8620c78 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1153,7 +1153,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1d60356462f111ed147aa865dd17a13511db6ab7 -R 21f5b7d1de45908a8d9a08e2efe5ba51 +P ba8993727e5752cbc92c7d78437e9e0510c899f7 +R 84a57b7d3530d77238186688acf2fb15 U drh -Z bf94921d4ecf2467a56de90bb29e8bf1 +Z 743719e98998c2037f212aec24c916c8 diff --git a/manifest.uuid b/manifest.uuid index 19e8b9d202..2f5a48fd68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba8993727e5752cbc92c7d78437e9e0510c899f7 \ No newline at end of file +1f4991ab1676bf85599fc32068ceeddeb05f648c \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a2f11ab2b..47a3f0899b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3846,7 +3846,7 @@ int sqlite3VdbeIdxKeyCompare( assert( sqlite3BtreeCursorIsValid(pCur) ); VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ - /* nCellKey will always be between 0 and 0xffffffff because of the say + /* nCellKey will always be between 0 and 0xffffffff because of the way ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ *res = 0; From 9b8afef260c0ed02cc0d1a195fa697844eb1c02e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Mar 2014 20:48:50 +0000 Subject: [PATCH 033/177] Avoid a buffer overread in vdbeCompareRecordInt() that might occur if the database is corrupt. FossilOrigin-Name: 7c0b4381f0e6f33cb13299a915851d9431bf3850 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 24 +++++++++++++++-------- test/corruptI.test | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 16 deletions(-) create mode 100644 test/corruptI.test diff --git a/manifest b/manifest index 4302dac32e..1f36291c28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2014-03-03T19:29:28.807 +C Avoid\sa\sbuffer\soverread\sin\svdbeCompareRecordInt()\sthat\smight\soccur\sif\sthe\sdatabase\sis\scorrupt. +D 2014-03-03T20:48:50.982 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c b817480055ea37f04cbda6524cbcda56d8620c78 +F src/vdbeaux.c 687165bd405b33553c9aaf5da74c249cea083d24 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -404,6 +404,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 9d8186f6f8751efdfd445d8546fd98f073499039 +F test/corruptI.test d9eca60cb373215d97e0994bf104f9a37e1ac0fc F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1153,7 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ba8993727e5752cbc92c7d78437e9e0510c899f7 -R 84a57b7d3530d77238186688acf2fb15 -U drh -Z 743719e98998c2037f212aec24c916c8 +P 1f4991ab1676bf85599fc32068ceeddeb05f648c +R 112194c4c042293d2f5fc41663c08a7a +U dan +Z e0c8dc8360525071b6a4931f6b50a318 diff --git a/manifest.uuid b/manifest.uuid index 2f5a48fd68..f96d8cdcc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f4991ab1676bf85599fc32068ceeddeb05f648c \ No newline at end of file +7c0b4381f0e6f33cb13299a915851d9431bf3850 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 47a3f0899b..dd724cc39e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3575,7 +3575,8 @@ static int vdbeRecordCompareInt( const UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ - const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1]; + int szHdr = *(const u8*)pKey1; + const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; int res; i64 v = pPKey2->aMem[0].u.i; @@ -3583,7 +3584,6 @@ static int vdbeRecordCompareInt( UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); - switch( serial_type ){ case 1: lhs = (char)(aKey[0]); @@ -3718,12 +3718,20 @@ static int vdbeRecordCompareString( ** as the only argument. */ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ - /* As the varints that make up a record header are all 5 bytes in size - ** or less, if the binary keys being compared have 25 or fewer fields - ** then it is guaranteed that the varint at the start of every record - ** (the record-header size in bytes) fits in a single byte. If this - ** is not the case, then sqlite3VdbeRecordCompare() must be used. */ - if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=25 ){ + /* varintRecordCompareInt() and varintRecordCompareString() both assume + ** that the size-of-header varint that occurs at the start of each record + ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt() + ** also assumes that it is safe to overread a buffer by at least the + ** maximum possible legal header size plus 8 bytes. Because there is + ** guaranteed to be at least 74 (but not 136) bytes of padding following each + ** buffer passed to varintRecordCompareInt() this makes it convenient to + ** limit the size of the header to 64 bytes in cases where the first field + ** is an integer. + ** + ** The easiest way to enforce this limit is to consider only records with + ** 13 fields or less. If the first field is an integer, the maximum legal + ** header size is (12*5 + 1 + 1) bytes. */ + if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=13 ){ int flags = p->aMem[0].flags; if( p->pKeyInfo->aSortOrder[0] ){ p->r1 = 1; diff --git a/test/corruptI.test b/test/corruptI.test new file mode 100644 index 0000000000..9c9a19310b --- /dev/null +++ b/test/corruptI.test @@ -0,0 +1,47 @@ +# 2014-01-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix corruptI + +# Do not use a codec for tests in this file, as the database file is +# manipulated directly using tcl scripts (using the [hexio_write] command). +# +do_not_use_codec +database_may_be_corrupt + +# Initialize the database. +# +do_execsql_test 1.1 { + PRAGMA page_size=1024; + PRAGMA auto_vacuum=0; + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES('a'); +} {} +db close + +do_test 1.2 { + set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] + set off [expr 2*1024 + $offset + 1] + hexio_write test.db $off FF06 + + breakpoint + + sqlite3 db test.db + catchsql { SELECT * FROM t1 WHERE a = 10 } +} {1 {database disk image is malformed}} + + +finish_test + From 13a747e5e4ef5a8ab69169ad9d2dec2653d3490b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Mar 2014 21:46:55 +0000 Subject: [PATCH 034/177] Fix a couple of harmless compiler warnings. FossilOrigin-Name: fcf480cc630976b619aabd5f7fb7b09601a178fb --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 ++ src/vdbeaux.c | 1 - 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1f36291c28..52a206ae83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sbuffer\soverread\sin\svdbeCompareRecordInt()\sthat\smight\soccur\sif\sthe\sdatabase\sis\scorrupt. -D 2014-03-03T20:48:50.982 +C Fix\sa\scouple\sof\sharmless\scompiler\swarnings. +D 2014-03-03T21:46:55.799 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c d288e668614449571ec269535dc4aaf216a23db2 +F src/btree.c 7a81c2c6b5c0bf6334b4247e709c639c573fbcc1 F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 687165bd405b33553c9aaf5da74c249cea083d24 +F src/vdbeaux.c 5adf67ef9cdaa57b393c0a8bf0a29eadbd6f9158 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1154,7 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1f4991ab1676bf85599fc32068ceeddeb05f648c -R 112194c4c042293d2f5fc41663c08a7a -U dan -Z e0c8dc8360525071b6a4931f6b50a318 +P 7c0b4381f0e6f33cb13299a915851d9431bf3850 +R 4042a62ac4c0618104f381105ba13add +U drh +Z 0037dd9417989b6d6f935caf436e0a2a diff --git a/manifest.uuid b/manifest.uuid index f96d8cdcc1..aaa189ba4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c0b4381f0e6f33cb13299a915851d9431bf3850 \ No newline at end of file +fcf480cc630976b619aabd5f7fb7b09601a178fb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4d42969be8..ce35afbd58 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4575,6 +4575,8 @@ int sqlite3BtreeMovetoUnpacked( || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 ); + }else{ + xRecordCompare = 0; /* Not actually used. Avoids a compiler warning. */ } rc = moveToRoot(pCur); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index dd724cc39e..2526d6fc50 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3575,7 +3575,6 @@ static int vdbeRecordCompareInt( const UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ - int szHdr = *(const u8*)pKey1; const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; int res; From a9e0aeb552811d8aecedd872a32cd79af404dcc6 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 00:15:16 +0000 Subject: [PATCH 035/177] Do not assume that 'char' is signed. Make it explicit. FossilOrigin-Name: 979da752e6f8767a61a8efed824ffad9605d0f4c --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 36f6559f55..eea0f83b79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3VdbeRecordCompare()\sroutine\sused\sto\scompare\sbtree\srecords.\nCreate\sa\scouple\sof\sfast-track\sroutines\sto\shandle\sthe\scommon\scases\sof\sa\sstring\nwith\sBINARY\scollation\sor\sinteger\svalues\sas\sthe\sleft-most\scolumn.\s\sThis\sgives\na\ssignificant\sperformance\sboost\sin\scommon\suse. -D 2014-03-03T21:59:33.388 +C Do\snot\sassume\sthat\s'char'\sis\ssigned.\s\sMake\sit\sexplicit. +D 2014-03-04T00:15:16.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h b06500d391d4b7bf4c69fc110e37dd45719b760c +F src/sqliteInt.h bf529055591dd81ec92a84d35dba0854642f6cc2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1154,8 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c0fa0c0e2de50d7eda19ab8862496b18eff93538 fcf480cc630976b619aabd5f7fb7b09601a178fb -R 4042a62ac4c0618104f381105ba13add -T +closed fcf480cc630976b619aabd5f7fb7b09601a178fb +P aec5473a750e412eb1e11e17bbafd760db086c86 +R 3fc76904aa4a71ad8f46b000aee0d4b3 U drh -Z f2b997d224dae00836e71263e0e4d70e +Z f6bc4780db4acc20db7127296b21d314 diff --git a/manifest.uuid b/manifest.uuid index 93e0969edd..0b0b678799 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aec5473a750e412eb1e11e17bbafd760db086c86 \ No newline at end of file +979da752e6f8767a61a8efed824ffad9605d0f4c \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a368b96c63..90e83206ad 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1593,7 +1593,7 @@ struct KeyInfo { struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ - char default_rc; /* Comparison result if keys are equal */ + i8 default_rc; /* Comparison result if keys are equal */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ From f926d1ea47939fe1c60d04bcf269b4ad6c1f50a1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 04:04:33 +0000 Subject: [PATCH 036/177] Fix more instances of assuming 'char' is signed. And, make sure to never shift a signed integer. FossilOrigin-Name: f10130ede433a19b3945753f23962871c8d2dcf3 --- manifest | 12 +++---- manifest.uuid | 2 +- src/vdbeaux.c | 96 ++++++++++++++++++++++++++++----------------------- 3 files changed, 59 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index eea0f83b79..61023f47d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sassume\sthat\s'char'\sis\ssigned.\s\sMake\sit\sexplicit. -D 2014-03-04T00:15:16.875 +C Fix\smore\sinstances\sof\sassuming\s'char'\sis\ssigned.\s\sAnd,\smake\ssure\sto\snever\sshift\na\ssigned\sinteger. +D 2014-03-04T04:04:33.269 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 5adf67ef9cdaa57b393c0a8bf0a29eadbd6f9158 +F src/vdbeaux.c b3bd75b2dfe314fb411c967034314b5316342820 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1154,7 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P aec5473a750e412eb1e11e17bbafd760db086c86 -R 3fc76904aa4a71ad8f46b000aee0d4b3 +P 979da752e6f8767a61a8efed824ffad9605d0f4c +R 76afae803c9615945e7c1ed11c396a45 U drh -Z f6bc4780db4acc20db7127296b21d314 +Z 4679453e8217252329a19e88ce1ddc0d diff --git a/manifest.uuid b/manifest.uuid index 0b0b678799..b38b734ab9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -979da752e6f8767a61a8efed824ffad9605d0f4c \ No newline at end of file +f10130ede433a19b3945753f23962871c8d2dcf3 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2526d6fc50..776821d398 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2942,6 +2942,14 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ return 0; } +/* Input "x" is a sequence of unsigned characters that represent a +** big-endian integer. Return the equivalent native integer +*/ +#define ONE_BYTE_INT(x) ((i8)(x)[0]) +#define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) +#define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) +#define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) + /* ** Deserialize the data blob pointed to by buf as serial type serial_type ** and store the result in pMem. Return the number of bytes read. @@ -2953,7 +2961,6 @@ u32 sqlite3VdbeSerialGet( ){ u64 x; u32 y; - int i; switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ @@ -2962,33 +2969,28 @@ u32 sqlite3VdbeSerialGet( break; } case 1: { /* 1-byte signed integer */ - pMem->u.i = (signed char)buf[0]; + pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; return 1; } case 2: { /* 2-byte signed integer */ - i = 256*(signed char)buf[0] | buf[1]; - pMem->u.i = (i64)i; + pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; return 2; } case 3: { /* 3-byte signed integer */ - i = 65536*(signed char)buf[0] | (buf[1]<<8) | buf[2]; - pMem->u.i = (i64)i; + pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; return 3; } case 4: { /* 4-byte signed integer */ - y = ((unsigned)buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; + y = FOUR_BYTE_UINT(buf); pMem->u.i = (i64)*(int*)&y; pMem->flags = MEM_Int; return 4; } case 5: { /* 6-byte signed integer */ - x = 256*(signed char)buf[0] + buf[1]; - y = ((unsigned)buf[2]<<24) | (buf[3]<<16) | (buf[4]<<8) | buf[5]; - x = (x<<32) | y; - pMem->u.i = *(i64*)&x; + pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; return 6; } @@ -3006,8 +3008,8 @@ u32 sqlite3VdbeSerialGet( swapMixedEndianFloat(t2); assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); #endif - x = ((unsigned)buf[0]<<24) | (buf[1]<<16) | (buf[2]<<8) | buf[3]; - y = ((unsigned)buf[4]<<24) | (buf[5]<<16) | (buf[6]<<8) | buf[7]; + x = FOUR_BYTE_UINT(buf); + y = FOUR_BYTE_UINT(buf+4); x = (x<<32) | y; if( serial_type==6 ){ pMem->u.i = *(i64*)&x; @@ -3352,26 +3354,27 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ ** and returns the value. */ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ + u32 y; assert( CORRUPT_DB || (serial_type>=1 && serial_type<=9 && serial_type!=7) ); switch( serial_type ){ case 0: case 1: - return (char)aKey[0]; + return ONE_BYTE_INT(aKey); case 2: - return ((char)aKey[0] << 8) | aKey[1]; + return TWO_BYTE_INT(aKey); case 3: - return ((char)aKey[0] << 16) | (aKey[1] << 8) | aKey[2]; - case 4: - return ((char)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]; + return THREE_BYTE_INT(aKey); + case 4: { + y = FOUR_BYTE_UINT(aKey); + return (i64)*(int*)&y; + } case 5: { - i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; - u32 lsw = (aKey[4] << 8) | aKey[5]; - return (i64)( msw << 16 | (u64)lsw ); + return FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); } case 6: { - i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; - u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; - return (i64)( msw << 32 | (u64)lsw ); + u64 x = FOUR_BYTE_UINT(aKey); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + return (i64)*(i64*)&x; } } @@ -3578,34 +3581,39 @@ static int vdbeRecordCompareInt( const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; int res; + u32 y; + u64 x; i64 v = pPKey2->aMem[0].u.i; i64 lhs; UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); switch( serial_type ){ - case 1: - lhs = (char)(aKey[0]); - break; - case 2: - lhs = 256*(signed char)aKey[0] + aKey[1]; - break; - case 3: - lhs = 65536*(char)aKey[0] | (aKey[1]<<8) | aKey[2]; - break; - case 4: - lhs = (int)(((u32)aKey[0]<<24) | (aKey[1]<<16) | (aKey[2]<<8)| aKey[3]); - break; - case 5: { - i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; - u32 lsw = (aKey[4] << 8) | aKey[5]; - lhs = (i64)( msw << 16 | (u64)lsw ); + case 1: { /* 1-byte signed integer */ + lhs = ONE_BYTE_INT(aKey); break; } - case 6: { - i64 msw = ((char)aKey[0]<<24)|(aKey[1]<<16)|(aKey[2]<<8)|aKey[3]; - u32 lsw = ((unsigned)aKey[4]<<24)|(aKey[5]<<16)|(aKey[6]<<8)|aKey[7]; - lhs = (i64)( msw << 32 | (u64)lsw ); + case 2: { /* 2-byte signed integer */ + lhs = TWO_BYTE_INT(aKey); + break; + } + case 3: { /* 3-byte signed integer */ + lhs = THREE_BYTE_INT(aKey); + break; + } + case 4: { /* 4-byte signed integer */ + y = FOUR_BYTE_UINT(aKey); + lhs = (i64)*(int*)&y; + break; + } + case 5: { /* 6-byte signed integer */ + lhs = FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + break; + } + case 6: { /* 8-byte signed integer */ + x = FOUR_BYTE_UINT(aKey); + x = (x<<32) | FOUR_BYTE_UINT(aKey+4); + lhs = *(i64*)&x; break; } case 8: From ffe6bc2b93fe2d02f166e1e18f383ac0b8fae0c3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Mar 2014 11:16:20 +0000 Subject: [PATCH 037/177] Fix harmless compiler warning. FossilOrigin-Name: 8d18a803bdeab290a6e9ff26911897f5f8683876 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6590df8169..2850e252c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3VdbeRecordCompare()\sroutine\sused\sto\scompare\sbtree\srecords.\nCreate\sfast-track\sspecial\scase\sroutines\sto\shandle\sthe\scommon\scases\smore\nquickly.\s\sThis\sgives\sa\ssignificant\sperformance\sboost. -D 2014-03-04T04:12:56.404 +C Fix\sharmless\scompiler\swarning. +D 2014-03-04T11:16:20.925 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c b3bd75b2dfe314fb411c967034314b5316342820 +F src/vdbeaux.c 636a7e2eaaa48c3e3585b28b9b1673df78fc0fad F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1154,8 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c0fa0c0e2de50d7eda19ab8862496b18eff93538 f10130ede433a19b3945753f23962871c8d2dcf3 -R 76afae803c9615945e7c1ed11c396a45 -T +closed f10130ede433a19b3945753f23962871c8d2dcf3 -U drh -Z ac0c95f2ad16dfe0178099b125074c2f +P 3325ad5bdc2f81f63b556d6f4d0589d89b142b2b +R cb72f483f7d75c1651c42a378d8bb016 +U mistachkin +Z 266049d6eb1d68d04ea732570e934710 diff --git a/manifest.uuid b/manifest.uuid index 3c00b464cf..2600f64d40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3325ad5bdc2f81f63b556d6f4d0589d89b142b2b \ No newline at end of file +8d18a803bdeab290a6e9ff26911897f5f8683876 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 776821d398..90cc415342 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3404,7 +3404,7 @@ int sqlite3VdbeRecordCompare( ){ u32 d1; /* Offset into aKey[] of next data element */ int i; /* Index of next field to compare */ - int szHdr1; /* Size of record header in bytes */ + u32 szHdr1; /* Size of record header in bytes */ u32 idx1; /* Offset of first type in header */ int rc = 0; /* Return value */ Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ From 9d42cc994cd2c06b504f8eba50a49acfda76a050 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Mar 2014 11:29:42 +0000 Subject: [PATCH 038/177] Fix name resolution problem in sub-selects within triggers, ticket [4ef7e3cfca]. FossilOrigin-Name: 5bcd0b1ca5d73ffbe7978ee9d73fe5e769e3d3a2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/resolve.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2850e252c6..515845694f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2014-03-04T11:16:20.925 +C Fix\sname\sresolution\sproblem\sin\ssub-selects\swithin\striggers,\sticket\s[4ef7e3cfca]. +D 2014-03-04T11:29:42.399 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/pragma.c a46ee83671f5c95f53d2ceeb5e1a818d7b1df99a F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c ca8b99d894164435f5c55cb304c1b8121705c51e +F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 780bbf39e401091845ba745a38326eabe5d44293 F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed @@ -1154,7 +1154,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3325ad5bdc2f81f63b556d6f4d0589d89b142b2b -R cb72f483f7d75c1651c42a378d8bb016 +P 8d18a803bdeab290a6e9ff26911897f5f8683876 +R a0a3e47c5907244e634e0e73af124fa7 U mistachkin -Z 266049d6eb1d68d04ea732570e934710 +Z 20401020b81859c6f4c872d433156214 diff --git a/manifest.uuid b/manifest.uuid index 2600f64d40..d8e1e43d94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d18a803bdeab290a6e9ff26911897f5f8683876 \ No newline at end of file +5bcd0b1ca5d73ffbe7978ee9d73fe5e769e3d3a2 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 07e0e919a5..86169c51c1 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -337,6 +337,8 @@ static int lookupName( }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ pExpr->iTable = 0; pTab = pParse->pTriggerTab; + }else{ + pTab = 0; } if( pTab ){ From 5810d0315b21a821a2882801d7676f35451fa3ea Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Mar 2014 11:35:20 +0000 Subject: [PATCH 039/177] Add tests to verify the fix for bug [4ef7e3cfca]. FossilOrigin-Name: 62410bb8a7b33efcd13bce8fd75b83523922adb8 --- manifest | 13 ++++---- manifest.uuid | 2 +- test/tkt-4ef7e3cfca.test | 71 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 test/tkt-4ef7e3cfca.test diff --git a/manifest b/manifest index 515845694f..59749ee867 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sname\sresolution\sproblem\sin\ssub-selects\swithin\striggers,\sticket\s[4ef7e3cfca]. -D 2014-03-04T11:29:42.399 +C Add\stests\sto\sverify\sthe\sfix\sfor\sbug\s[4ef7e3cfca]. +D 2014-03-04T11:35:20.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -870,6 +870,7 @@ F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70 F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0 F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 +F test/tkt-4ef7e3cfca.test 9d7ac520e022c62ac69aea55df58f06ae2fdb86c F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 @@ -1154,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8d18a803bdeab290a6e9ff26911897f5f8683876 -R a0a3e47c5907244e634e0e73af124fa7 -U mistachkin -Z 20401020b81859c6f4c872d433156214 +P 5bcd0b1ca5d73ffbe7978ee9d73fe5e769e3d3a2 +R ada57a884b5a4f5da4f1cf238fd54717 +U dan +Z 0e893b0948d9d4cf092e0210eb113968 diff --git a/manifest.uuid b/manifest.uuid index d8e1e43d94..5f64e27616 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5bcd0b1ca5d73ffbe7978ee9d73fe5e769e3d3a2 \ No newline at end of file +62410bb8a7b33efcd13bce8fd75b83523922adb8 \ No newline at end of file diff --git a/test/tkt-4ef7e3cfca.test b/test/tkt-4ef7e3cfca.test new file mode 100644 index 0000000000..94143c029d --- /dev/null +++ b/test/tkt-4ef7e3cfca.test @@ -0,0 +1,71 @@ +# 2013-10-30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements regression tests for SQLite library. The +# focus of this file is testing WITHOUT ROWID tables. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-4ef7e3cfca.test + +do_catchsql_test 1.1 { + CREATE TABLE x(a); + CREATE TRIGGER t AFTER INSERT ON x BEGIN + SELECT * FROM x WHERE abc.a = 1; + END; + INSERT INTO x VALUES('assert'); +} {1 {no such column: abc.a}} + +reset_db +do_execsql_test 2.1 { + CREATE TABLE w(a); + CREATE TABLE x(a); + CREATE TABLE y(a); + CREATE TABLE z(a); + + INSERT INTO x(a) VALUES(5); + INSERT INTO y(a) VALUES(10); + + CREATE TRIGGER t AFTER INSERT ON w BEGIN + INSERT INTO z + SELECT (SELECT x.a + y.a FROM y) FROM x; + END; + INSERT INTO w VALUES('incorrect'); +} +do_execsql_test 2.2 { + SELECT * FROM z; +} {15} + +reset_db +do_execsql_test 3.1 { + CREATE TABLE w(a); + CREATE TABLE x(b); + CREATE TABLE y(a); + CREATE TABLE z(a); + + INSERT INTO x(b) VALUES(5); + INSERT INTO y(a) VALUES(10); + + CREATE TRIGGER t AFTER INSERT ON w BEGIN + INSERT INTO z + SELECT (SELECT x.b + y.a FROM y) FROM x; + END; + INSERT INTO w VALUES('assert'); +} +do_execsql_test 3.2 { + SELECT * FROM z; +} {15} + + +finish_test + + From ad4b262545245926a57dd4c148da757c24ba1c6e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Mar 2014 11:54:35 +0000 Subject: [PATCH 040/177] Fix minor typos in the new test file. FossilOrigin-Name: 3cbb9b1b4f8c5c2735799e84dc80614586222cd2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/tkt-4ef7e3cfca.test | 7 ++----- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 59749ee867..fcfd86e17a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\sverify\sthe\sfix\sfor\sbug\s[4ef7e3cfca]. -D 2014-03-04T11:35:20.387 +C Fix\sminor\stypos\sin\sthe\snew\stest\sfile. +D 2014-03-04T11:54:35.173 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -870,7 +870,7 @@ F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70 F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0 F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 -F test/tkt-4ef7e3cfca.test 9d7ac520e022c62ac69aea55df58f06ae2fdb86c +F test/tkt-4ef7e3cfca.test 24aaa0a94a39dbed9032de6628910f694ee55f57 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5bcd0b1ca5d73ffbe7978ee9d73fe5e769e3d3a2 -R ada57a884b5a4f5da4f1cf238fd54717 -U dan -Z 0e893b0948d9d4cf092e0210eb113968 +P 62410bb8a7b33efcd13bce8fd75b83523922adb8 +R 21629d5246fd4554d222f50cf7c4ec60 +U mistachkin +Z 1848296ec199a8f8ef5d71969502e9a2 diff --git a/manifest.uuid b/manifest.uuid index 5f64e27616..a88b46658b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62410bb8a7b33efcd13bce8fd75b83523922adb8 \ No newline at end of file +3cbb9b1b4f8c5c2735799e84dc80614586222cd2 \ No newline at end of file diff --git a/test/tkt-4ef7e3cfca.test b/test/tkt-4ef7e3cfca.test index 94143c029d..7a8f3b6a12 100644 --- a/test/tkt-4ef7e3cfca.test +++ b/test/tkt-4ef7e3cfca.test @@ -1,4 +1,4 @@ -# 2013-10-30 +# 2014-03-04 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -15,7 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -set testprefix tkt-4ef7e3cfca.test +set testprefix tkt-4ef7e3cfca do_catchsql_test 1.1 { CREATE TABLE x(a); @@ -65,7 +65,4 @@ do_execsql_test 3.2 { SELECT * FROM z; } {15} - finish_test - - From b64fa0adf7d3a8f0354aa90f06926fd8ef2b0a77 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Mar 2014 12:07:24 +0000 Subject: [PATCH 041/177] Fix comment in test. FossilOrigin-Name: 1e57f67b20a1c01ede36d684be628e9270bf109a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/tkt-4ef7e3cfca.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index fcfd86e17a..9e2889a80a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\stypos\sin\sthe\snew\stest\sfile. -D 2014-03-04T11:54:35.173 +C Fix\scomment\sin\stest. +D 2014-03-04T12:07:24.678 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -870,7 +870,7 @@ F test/tkt-3fe897352e.test 27e26eb0f1811aeba4d65aba43a4c52e99da5e70 F test/tkt-4a03edc4c8.test 91c0e135888cdc3d4eea82406a44b05c8c1648d0 F test/tkt-4c86b126f2.test cbcc611becd0396890169ab23102dd70048bbc9a F test/tkt-4dd95f6943.test 3d0ce415d2ee15d3d564121960016b9c7be79407 -F test/tkt-4ef7e3cfca.test 24aaa0a94a39dbed9032de6628910f694ee55f57 +F test/tkt-4ef7e3cfca.test 3965ae11cc9cf6e334f9d7d3c1e20bf8d56254b1 F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 62410bb8a7b33efcd13bce8fd75b83523922adb8 -R 21629d5246fd4554d222f50cf7c4ec60 +P 3cbb9b1b4f8c5c2735799e84dc80614586222cd2 +R 6bc642348ddf6f33974f36f3d8fd1bd7 U mistachkin -Z 1848296ec199a8f8ef5d71969502e9a2 +Z 8b1594ffbd0318ed08d8afbfa1da0977 diff --git a/manifest.uuid b/manifest.uuid index a88b46658b..414d8215f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3cbb9b1b4f8c5c2735799e84dc80614586222cd2 \ No newline at end of file +1e57f67b20a1c01ede36d684be628e9270bf109a \ No newline at end of file diff --git a/test/tkt-4ef7e3cfca.test b/test/tkt-4ef7e3cfca.test index 7a8f3b6a12..fc1efdef66 100644 --- a/test/tkt-4ef7e3cfca.test +++ b/test/tkt-4ef7e3cfca.test @@ -9,8 +9,8 @@ # #*********************************************************************** # -# This file implements regression tests for SQLite library. The -# focus of this file is testing WITHOUT ROWID tables. +# This file implements tests to verify that ticket [4ef7e3cfca] has been +# fixed. # set testdir [file dirname $argv0] From 5f1d536b537429454262a5907ec734444e32f075 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 13:18:23 +0000 Subject: [PATCH 042/177] Improve clarity of presentation in the sqlite3VdbeMemFromBtree() routine. FossilOrigin-Name: 9830c343bc954b828f6ca752f8ae63e2c0a980c1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 15 ++++++++------- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9e2889a80a..c0ec8eeea1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scomment\sin\stest. -D 2014-03-04T12:07:24.678 +C Improve\sclarity\sof\spresentation\sin\sthe\ssqlite3VdbeMemFromBtree()\sroutine. +D 2014-03-04T13:18:23.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 F src/vdbeaux.c 636a7e2eaaa48c3e3585b28b9b1673df78fc0fad F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 2d7918e4c80546d943414668b1485b2581f58a28 +F src/vdbemem.c 10b250f09a3843ee2bcabcadf50ca21fc3ff1f87 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3cbb9b1b4f8c5c2735799e84dc80614586222cd2 -R 6bc642348ddf6f33974f36f3d8fd1bd7 -U mistachkin -Z 8b1594ffbd0318ed08d8afbfa1da0977 +P 1e57f67b20a1c01ede36d684be628e9270bf109a +R a19a15248ea3d7c53bb59593fa01c33d +U drh +Z e6c94fb8a2515aaa56cdd644b73247d7 diff --git a/manifest.uuid b/manifest.uuid index 414d8215f6..0ddf6b66df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e57f67b20a1c01ede36d684be628e9270bf109a \ No newline at end of file +9830c343bc954b828f6ca752f8ae63e2c0a980c1 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index d782696149..378de2deb7 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -827,22 +827,23 @@ int sqlite3VdbeMemFromBtree( sqlite3VdbeMemRelease(pMem); pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; + pMem->n = (int)amt; }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - pMem->flags = MEM_Blob|MEM_Term; - pMem->enc = 0; - pMem->memType = MEM_Blob; if( key ){ rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); }else{ rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); } - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->memType = MEM_Blob; + pMem->n = (int)amt; + }else{ sqlite3VdbeMemRelease(pMem); } } - pMem->n = (int)amt; return rc; } From a4660bd6956efb0a4fe235e6987bc7a17a040d48 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Mar 2014 16:05:25 +0000 Subject: [PATCH 043/177] Ensure that BtreeMovetoUnpacked() does not try to invoke xRecordCompare when it is set to NULL. FossilOrigin-Name: 2879234b7c373be422af3fe180cae9840943e29a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c0ec8eeea1..41a1585390 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sclarity\sof\spresentation\sin\sthe\ssqlite3VdbeMemFromBtree()\sroutine. -D 2014-03-04T13:18:23.018 +C Ensure\sthat\sBtreeMovetoUnpacked()\sdoes\snot\stry\sto\sinvoke\sxRecordCompare\swhen\sit\sis\sset\sto\sNULL. +D 2014-03-04T16:05:25.328 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 7a81c2c6b5c0bf6334b4247e709c639c573fbcc1 +F src/btree.c ad795be588e394a0921796f87cc27717359738cc F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1e57f67b20a1c01ede36d684be628e9270bf109a -R a19a15248ea3d7c53bb59593fa01c33d -U drh -Z e6c94fb8a2515aaa56cdd644b73247d7 +P 9830c343bc954b828f6ca752f8ae63e2c0a980c1 +R b15237a6eb7d81dbdc119d4772ab8878 +U dan +Z 8043d866c01ebc393e98f71c0dcb2e87 diff --git a/manifest.uuid b/manifest.uuid index 0ddf6b66df..64f338da40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9830c343bc954b828f6ca752f8ae63e2c0a980c1 \ No newline at end of file +2879234b7c373be422af3fe180cae9840943e29a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ce35afbd58..eebf6f1e89 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4611,7 +4611,7 @@ int sqlite3BtreeMovetoUnpacked( assert( biasRight==0 || biasRight==1 ); idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pPage->intKey ){ + if( xRecordCompare==0 ){ for(;;){ i64 nCellKey; pCell = findCell(pPage, idx) + pPage->childPtrSize; From ed79b45cf7891236ff377fda971da66c61d01163 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 16:21:18 +0000 Subject: [PATCH 044/177] Fix an assert() so that it takes OOM errors into account. FossilOrigin-Name: 53f6ec792f5f62fcdbf25e0fce5545dd1c3048a4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 41a1585390..651dbf3147 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sBtreeMovetoUnpacked()\sdoes\snot\stry\sto\sinvoke\sxRecordCompare\swhen\sit\sis\sset\sto\sNULL. -D 2014-03-04T16:05:25.328 +C Fix\san\sassert()\sso\sthat\sit\stakes\sOOM\serrors\sinto\saccount. +D 2014-03-04T16:21:18.103 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 636a7e2eaaa48c3e3585b28b9b1673df78fc0fad +F src/vdbeaux.c 0be881d8da9c4413038bb0722de9ecc54788de39 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 10b250f09a3843ee2bcabcadf50ca21fc3ff1f87 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9830c343bc954b828f6ca752f8ae63e2c0a980c1 -R b15237a6eb7d81dbdc119d4772ab8878 -U dan -Z 8043d866c01ebc393e98f71c0dcb2e87 +P 2879234b7c373be422af3fe180cae9840943e29a +R 2b2904237b1b4d9d5fce461560fa87d0 +U drh +Z 60f0b04eed352e57b9c570cb2f1addac diff --git a/manifest.uuid b/manifest.uuid index 64f338da40..9e0ced7e12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2879234b7c373be422af3fe180cae9840943e29a \ No newline at end of file +53f6ec792f5f62fcdbf25e0fce5545dd1c3048a4 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 90cc415342..b961174c47 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3539,9 +3539,10 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB + assert( CORRUPT_DB || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) + || pKeyInfo->db->mallocFailed ); assert( mem1.zMalloc==0 ); /* See comment below */ return rc; From d1053a4314b6da60d30ef622dd39c3fd5cf685ea Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 18:06:04 +0000 Subject: [PATCH 045/177] Fix a potential memory use-after-free problem following an OOM error. FossilOrigin-Name: 767ccb1fa11b3e7b895fb8c2e91f79e3b4202907 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 651dbf3147..e45868c0f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sso\sthat\sit\stakes\sOOM\serrors\sinto\saccount. -D 2014-03-04T16:21:18.103 +C Fix\sa\spotential\smemory\suse-after-free\sproblem\sfollowing\san\sOOM\serror. +D 2014-03-04T18:06:04.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 F src/vdbeaux.c 0be881d8da9c4413038bb0722de9ecc54788de39 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 10b250f09a3843ee2bcabcadf50ca21fc3ff1f87 +F src/vdbemem.c 1b1ca620f2638e800a5bef6508d1c4320a3a4f11 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2879234b7c373be422af3fe180cae9840943e29a -R 2b2904237b1b4d9d5fce461560fa87d0 +P 53f6ec792f5f62fcdbf25e0fce5545dd1c3048a4 +R b289979daf151f0fbafb28fbb7956db7 U drh -Z 60f0b04eed352e57b9c570cb2f1addac +Z f463125a8b67b6cdc900f4915889d477 diff --git a/manifest.uuid b/manifest.uuid index 9e0ced7e12..38b7f2ed5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -53f6ec792f5f62fcdbf25e0fce5545dd1c3048a4 \ No newline at end of file +767ccb1fa11b3e7b895fb8c2e91f79e3b4202907 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 378de2deb7..b81e50d2e9 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -122,6 +122,7 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ } if( pMem->zMalloc==0 ){ VdbeMemRelease(pMem); + pMem->z = 0; pMem->flags = MEM_Null; return SQLITE_NOMEM; } From 052c6784eed4dd88116872182c9ff8ec962f1bfa Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Mar 2014 21:00:20 +0000 Subject: [PATCH 046/177] Fix a problem with "DEFAULT (-(-9223372036854775808))" clauses in ALTER TABLE ... ADD COLUMN commands. FossilOrigin-Name: e072cb3ee2a03d786c08230cecc6e970de2cec5b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 4 ++-- test/alter4.test | 21 +++++++++++++++++++++ 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e45868c0f7..d749fdfeb6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\smemory\suse-after-free\sproblem\sfollowing\san\sOOM\serror. -D 2014-03-04T18:06:04.531 +C Fix\sa\sproblem\swith\s"DEFAULT\s(-(-9223372036854775808))"\sclauses\sin\sALTER\sTABLE\s...\sADD\sCOLUMN\scommands. +D 2014-03-04T21:00:20.840 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 F src/vdbeaux.c 0be881d8da9c4413038bb0722de9ecc54788de39 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c 1b1ca620f2638e800a5bef6508d1c4320a3a4f11 +F src/vdbemem.c ba5cbafdbf123cb51208580d2281312cc45905eb F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -300,7 +300,7 @@ F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test e88dfa77e020c2b48e52a8020c70171ab828e079 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d -F test/alter4.test 8e93bf7a7e6919b14b0c9a6c1e4908bcf21b0165 +F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 53f6ec792f5f62fcdbf25e0fce5545dd1c3048a4 -R b289979daf151f0fbafb28fbb7956db7 -U drh -Z f463125a8b67b6cdc900f4915889d477 +P 767ccb1fa11b3e7b895fb8c2e91f79e3b4202907 +R 8626b9d0aec4fcc431f7a3c606cbb670 +U dan +Z 5677ac60eef96d141429e6ff969758a5 diff --git a/manifest.uuid b/manifest.uuid index 38b7f2ed5c..5bcec6a617 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -767ccb1fa11b3e7b895fb8c2e91f79e3b4202907 \ No newline at end of file +e072cb3ee2a03d786c08230cecc6e970de2cec5b \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index b81e50d2e9..5dead85208 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1042,9 +1042,9 @@ static int valueFromExpr( ){ sqlite3VdbeMemNumerify(pVal); if( pVal->u.i==SMALLEST_INT64 ){ - pVal->flags &= MEM_Int; + pVal->flags &= ~MEM_Int; pVal->flags |= MEM_Real; - pVal->r = (double)LARGEST_INT64; + pVal->r = (double)SMALLEST_INT64; }else{ pVal->u.i = -pVal->u.i; } diff --git a/test/alter4.test b/test/alter4.test index 59704fed4c..eaad37e001 100644 --- a/test/alter4.test +++ b/test/alter4.test @@ -334,4 +334,25 @@ do_test alter4-8.2 { } } [list $::sql] + +# Test that a default value equal to -1 multipied by the smallest possible +# 64-bit integer is correctly converted to a real. +do_execsql_test alter4-9.1 { + CREATE TABLE t5( + a INTEGER DEFAULT -9223372036854775808, + b INTEGER DEFAULT (-(-9223372036854775808)) + ); + INSERT INTO t5 DEFAULT VALUES; +} + +do_execsql_test alter4-9.2 { SELECT typeof(a), a, typeof(b), b FROM t5; } { + integer -9223372036854775808 + real 9.22337203685478e+18 +} + +do_execsql_test alter4-9.3 { + ALTER TABLE t5 ADD COLUMN c INTEGER DEFAULT (-(-9223372036854775808)); + SELECT typeof(c), c FROM t5; +} {real 9.22337203685478e+18} + finish_test From 19c6d96aec7545d5c8723c48bde5089a8f89a5c7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Mar 2014 21:19:51 +0000 Subject: [PATCH 047/177] Avoid indexing off the front end of an array when creating a view with two or more blank column names in the SELECT statement that defines the view. FossilOrigin-Name: 554501f158534f9c27fd51900589c2ab7cf300d8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- test/view.test | 13 +++++++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d749fdfeb6..c20e72d114 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\s"DEFAULT\s(-(-9223372036854775808))"\sclauses\sin\sALTER\sTABLE\s...\sADD\sCOLUMN\scommands. -D 2014-03-04T21:00:20.840 +C Avoid\sindexing\soff\sthe\sfront\send\sof\san\sarray\swhen\screating\sa\sview\swith\ntwo\sor\smore\sblank\scolumn\snames\sin\sthe\sSELECT\sstatement\sthat\sdefines\sthe\sview. +D 2014-03-04T21:19:51.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 780bbf39e401091845ba745a38326eabe5d44293 +F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1028,7 +1028,7 @@ F test/vacuum3.test 77ecdd54592b45a0bcb133339f99f1ae0ae94d0d F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 -F test/view.test 4057630287bfa5955628fe90a13d4c225d1c7352 +F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 767ccb1fa11b3e7b895fb8c2e91f79e3b4202907 -R 8626b9d0aec4fcc431f7a3c606cbb670 -U dan -Z 5677ac60eef96d141429e6ff969758a5 +P e072cb3ee2a03d786c08230cecc6e970de2cec5b +R 213bc20e67a019cbd9a16484715bd6d2 +U drh +Z 02c018f7f11d0867eb15887f2517d7cb diff --git a/manifest.uuid b/manifest.uuid index 5bcec6a617..41b8fab845 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e072cb3ee2a03d786c08230cecc6e970de2cec5b \ No newline at end of file +554501f158534f9c27fd51900589c2ab7cf300d8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 9430888c4a..850bc6a902 100644 --- a/src/select.c +++ b/src/select.c @@ -1522,7 +1522,7 @@ static int selectColumnsFromExprList( char *zNewName; int k; for(k=nName-1; k>1 && sqlite3Isdigit(zName[k]); k--){} - if( zName[k]==':' ) nName = k; + if( k>=0 && zName[k]==':' ) nName = k; zName[nName] = 0; zNewName = sqlite3MPrintf(db, "%s:%d", zName, ++cnt); sqlite3DbFree(db, zName); diff --git a/test/view.test b/test/view.test index 779f77b6ae..3ba6c0b465 100644 --- a/test/view.test +++ b/test/view.test @@ -611,4 +611,17 @@ ifcapable progress { } {1 interrupted} } +db close +sqlite3 db :memory: +do_execsql_test view-22.1 { + CREATE VIEW x1 AS SELECT 123 AS '', 234 AS '', 345 AS ''; + SELECT * FROM x1; +} {123 234 345} +do_test view-22.2 { + unset -nocomplain x + db eval {SELECT * FROM x1} x break + lsort [array names x] +} {{} * :1 :2} + + finish_test From afc14f72ef54b9f983ca16a96da85cf6aee40b7f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 5 Mar 2014 01:29:18 +0000 Subject: [PATCH 048/177] Experimental simplification of memory flags/type handling. FossilOrigin-Name: bac2820e13a79d91d0f8938c643134a9d6900327 --- manifest | 29 ++++++++++++++++------------- manifest.uuid | 2 +- src/test5.c | 1 - src/test_func.c | 2 -- src/vdbe.c | 6 ------ src/vdbeInt.h | 4 +--- src/vdbeapi.c | 4 ++-- src/vdbeaux.c | 10 ---------- src/vdbemem.c | 15 +-------------- 9 files changed, 21 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index c20e72d114..f769869fdb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sindexing\soff\sthe\sfront\send\sof\san\sarray\swhen\screating\sa\sview\swith\ntwo\sor\smore\sblank\scolumn\snames\sin\sthe\sSELECT\sstatement\sthat\sdefines\sthe\sview. -D 2014-03-04T21:19:51.375 +C Experimental\ssimplification\sof\smemory\sflags/type\shandling. +D 2014-03-05T01:29:18.160 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/test1.c 2401eee14a4309a7cfe2aeb2f30ad517a1d9c299 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df -F src/test5.c 41e6e732f14a54c7b47f753e364700760f6521b0 +F src/test5.c 5a34feec76d9b3a86aab30fd4f6cc9c48cbab4c1 F src/test6.c 41cacf3b0dd180823919bf9e1fbab287c9266723 F src/test7.c 72b732baa5642f795655ba1126ea032af46ecfd2 F src/test8.c 54ccd7b1df5062f0ecbf50a8f7b618f8b1f13b20 @@ -243,7 +243,7 @@ F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c f8235719dff4bf9ffee04c55a190af8782ce9ab5 +F src/test_func.c d3013ce36f19ac72a99c73864930fd1fa41832f8 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 @@ -277,13 +277,13 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 75c3f5d27ab79af214646cf37d7551bc8cec09c2 +F src/vdbe.c 8c6fc7bb9f2218c0e43f24d847e596effa8671e2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 -F src/vdbeInt.h 9ccca0bc7646c918d065943e44bead4bf5de213d -F src/vdbeapi.c 5bc41aaea448a7fc250902c418f1795859be3820 -F src/vdbeaux.c 0be881d8da9c4413038bb0722de9ecc54788de39 +F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b +F src/vdbeapi.c a5c48f2b015679ddb7c25ae866abf5e2d6e42857 +F src/vdbeaux.c 43bee29ac866f7ce6af90c4f084bb22c160b8b70 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 -F src/vdbemem.c ba5cbafdbf123cb51208580d2281312cc45905eb +F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1155,7 +1155,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e072cb3ee2a03d786c08230cecc6e970de2cec5b -R 213bc20e67a019cbd9a16484715bd6d2 -U drh -Z 02c018f7f11d0867eb15887f2517d7cb +P 554501f158534f9c27fd51900589c2ab7cf300d8 +R f9e55f86cd630e3c544c56ddb75f2871 +T *branch * noMemType +T *sym-noMemType * +T -sym-trunk * +U mistachkin +Z 24e103c83e028827d3da7d1aab872449 diff --git a/manifest.uuid b/manifest.uuid index 41b8fab845..91862e5073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -554501f158534f9c27fd51900589c2ab7cf300d8 \ No newline at end of file +bac2820e13a79d91d0f8938c643134a9d6900327 \ No newline at end of file diff --git a/src/test5.c b/src/test5.c index bf574e3c3c..952e3325e5 100644 --- a/src/test5.c +++ b/src/test5.c @@ -76,7 +76,6 @@ static int test_value_overhead( val.flags = MEM_Str|MEM_Term|MEM_Static; val.z = "hello world"; - val.memType = MEM_Str; val.enc = SQLITE_UTF8; for(i=0; ip1+i, &pMem[i]); } if( db->mallocFailed ) goto no_mem; @@ -1489,7 +1487,6 @@ case OP_Function: { assert( memIsValid(pArg) ); apVal[i] = pArg; Deephemeralize(pArg); - sqlite3VdbeMemStoreType(pArg); REGISTER_TRACE(pOp->p2+i, pArg); } @@ -5494,7 +5491,6 @@ case OP_AggStep: { assert( memIsValid(pRec) ); apVal[i] = pRec; memAboutToChange(p, pRec); - sqlite3VdbeMemStoreType(pRec); } ctx.pFunc = pOp->p4.pFunc; assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); @@ -5928,7 +5924,6 @@ case OP_VFilter: { /* jump */ apArg = p->apArg; for(i = 0; iinVtabMethod = 1; @@ -6135,7 +6130,6 @@ case OP_VUpdate: { for(i=0; imemType = (u8)((X)->flags&0x1f) -/* void sqlite3VdbeMemStoreType(Mem *pMem); */ int sqlite3VdbeTransferError(Vdbe *p); int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c1edbe48e2..4c9d4870d0 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -206,7 +206,7 @@ int sqlite3_value_type(sqlite3_value* pVal){ SQLITE_INTEGER, /* 0x1e */ SQLITE_NULL, /* 0x1f */ }; - return aType[pVal->memType&0x1f]; + return aType[pVal->flags&MEM_AffMask]; } /**************************** sqlite3_result_ ******************************* @@ -758,7 +758,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ #if defined(SQLITE_DEBUG) && defined(__GNUC__) __attribute__((aligned(8))) #endif - = {0, "", (double)0, {0}, 0, MEM_Null, SQLITE_NULL, 0, + = {0, "", (double)0, {0}, 0, MEM_Null, 0, #ifdef SQLITE_DEBUG 0, 0, /* pScopyFrom, pFiller */ #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b961174c47..00ec901588 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1363,7 +1363,6 @@ int sqlite3VdbeList( } if( p->explain==1 ){ pMem->flags = MEM_Int; - pMem->memType = MEM_Int; pMem->u.i = i; /* Program counter */ pMem++; @@ -1371,7 +1370,6 @@ int sqlite3VdbeList( pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ assert( pMem->z!=0 ); pMem->n = sqlite3Strlen30(pMem->z); - pMem->memType = MEM_Str; pMem->enc = SQLITE_UTF8; pMem++; @@ -1397,17 +1395,14 @@ int sqlite3VdbeList( pMem->flags = MEM_Int; pMem->u.i = pOp->p1; /* P1 */ - pMem->memType = MEM_Int; pMem++; pMem->flags = MEM_Int; pMem->u.i = pOp->p2; /* P2 */ - pMem->memType = MEM_Int; pMem++; pMem->flags = MEM_Int; pMem->u.i = pOp->p3; /* P3 */ - pMem->memType = MEM_Int; pMem++; if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ @@ -1423,7 +1418,6 @@ int sqlite3VdbeList( pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; } - pMem->memType = MEM_Str; pMem++; if( p->explain==1 ){ @@ -1434,7 +1428,6 @@ int sqlite3VdbeList( pMem->flags = MEM_Str|MEM_Term; pMem->n = 2; sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ - pMem->memType = MEM_Str; pMem->enc = SQLITE_UTF8; pMem++; @@ -1445,11 +1438,9 @@ int sqlite3VdbeList( } pMem->flags = MEM_Str|MEM_Term; pMem->n = displayComment(pOp, zP4, pMem->z, 500); - pMem->memType = MEM_Str; pMem->enc = SQLITE_UTF8; #else pMem->flags = MEM_Null; /* Comment */ - pMem->memType = MEM_Null; #endif } @@ -3937,7 +3928,6 @@ sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff){ if( pRet ){ sqlite3VdbeMemCopy((Mem *)pRet, pMem); sqlite3ValueApplyAffinity(pRet, aff, SQLITE_UTF8); - sqlite3VdbeMemStoreType((Mem *)pRet); } return pRet; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 5dead85208..2c4aa4ad7f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -321,7 +321,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ /* ** Release any memory held by the Mem. This may leave the Mem in an ** inconsistent state, for example with (Mem.z==0) and -** (Mem.memType==MEM_Str). +** (Mem.flags==MEM_Str). */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); @@ -513,7 +513,6 @@ void sqlite3VdbeMemSetNull(Mem *pMem){ sqlite3RowSetClear(pMem->u.pRowSet); } MemSetTypeFlag(pMem, MEM_Null); - pMem->memType = MEM_Null; } void sqlite3ValueSetNull(sqlite3_value *p){ sqlite3VdbeMemSetNull((Mem*)p); @@ -526,7 +525,6 @@ void sqlite3ValueSetNull(sqlite3_value *p){ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ sqlite3VdbeMemRelease(pMem); pMem->flags = MEM_Blob|MEM_Zero; - pMem->memType = MEM_Blob; pMem->n = 0; if( n<0 ) n = 0; pMem->u.nZero = n; @@ -549,7 +547,6 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ sqlite3VdbeMemRelease(pMem); pMem->u.i = val; pMem->flags = MEM_Int; - pMem->memType = MEM_Int; } #ifndef SQLITE_OMIT_FLOATING_POINT @@ -564,7 +561,6 @@ void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ sqlite3VdbeMemRelease(pMem); pMem->r = val; pMem->flags = MEM_Real; - pMem->memType = MEM_Real; } } #endif @@ -773,7 +769,6 @@ int sqlite3VdbeMemSetStr( pMem->n = nByte; pMem->flags = flags; pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); - pMem->memType = flags&0x1f; #ifndef SQLITE_OMIT_UTF16 if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ @@ -839,7 +834,6 @@ int sqlite3VdbeMemFromBtree( pMem->z[amt] = 0; pMem->z[amt+1] = 0; pMem->flags = MEM_Blob|MEM_Term; - pMem->memType = MEM_Blob; pMem->n = (int)amt; }else{ sqlite3VdbeMemRelease(pMem); @@ -902,7 +896,6 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){ Mem *p = sqlite3DbMallocZero(db, sizeof(*p)); if( p ){ p->flags = MEM_Null; - p->memType = MEM_Null; p->db = db; } return p; @@ -951,7 +944,6 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); for(i=0; iaMem[i].flags = MEM_Null; - pRec->aMem[i].memType = MEM_Null; pRec->aMem[i].db = db; } }else{ @@ -1024,7 +1016,6 @@ static int valueFromExpr( zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); if( zVal==0 ) goto no_mem; sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); - if( op==TK_FLOAT ) pVal->memType = MEM_Real; } if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_NONE ){ sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); @@ -1070,9 +1061,6 @@ static int valueFromExpr( } #endif - if( pVal ){ - sqlite3VdbeMemStoreType(pVal); - } *ppVal = pVal; return rc; @@ -1236,7 +1224,6 @@ int sqlite3Stat4ProbeSetValue( sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); } pVal->db = pParse->db; - sqlite3VdbeMemStoreType((Mem*)pVal); } } }else{ From d520041bc0805b50264072ed19ba603ff528632e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Mar 2014 11:22:33 +0000 Subject: [PATCH 049/177] Do not run corruptI.test with the mmap permutation. FossilOrigin-Name: 378b290da85cadc1acba081322d1f0e8d7694e17 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/corruptI.test | 5 +++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c20e72d114..fc35092902 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sindexing\soff\sthe\sfront\send\sof\san\sarray\swhen\screating\sa\sview\swith\ntwo\sor\smore\sblank\scolumn\snames\sin\sthe\sSELECT\sstatement\sthat\sdefines\sthe\sview. -D 2014-03-04T21:19:51.375 +C Do\snot\srun\scorruptI.test\swith\sthe\smmap\spermutation. +D 2014-03-05T11:22:33.187 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -404,7 +404,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 9d8186f6f8751efdfd445d8546fd98f073499039 -F test/corruptI.test d9eca60cb373215d97e0994bf104f9a37e1ac0fc +F test/corruptI.test 88886ec9cd1bdba835263566bbf60ee009c6ea09 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e072cb3ee2a03d786c08230cecc6e970de2cec5b -R 213bc20e67a019cbd9a16484715bd6d2 -U drh -Z 02c018f7f11d0867eb15887f2517d7cb +P 554501f158534f9c27fd51900589c2ab7cf300d8 +R f63b7b11b47b09d3cce73ea26cd9a6da +U dan +Z 381756cc53474accdfff4f671992e335 diff --git a/manifest.uuid b/manifest.uuid index 41b8fab845..5254bd10f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -554501f158534f9c27fd51900589c2ab7cf300d8 \ No newline at end of file +378b290da85cadc1acba081322d1f0e8d7694e17 \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index 9c9a19310b..51cf64c9be 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -14,6 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix corruptI +if {[permutation]=="mmap"} { + finish_test + return +} + # Do not use a codec for tests in this file, as the database file is # manipulated directly using tcl scripts (using the [hexio_write] command). # From b1a1c29a0798c068dfbc9e9b87fd3ccaead6ae34 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Mar 2014 12:47:55 +0000 Subject: [PATCH 050/177] Separate the column NULL value constant into a separate routine for greater commonality with the sessions branch. FossilOrigin-Name: 12cbebb997705e37769460e00a4aaa52c12f305e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 47 +++++++++++++++++++++++++---------------------- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index cc4e004388..8be6959cd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sredundant\smemType\sfield\sfrom\sstruct\sMem. -D 2014-03-05T11:48:35.825 +C Separate\sthe\scolumn\sNULL\svalue\sconstant\sinto\sa\sseparate\sroutine\sfor\sgreater\ncommonality\swith\sthe\ssessions\sbranch. +D 2014-03-05T12:47:55.519 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -280,7 +280,7 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 8c6fc7bb9f2218c0e43f24d847e596effa8671e2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b -F src/vdbeapi.c a5c48f2b015679ddb7c25ae866abf5e2d6e42857 +F src/vdbeapi.c 77fbd57c0340a7d548a61b666143637fff726e06 F src/vdbeaux.c 43bee29ac866f7ce6af90c4f084bb22c160b8b70 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 378b290da85cadc1acba081322d1f0e8d7694e17 bac2820e13a79d91d0f8938c643134a9d6900327 -R 0b493f27b646031be527e01b67828619 -U dan -Z 2c30198d39dfe9f58d28fb7d11ffc819 +P 9e8528578966e4f1a16d63333de648fd8cdaf0f2 +R cd61be35c414ec0c232f9ea3f0efe193 +U drh +Z d8b63809d3f1784ecbab68dab389a740 diff --git a/manifest.uuid b/manifest.uuid index a974d4a9a0..ad874928a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e8528578966e4f1a16d63333de648fd8cdaf0f2 \ No newline at end of file +12cbebb997705e37769460e00a4aaa52c12f305e \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 4c9d4870d0..dc11540deb 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -727,6 +727,30 @@ int sqlite3_data_count(sqlite3_stmt *pStmt){ return pVm->nResColumn; } +/* +** Return a pointer to static memory containing an SQL NULL value. +*/ +static const Mem *columnNullValue(void){ + /* Even though the Mem structure contains an element + ** of type i64, on certain architectures (x86) with certain compiler + ** switches (-Os), gcc may align this Mem object on a 4-byte boundary + ** instead of an 8-byte one. This all works fine, except that when + ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s + ** that a Mem structure is located on an 8-byte boundary. To prevent + ** these assert()s from failing, when building with SQLITE_DEBUG defined + ** using gcc, we force nullMem to be 8-byte aligned using the magical + ** __attribute__((aligned(8))) macro. */ + static const Mem nullMem +#if defined(SQLITE_DEBUG) && defined(__GNUC__) + __attribute__((aligned(8))) +#endif + = {0, "", (double)0, {0}, 0, MEM_Null, 0, +#ifdef SQLITE_DEBUG + 0, 0, /* pScopyFrom, pFiller */ +#endif + 0, 0 }; + return &nullMem; +} /* ** Check to see if column iCol of the given statement is valid. If @@ -743,32 +767,11 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ sqlite3_mutex_enter(pVm->db->mutex); pOut = &pVm->pResultSet[i]; }else{ - /* If the value passed as the second argument is out of range, return - ** a pointer to the following static Mem object which contains the - ** value SQL NULL. Even though the Mem structure contains an element - ** of type i64, on certain architectures (x86) with certain compiler - ** switches (-Os), gcc may align this Mem object on a 4-byte boundary - ** instead of an 8-byte one. This all works fine, except that when - ** running with SQLITE_DEBUG defined the SQLite code sometimes assert()s - ** that a Mem structure is located on an 8-byte boundary. To prevent - ** these assert()s from failing, when building with SQLITE_DEBUG defined - ** using gcc, we force nullMem to be 8-byte aligned using the magical - ** __attribute__((aligned(8))) macro. */ - static const Mem nullMem -#if defined(SQLITE_DEBUG) && defined(__GNUC__) - __attribute__((aligned(8))) -#endif - = {0, "", (double)0, {0}, 0, MEM_Null, 0, -#ifdef SQLITE_DEBUG - 0, 0, /* pScopyFrom, pFiller */ -#endif - 0, 0 }; - if( pVm && ALWAYS(pVm->db) ){ sqlite3_mutex_enter(pVm->db->mutex); sqlite3Error(pVm->db, SQLITE_RANGE, 0); } - pOut = (Mem*)&nullMem; + pOut = (Mem*)columnNullValue(); } return pOut; } From 817492630a59c9ec4b491de15b68cd0d90106300 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Mar 2014 14:40:22 +0000 Subject: [PATCH 051/177] When converting a result type from TEXT to BLOB using the sqlite3_value_blob() interface, continue to report SQLITE_TEXT as the true type from sqlite3_value_text() as long as that text is still valid. The maintains legacy behavior from before the noMemType change. FossilOrigin-Name: 1d134ba2edbdb8c0cf9e99590a69cd17e0b874a9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8be6959cd4..89fddcc445 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Separate\sthe\scolumn\sNULL\svalue\sconstant\sinto\sa\sseparate\sroutine\sfor\sgreater\ncommonality\swith\sthe\ssessions\sbranch. -D 2014-03-05T12:47:55.519 +C When\sconverting\sa\sresult\stype\sfrom\sTEXT\sto\sBLOB\susing\sthe\nsqlite3_value_blob()\sinterface,\scontinue\sto\sreport\sSQLITE_TEXT\sas\nthe\strue\stype\sfrom\ssqlite3_value_text()\sas\slong\sas\sthat\stext\sis\sstill\nvalid.\s\sThe\smaintains\slegacy\sbehavior\sfrom\sbefore\sthe\snoMemType\schange. +D 2014-03-05T14:40:22.888 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -280,7 +280,7 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 8c6fc7bb9f2218c0e43f24d847e596effa8671e2 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b -F src/vdbeapi.c 77fbd57c0340a7d548a61b666143637fff726e06 +F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 43bee29ac866f7ce6af90c4f084bb22c160b8b70 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9e8528578966e4f1a16d63333de648fd8cdaf0f2 -R cd61be35c414ec0c232f9ea3f0efe193 +P 12cbebb997705e37769460e00a4aaa52c12f305e +R d261f37d595757cff592b0cc63c2909e U drh -Z d8b63809d3f1784ecbab68dab389a740 +Z 2ff0e0e9e50bbeda593af44f5801ea80 diff --git a/manifest.uuid b/manifest.uuid index ad874928a1..dbbf09faa4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12cbebb997705e37769460e00a4aaa52c12f305e \ No newline at end of file +1d134ba2edbdb8c0cf9e99590a69cd17e0b874a9 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index dc11540deb..5d7a0b0608 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -135,7 +135,6 @@ const void *sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ sqlite3VdbeMemExpandBlob(p); - p->flags &= ~MEM_Str; p->flags |= MEM_Blob; return p->n ? p->z : 0; }else{ From 14965665602dfea528cd712b0ed5073de06a00d8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Mar 2014 15:04:36 +0000 Subject: [PATCH 052/177] Add a test case for the problem fixed by [1d134ba2ed]. FossilOrigin-Name: d4fbc05df9501a09623d420fcd5b2bce3a57201b --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/capi3.test | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 89fddcc445..d10be5fe63 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sconverting\sa\sresult\stype\sfrom\sTEXT\sto\sBLOB\susing\sthe\nsqlite3_value_blob()\sinterface,\scontinue\sto\sreport\sSQLITE_TEXT\sas\nthe\strue\stype\sfrom\ssqlite3_value_text()\sas\slong\sas\sthat\stext\sis\sstill\nvalid.\s\sThe\smaintains\slegacy\sbehavior\sfrom\sbefore\sthe\snoMemType\schange. -D 2014-03-05T14:40:22.888 +C Add\sa\stest\scase\sfor\sthe\sproblem\sfixed\sby\s[1d134ba2ed]. +D 2014-03-05T15:04:36.099 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -361,7 +361,7 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 6cdd49656bd62a296924f4d2fcfd05cd2a298369 +F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 F test/capi3d.test 6d0fc0a86d73f42dd19a7d8b7761ab9bc02277d0 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 12cbebb997705e37769460e00a4aaa52c12f305e -R d261f37d595757cff592b0cc63c2909e -U drh -Z 2ff0e0e9e50bbeda593af44f5801ea80 +P 1d134ba2edbdb8c0cf9e99590a69cd17e0b874a9 +R 374319ceadfe35c573eb8d6fe8e86cec +U dan +Z 517538463510fd631bac600f6056aa59 diff --git a/manifest.uuid b/manifest.uuid index dbbf09faa4..621c82a8e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d134ba2edbdb8c0cf9e99590a69cd17e0b874a9 \ No newline at end of file +d4fbc05df9501a09623d420fcd5b2bce3a57201b \ No newline at end of file diff --git a/test/capi3.test b/test/capi3.test index f0234e8897..cbaa5c5e5b 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix capi3 # Do not use a codec for tests in this file, as the database file is # manipulated directly using tcl scripts (using the [hexio_write] command). @@ -1221,6 +1222,23 @@ do_test capi3-19.1 { sqlite3_prepare_tkt3134 db } {} +# Test that calling sqlite3_column_blob() on a TEXT value does not change +# the return type of subsequent calls to sqlite3_column_type(). +# +do_execsql_test 20.1 { + CREATE TABLE t4(x); + INSERT INTO t4 VALUES('abcdefghij'); +} +do_test 20.2 { + set stmt [sqlite3_prepare db "SELECT * FROM t4" -1 dummy] + sqlite3_step $stmt +} {SQLITE_ROW} +do_test 20.3 { sqlite3_column_type $stmt 0 } {TEXT} +do_test 20.4 { sqlite3_column_blob $stmt 0 } {abcdefghij} +do_test 20.5 { sqlite3_column_type $stmt 0 } {TEXT} +do_test 20.6 { sqlite3_finalize $stmt } SQLITE_OK + + # Tests of the interface when no VFS is registered. # if {![info exists tester_do_binarylog]} { From 6d1626eb7ccd6dd78c13c2f67bdc624a93278a3b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Mar 2014 15:52:43 +0000 Subject: [PATCH 053/177] Count the number of elements in a SrcList object using an "int" rather than a "u8", to avoid overflows and to work around an issue in the C compiler on AIX. FossilOrigin-Name: eee2a13f2caa48b7b8a693680edc2bbc9275292c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 4 ++-- src/sqliteInt.h | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d10be5fe63..f61d243cff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sthe\sproblem\sfixed\sby\s[1d134ba2ed]. -D 2014-03-05T15:04:36.099 +C Count\sthe\snumber\sof\selements\sin\sa\sSrcList\sobject\susing\san\s"int"\srather\sthan\na\s"u8",\sto\savoid\soverflows\sand\sto\swork\saround\san\sissue\sin\sthe\sC\scompiler\son\nAIX. +D 2014-03-05T15:52:43.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c ad795be588e394a0921796f87cc27717359738cc F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c 00ce613bc2256e525c9195cb10d0df7bcc48d1f0 +F src/build.c 04acd702f9c3ffd6670cf50be89d3561cb886fe6 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -221,7 +221,7 @@ F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bf529055591dd81ec92a84d35dba0854642f6cc2 +F src/sqliteInt.h 7497e95a42f3ec87bf524f1b7231836ac69ed5b0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1d134ba2edbdb8c0cf9e99590a69cd17e0b874a9 -R 374319ceadfe35c573eb8d6fe8e86cec -U dan -Z 517538463510fd631bac600f6056aa59 +P d4fbc05df9501a09623d420fcd5b2bce3a57201b +R 9cc9aba61da37a98f68738296ca8fbec +U drh +Z ce6864cda23669740d5f0005c13c1ece diff --git a/manifest.uuid b/manifest.uuid index 621c82a8e9..65508caffb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4fbc05df9501a09623d420fcd5b2bce3a57201b \ No newline at end of file +eee2a13f2caa48b7b8a693680edc2bbc9275292c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 5b6c87f220..44767fa5a1 100644 --- a/src/build.c +++ b/src/build.c @@ -3485,7 +3485,7 @@ SrcList *sqlite3SrcListEnlarge( } pSrc = pNew; nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = (u8)nGot; + pSrc->nAlloc = nGot; } /* Move existing slots that come after the newly inserted slots @@ -3493,7 +3493,7 @@ SrcList *sqlite3SrcListEnlarge( for(i=pSrc->nSrc-1; i>=iStart; i--){ pSrc->a[i+nExtra] = pSrc->a[i]; } - pSrc->nSrc += (i8)nExtra; + pSrc->nSrc += nExtra; /* Zero the newly allocated slots */ memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 90e83206ad..cdead4bd2f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2018,8 +2018,8 @@ typedef u64 Bitmask; ** contains more than 63 columns and the 64-th or later column is used. */ struct SrcList { - u8 nSrc; /* Number of tables or subqueries in the FROM clause */ - u8 nAlloc; /* Number of entries allocated in a[] below */ + int nSrc; /* Number of tables or subqueries in the FROM clause */ + u32 nAlloc; /* Number of entries allocated in a[] below */ struct SrcList_item { Schema *pSchema; /* Schema to which this item is fixed */ char *zDatabase; /* Name of database holding this table */ From 13ef14afc1f8e97734cee3a1af8dcb571016a830 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Mar 2014 16:15:07 +0000 Subject: [PATCH 054/177] Add tests to join.test to verify that it is not possible to use more than 64 tables in a join. FossilOrigin-Name: ef1a998625e1fc051732228844388d6c59665382 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/join.test | 26 ++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f61d243cff..324e9afc4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Count\sthe\snumber\sof\selements\sin\sa\sSrcList\sobject\susing\san\s"int"\srather\sthan\na\s"u8",\sto\savoid\soverflows\sand\sto\swork\saround\san\sissue\sin\sthe\sC\scompiler\son\nAIX. -D 2014-03-05T15:52:43.396 +C Add\stests\sto\sjoin.test\sto\sverify\sthat\sit\sis\snot\spossible\sto\suse\smore\sthan\s64\stables\sin\sa\sjoin. +D 2014-03-05T16:15:07.836 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -630,7 +630,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/join.test 8d63cc4d230a7affafa4b6ab0b97c49b8ccb365c +F test/join.test 533dc4c1ca70d1c876a58df058307ab9c6931a05 F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d4fbc05df9501a09623d420fcd5b2bce3a57201b -R 9cc9aba61da37a98f68738296ca8fbec -U drh -Z ce6864cda23669740d5f0005c13c1ece +P eee2a13f2caa48b7b8a693680edc2bbc9275292c +R e372b7c4ce133454234e558c8bea46b7 +U dan +Z bf5a892fb8860fb651f9bf5f3493984f diff --git a/manifest.uuid b/manifest.uuid index 65508caffb..5b0b84a8d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eee2a13f2caa48b7b8a693680edc2bbc9275292c \ No newline at end of file +ef1a998625e1fc051732228844388d6c59665382 \ No newline at end of file diff --git a/test/join.test b/test/join.test index 88ac04f950..d25fa35115 100644 --- a/test/join.test +++ b/test/join.test @@ -641,4 +641,30 @@ do_test join-11.10 { execsql { SELECT * FROM t2 NATURAL JOIN t1 } } {1 one 2 two} +#------------------------------------------------------------------------- +# Test that at most 64 tables are allowed in a join. +# +do_execsql_test join-12.1 { + CREATE TABLE t14(x); + INSERT INTO t14 VALUES('abcdefghij'); +} + +proc jointest {tn nTbl res} { + set sql "SELECT 1 FROM [string repeat t14, [expr $nTbl-1]] t14;" + uplevel [list do_catchsql_test $tn $sql $res] +} + +jointest join-12.2 30 {0 1} +jointest join-12.3 63 {0 1} +jointest join-12.4 64 {0 1} +jointest join-12.5 65 {1 {at most 64 tables in a join}} +jointest join-12.6 66 {1 {at most 64 tables in a join}} +jointest join-12.7 127 {1 {at most 64 tables in a join}} +jointest join-12.8 128 {1 {at most 64 tables in a join}} +jointest join-12.9 1000 {1 {at most 64 tables in a join}} +jointest join-12.10 65534 {1 {at most 64 tables in a join}} +jointest join-12.11 65535 {1 {too many references to "t14": max 65535}} +jointest join-12.12 65536 {1 {too many references to "t14": max 65535}} +jointest join-12.13 65537 {1 {too many references to "t14": max 65535}} + finish_test From 03af2168db869a779d89001f141e6e1d68cda695 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Mar 2014 17:29:34 +0000 Subject: [PATCH 055/177] Add SQLITE_OMIT_CTE to the list of compile options that might be returned by "PRAGMA compile_options". FossilOrigin-Name: f33f2b3f924347e3da26741d153749330cb645ec --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/ctime.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 324e9afc4e..5674527ca1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\sjoin.test\sto\sverify\sthat\sit\sis\snot\spossible\sto\suse\smore\sthan\s64\stables\sin\sa\sjoin. -D 2014-03-05T16:15:07.836 +C Add\sSQLITE_OMIT_CTE\sto\sthe\slist\sof\scompile\soptions\sthat\smight\sbe\sreturned\sby\s"PRAGMA\scompile_options". +D 2014-03-05T17:29:34.044 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 04acd702f9c3ffd6670cf50be89d3561cb886fe6 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd +F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eee2a13f2caa48b7b8a693680edc2bbc9275292c -R e372b7c4ce133454234e558c8bea46b7 +P ef1a998625e1fc051732228844388d6c59665382 +R 466a7e3f69162573de9bfcfa517638a1 U dan -Z bf5a892fb8860fb651f9bf5f3493984f +Z 7d97bf825e792d4219e6e7e5c97b5b1b diff --git a/manifest.uuid b/manifest.uuid index 5b0b84a8d1..9df8f288b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef1a998625e1fc051732228844388d6c59665382 \ No newline at end of file +f33f2b3f924347e3da26741d153749330cb645ec \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index c863cbed50..286f66e061 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -215,6 +215,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_COMPOUND_SELECT "OMIT_COMPOUND_SELECT", #endif +#ifdef SQLITE_OMIT_CTE + "OMIT_CTE", +#endif #ifdef SQLITE_OMIT_DATETIME_FUNCS "OMIT_DATETIME_FUNCS", #endif From fc5717cc99b61237fcc56087b99d4b5d1b4c4fb7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Mar 2014 19:04:46 +0000 Subject: [PATCH 056/177] Fix a harmless compiler warning on MSVC. FossilOrigin-Name: 0723effc9ccae7c660fb847b36ce9324e0cb5042 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5674527ca1..ab256e6e6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_OMIT_CTE\sto\sthe\slist\sof\scompile\soptions\sthat\smight\sbe\sreturned\sby\s"PRAGMA\scompile_options". -D 2014-03-05T17:29:34.044 +C Fix\sa\sharmless\scompiler\swarning\son\sMSVC. +D 2014-03-05T19:04:46.223 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c ad795be588e394a0921796f87cc27717359738cc F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c 04acd702f9c3ffd6670cf50be89d3561cb886fe6 +F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ef1a998625e1fc051732228844388d6c59665382 -R 466a7e3f69162573de9bfcfa517638a1 -U dan -Z 7d97bf825e792d4219e6e7e5c97b5b1b +P f33f2b3f924347e3da26741d153749330cb645ec +R bec805b30a62102ab84493bc76693952 +U drh +Z 267f872091e9a8fb36e5f21325487fbe diff --git a/manifest.uuid b/manifest.uuid index 9df8f288b0..8aca077064 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f33f2b3f924347e3da26741d153749330cb645ec \ No newline at end of file +0723effc9ccae7c660fb847b36ce9324e0cb5042 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 44767fa5a1..10077e5018 100644 --- a/src/build.c +++ b/src/build.c @@ -3473,7 +3473,7 @@ SrcList *sqlite3SrcListEnlarge( assert( iStart<=pSrc->nSrc ); /* Allocate additional space if needed */ - if( pSrc->nSrc+nExtra>pSrc->nAlloc ){ + if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; int nAlloc = pSrc->nSrc+nExtra; int nGot; From a6eaa635cc1cc1d00619fc51aed9ce585c2a68b5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Mar 2014 19:13:32 +0000 Subject: [PATCH 057/177] Avoid running a couple of test cases that use realloc() extensively if SQLITE_MEMDEBUG is defined. FossilOrigin-Name: e406d4bb786a34cdd5dd23ed39f35dfce3e1cabd --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/join.test | 17 +++++++++++++---- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ab256e6e6d..9c71992df0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning\son\sMSVC. -D 2014-03-05T19:04:46.223 +C Avoid\srunning\sa\scouple\sof\stest\scases\sthat\suse\srealloc()\sextensively\sif\sSQLITE_MEMDEBUG\sis\sdefined. +D 2014-03-05T19:13:32.745 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -630,7 +630,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/join.test 533dc4c1ca70d1c876a58df058307ab9c6931a05 +F test/join.test 559b81eb56ae350246f9c25986aa52c81c725c7e F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f33f2b3f924347e3da26741d153749330cb645ec -R bec805b30a62102ab84493bc76693952 -U drh -Z 267f872091e9a8fb36e5f21325487fbe +P 0723effc9ccae7c660fb847b36ce9324e0cb5042 +R fd55c551a93656dfa35ae57a4e3f4bce +U dan +Z 8fe6d9b1a050fe779092fc8bd960685b diff --git a/manifest.uuid b/manifest.uuid index 8aca077064..94a1773c6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0723effc9ccae7c660fb847b36ce9324e0cb5042 \ No newline at end of file +e406d4bb786a34cdd5dd23ed39f35dfce3e1cabd \ No newline at end of file diff --git a/test/join.test b/test/join.test index d25fa35115..28d9ddf8b2 100644 --- a/test/join.test +++ b/test/join.test @@ -662,9 +662,18 @@ jointest join-12.6 66 {1 {at most 64 tables in a join}} jointest join-12.7 127 {1 {at most 64 tables in a join}} jointest join-12.8 128 {1 {at most 64 tables in a join}} jointest join-12.9 1000 {1 {at most 64 tables in a join}} -jointest join-12.10 65534 {1 {at most 64 tables in a join}} -jointest join-12.11 65535 {1 {too many references to "t14": max 65535}} -jointest join-12.12 65536 {1 {too many references to "t14": max 65535}} -jointest join-12.13 65537 {1 {too many references to "t14": max 65535}} + +# If SQLite is built with SQLITE_MEMDEBUG, then the huge number of realloc() +# calls made by the following test cases are too time consuming to run. +# Without SQLITE_MEMDEBUG, realloc() is fast enough that these are not +# a problem. +ifcapable pragma&&compileoption_diags { + if {[lsearch [db eval {PRAGMA compile_options}] MEMDEBUG]<0} { + jointest join-12.10 65534 {1 {at most 64 tables in a join}} + jointest join-12.11 65535 {1 {too many references to "t14": max 65535}} + jointest join-12.12 65536 {1 {too many references to "t14": max 65535}} + jointest join-12.13 65537 {1 {too many references to "t14": max 65535}} + } +} finish_test From dc90d3d8af0c560869ce2dd1ed576ff6745ac58c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Mar 2014 19:36:52 +0000 Subject: [PATCH 058/177] Always include the ctype.h header in the spellfix.c extension. FossilOrigin-Name: bfd75f471a607b442d8cd8a7070491a158e8fb1a --- ext/misc/spellfix.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index d33740bf6d..a62d24c592 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -26,8 +26,8 @@ SQLITE_EXTENSION_INIT1 # define NEVER(X) 0 typedef unsigned char u8; typedef unsigned short u16; -# include #endif +#include #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/manifest b/manifest index 9c71992df0..c073f282a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\srunning\sa\scouple\sof\stest\scases\sthat\suse\srealloc()\sextensively\sif\sSQLITE_MEMDEBUG\sis\sdefined. -D 2014-03-05T19:13:32.745 +C Always\sinclude\sthe\sctype.h\sheader\sin\sthe\sspellfix.c\sextension. +D 2014-03-05T19:36:52.892 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c 3548c433f473c2054e080b6382771636fcaa2c4c +F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0723effc9ccae7c660fb847b36ce9324e0cb5042 -R fd55c551a93656dfa35ae57a4e3f4bce -U dan -Z 8fe6d9b1a050fe779092fc8bd960685b +P e406d4bb786a34cdd5dd23ed39f35dfce3e1cabd +R dbf6df7a91f37410ffccbad9151ed0f2 +U drh +Z 435f844ac1ec54f913d9ebdd6e1ebde0 diff --git a/manifest.uuid b/manifest.uuid index 94a1773c6e..96b73158b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e406d4bb786a34cdd5dd23ed39f35dfce3e1cabd \ No newline at end of file +bfd75f471a607b442d8cd8a7070491a158e8fb1a \ No newline at end of file From b6e8fd105c0923bde373188d0251d2440888ac8f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Mar 2014 01:56:33 +0000 Subject: [PATCH 059/177] Fix some incorrect comments and typos in comments. Add testcase() macros to some of the new record comparison code. FossilOrigin-Name: b83cfe899d84fe9d61540e9984321ca30401638c --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/btree.c | 2 +- src/insert.c | 9 ++++----- src/vdbe.c | 6 +++--- src/vdbeaux.c | 37 ++++++++++++++++++++++++++++++++++--- 6 files changed, 53 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 9ebba25618..0317e45b49 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Better\sloadable\sextension\shandling\sin\sthe\sWin32\sVFS\swhen\scompiled\sfor\sCygwin. -D 2014-03-06T00:30:27.980 +C Fix\ssome\sincorrect\scomments\sand\stypos\sin\scomments.\s\sAdd\stestcase()\smacros\nto\ssome\sof\sthe\snew\srecord\scomparison\scode. +D 2014-03-06T01:56:33.418 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ad795be588e394a0921796f87cc27717359738cc +F src/btree.c 4d28fb15543f0e071b1780b2af8cd2ee489de32d F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -180,7 +180,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d3d1164299faa53b5347f8ff676d3787537fa9e5 +F src/insert.c d8bb30535c8c0785876025a4a07f9074640a15d1 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -277,11 +277,11 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 8c6fc7bb9f2218c0e43f24d847e596effa8671e2 +F src/vdbe.c b2ca9b8e3a47a3bcecdb2b551279a2ba750a19e5 F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 43bee29ac866f7ce6af90c4f084bb22c160b8b70 +F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 170e0585202eec6882e9513eba05e6d5767bfc5b 854d410ad3a2bb3af5465d933ac9535fe3f22bfe -R dbc36dfda03a0ba225e8ff052c2c6fea -U mistachkin -Z 97b3517e69e758ff54d2564329fd6e38 +P 29b0a4f158785449b6f3da6fcceeb63442c9711c +R 6ebc5b98243298bdc1108377764c8e0d +U drh +Z 431ba6c46f093bf18085ee790e19125c diff --git a/manifest.uuid b/manifest.uuid index 41cd05f2ef..d5463c1070 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29b0a4f158785449b6f3da6fcceeb63442c9711c \ No newline at end of file +b83cfe899d84fe9d61540e9984321ca30401638c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index eebf6f1e89..29ead1c67a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4576,7 +4576,7 @@ int sqlite3BtreeMovetoUnpacked( || pIdxKey->default_rc==-1 ); }else{ - xRecordCompare = 0; /* Not actually used. Avoids a compiler warning. */ + xRecordCompare = 0; /* All keys are integers */ } rc = moveToRoot(pCur); diff --git a/src/insert.c b/src/insert.c index 1d29da3092..abdf1ada95 100644 --- a/src/insert.c +++ b/src/insert.c @@ -101,13 +101,13 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** Compute the affinity string for table pTab, if it has not already been ** computed. As an optimization, omit trailing SQLITE_AFF_NONE affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values and +** If the affinity exists (if it is no entirely SQLITE_AFF_NONE values) and ** if iReg>0 then code an OP_Affinity opcode that will set the affinities ** for register iReg and following. Or if affinities exists and iReg==0, ** then just set the P4 operand of the previous opcode (which should be ** an OP_MakeRecord) to the affinity string. ** -** A column affinity string has one character column: +** A column affinity string has one character per column: ** ** Character Column affinity ** ------------------------------ @@ -148,10 +148,9 @@ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ /* ** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program beginning at location -** iStartAddr throught the end of the program. This is used to see if +** have been opened at any point in the VDBE program. This is used to see if ** a statement of the form "INSERT INTO SELECT ..." can -** run without using temporary table for the results of the SELECT. +** run without using a temporary table for the results of the SELECT. */ static int readsTable(Parse *p, int iDb, Table *pTab){ Vdbe *v = sqlite3GetVdbe(p); diff --git a/src/vdbe.c b/src/vdbe.c index b9d49e3620..4c36e12090 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -28,7 +28,7 @@ ** pointer to the string or blob, not the content. If the original ** is changed while the copy is still in use, the string or blob might ** be changed out from under the copy. This macro verifies that nothing -** like that every happens. +** like that ever happens. */ #ifdef SQLITE_DEBUG # define memAboutToChange(P,M) sqlite3VdbeMemAboutToChange(P,M) @@ -2166,8 +2166,8 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ ** ** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, ** set the flag and fall through to the next instruction. In other words, -** this opcode causes all following up codes up through P2 (but not including -** P2) to run just once and skipped on subsequent times through the loop. +** this opcode causes all following opcodes up through P2 (but not including +** P2) to run just once and to be skipped on subsequent times through the loop. */ case OP_Once: { /* jump */ assert( pOp->p1nOnceFlag ); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 00ec901588..a641a72ec8 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1234,6 +1234,10 @@ static void releaseMemArray(Mem *p, int N){ ** with no indexes using a single prepared INSERT statement, bind() ** and reset(). Inserts are grouped into a transaction. */ + testcase( p->flags & MEM_Agg ); + testcase( p->flags & MEM_Dyn ); + testcase( p->flags & MEM_Frame ); + testcase( p->flags & MEM_RowSet ); if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ sqlite3VdbeMemRelease(p); }else if( p->zMalloc ){ @@ -2962,27 +2966,32 @@ u32 sqlite3VdbeSerialGet( case 1: { /* 1-byte signed integer */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ y = FOUR_BYTE_UINT(buf); pMem->u.i = (i64)*(int*)&y; pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); return 6; } case 6: /* 8-byte signed integer */ @@ -3005,6 +3014,7 @@ u32 sqlite3VdbeSerialGet( if( serial_type==6 ){ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); }else{ assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); swapMixedEndianFloat(x); @@ -3350,20 +3360,26 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ switch( serial_type ){ case 0: case 1: + testcase( aKey[0]&0x80 ); return ONE_BYTE_INT(aKey); case 2: + testcase( aKey[0]&0x80 ); return TWO_BYTE_INT(aKey); case 3: + testcase( aKey[0]&0x80 ); return THREE_BYTE_INT(aKey); case 4: { + testcase( aKey[0]&0x80 ); y = FOUR_BYTE_UINT(aKey); return (i64)*(int*)&y; } case 5: { + testcase( aKey[0]&0x80 ); return FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); } case 6: { u64 x = FOUR_BYTE_UINT(aKey); + testcase( aKey[0]&0x80 ); x = (x<<32) | FOUR_BYTE_UINT(aKey+4); return (i64)*(i64*)&x; } @@ -3431,6 +3447,7 @@ int sqlite3VdbeRecordCompare( /* RHS is an integer */ if( pRhs->flags & MEM_Int ){ serial_type = aKey1[idx1]; + testcase( serial_type==12 ); if( serial_type>=12 ){ rc = +1; }else if( serial_type==0 ){ @@ -3481,12 +3498,15 @@ int sqlite3VdbeRecordCompare( /* RHS is a string */ else if( pRhs->flags & MEM_Str ){ getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); if( serial_type<12 ){ rc = -1; }else if( !(serial_type & 0x01) ){ rc = +1; }else{ mem1.n = (serial_type - 12) / 2; + testcase( (d1+mem1.n)==(unsigned)nKey1 ); + testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ rc = 1; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ @@ -3506,10 +3526,13 @@ int sqlite3VdbeRecordCompare( /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ getVarint32(&aKey1[idx1], serial_type); + testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ rc = -1; }else{ int nStr = (serial_type - 12) / 2; + testcase( (d1+nStr)==(unsigned)nKey1 ); + testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ rc = 1; /* Corruption */ }else{ @@ -3583,29 +3606,35 @@ static int vdbeRecordCompareInt( switch( serial_type ){ case 1: { /* 1-byte signed integer */ lhs = ONE_BYTE_INT(aKey); + testcase( lhs<0 ); break; } case 2: { /* 2-byte signed integer */ lhs = TWO_BYTE_INT(aKey); + testcase( lhs<0 ); break; } case 3: { /* 3-byte signed integer */ lhs = THREE_BYTE_INT(aKey); + testcase( lhs<0 ); break; } case 4: { /* 4-byte signed integer */ y = FOUR_BYTE_UINT(aKey); lhs = (i64)*(int*)&y; + testcase( lhs<0 ); break; } case 5: { /* 6-byte signed integer */ lhs = FOUR_BYTE_UINT(aKey+2) + (((i64)1)<<32)*TWO_BYTE_INT(aKey); + testcase( lhs<0 ); break; } case 6: { /* 8-byte signed integer */ x = FOUR_BYTE_UINT(aKey); x = (x<<32) | FOUR_BYTE_UINT(aKey+4); lhs = *(i64*)&x; + testcase( lhs<0 ); break; } case 8: @@ -3742,9 +3771,11 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ if( (flags & MEM_Int) ){ return vdbeRecordCompareInt; } - if( (flags & (MEM_Int|MEM_Real|MEM_Null|MEM_Blob))==0 - && p->pKeyInfo->aColl[0]==0 - ){ + testcase( flags & MEM_Real ); + testcase( flags & MEM_Null ); + testcase( flags & MEM_Blob ); + if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + assert( flags & MEM_Str ); return vdbeRecordCompareString; } } From 378d01aa949d5b49b9f31e91909d4fe5ba2b598b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 6 Mar 2014 02:15:42 +0000 Subject: [PATCH 060/177] Cosmetic change to the shell in-memory database warning to skip coloring the trailing period. FossilOrigin-Name: c87d8e07ec596d5f0ef3003cfeb226d4b7c49549 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0317e45b49..d66f7070db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sincorrect\scomments\sand\stypos\sin\scomments.\s\sAdd\stestcase()\smacros\nto\ssome\sof\sthe\snew\srecord\scomparison\scode. -D 2014-03-06T01:56:33.418 +C Cosmetic\schange\sto\sthe\sshell\sin-memory\sdatabase\swarning\sto\sskip\scoloring\sthe\strailing\speriod. +D 2014-03-06T02:15:42.988 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 -F src/shell.c 7bf07bcacb181ecc3fc3ccacfdfeb4084aee67ed +F src/shell.c c0cbbbe5eec0923caafd1b9593925f8b241addb8 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 29b0a4f158785449b6f3da6fcceeb63442c9711c -R 6ebc5b98243298bdc1108377764c8e0d -U drh -Z 431ba6c46f093bf18085ee790e19125c +P b83cfe899d84fe9d61540e9984321ca30401638c +R a229689d692fc13174e0ca3780072f0f +U mistachkin +Z 057b46befe5b696e1f097c5fbeb7d34a diff --git a/manifest.uuid b/manifest.uuid index d5463c1070..adad07e5fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b83cfe899d84fe9d61540e9984321ca30401638c \ No newline at end of file +c87d8e07ec596d5f0ef3003cfeb226d4b7c49549 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7ecedfd3c1..f3f80bfd71 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3817,8 +3817,8 @@ int main(int argc, char **argv){ ); if( warnInmemoryDb ){ printf("Connected to a "); - printBold("transient in-memory database."); - printf("\nUse \".open FILENAME\" to reopen on a " + printBold("transient in-memory database"); + printf(".\nUse \".open FILENAME\" to reopen on a " "persistent database.\n"); } zHome = find_home_dir(); From 0df163a9f06f402bc985b6d53fe1257245b12b01 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Mar 2014 12:36:26 +0000 Subject: [PATCH 061/177] Remove a branch that is never taken from where.c. FossilOrigin-Name: 4a4997221f3d61c1411a993053b40c787beea736 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d66f7070db..8bc91eb7e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cosmetic\schange\sto\sthe\sshell\sin-memory\sdatabase\swarning\sto\sskip\scoloring\sthe\strailing\speriod. -D 2014-03-06T02:15:42.988 +C Remove\sa\sbranch\sthat\sis\snever\staken\sfrom\swhere.c. +D 2014-03-06T12:36:26.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2269342f82ea97596564bdc981c362a5177b72c5 +F src/where.c 95d30485879122acc5284d4523e01399b23b6148 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b83cfe899d84fe9d61540e9984321ca30401638c -R a229689d692fc13174e0ca3780072f0f -U mistachkin -Z 057b46befe5b696e1f097c5fbeb7d34a +P c87d8e07ec596d5f0ef3003cfeb226d4b7c49549 +R 8a41ab31fe82e4eb8dd3f6f960d87359 +U dan +Z 9504b4a4d2685ecc17b4e7b160b2fd5f diff --git a/manifest.uuid b/manifest.uuid index adad07e5fd..0a317e3270 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c87d8e07ec596d5f0ef3003cfeb226d4b7c49549 \ No newline at end of file +4a4997221f3d61c1411a993053b40c787beea736 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c29c94ae0a..963878d009 100644 --- a/src/where.c +++ b/src/where.c @@ -3057,13 +3057,13 @@ static Bitmask codeOneLoopStart( pRangeEnd = pLoop->aLTerm[j++]; nExtraReg = 1; if( pRangeStart==0 - && (pRangeEnd->wtFlags & TERM_VNULL)==0 && (j = pIdx->aiColumn[nEq])>=0 && pIdx->pTable->aCol[j].notNull==0 ){ bSeekPastNull = 1; } } + assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers From fe70510aa757c4579e9943052f828a3cad5d6d74 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Mar 2014 13:38:37 +0000 Subject: [PATCH 062/177] Improved EXPLAIN indentation of a loop in the ANALYZE logic for STAT4. Mark the not-found jump of a seek operation in that loop as never taken. FossilOrigin-Name: 0a4200f95cf46ad620b9fd91f4444114a0c74730 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 7 +++++-- src/shell.c | 5 ++++- src/vdbe.c | 5 +++++ 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 8bc91eb7e3..c1695f1700 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sbranch\sthat\sis\snever\staken\sfrom\swhere.c. -D 2014-03-06T12:36:26.240 +C Improved\sEXPLAIN\sindentation\sof\sa\sloop\sin\sthe\sANALYZE\slogic\sfor\sSTAT4.\nMark\sthe\snot-found\sjump\sof\sa\sseek\soperation\sin\sthat\sloop\sas\snever\staken. +D 2014-03-06T13:38:37.861 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 5d99edbac5bc416032772b723ee30182ee6e5de0 -F src/analyze.c 69761e1677142d180a9f55250dee2952f45e4793 +F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -217,7 +217,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 -F src/shell.c c0cbbbe5eec0923caafd1b9593925f8b241addb8 +F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -277,7 +277,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b2ca9b8e3a47a3bcecdb2b551279a2ba750a19e5 +F src/vdbe.c 5f0fffa9bf49a90c05dc3d46d8217603fd0ee00e F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c87d8e07ec596d5f0ef3003cfeb226d4b7c49549 -R 8a41ab31fe82e4eb8dd3f6f960d87359 -U dan -Z 9504b4a4d2685ecc17b4e7b160b2fd5f +P 4a4997221f3d61c1411a993053b40c787beea736 +R 6c77c921c71ee968c47f10dbbbd337e5 +U drh +Z 43128f0b9615c8a6ed241332450c076a diff --git a/manifest.uuid b/manifest.uuid index 0a317e3270..de8a5a7e02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a4997221f3d61c1411a993053b40c787beea736 \ No newline at end of file +0a4200f95cf46ad620b9fd91f4444114a0c74730 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 235a2abaf8..2a03e292c8 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1178,7 +1178,10 @@ static void analyzeOneTable( callStatGet(v, regStat4, STAT_GET_NLT, regLt); callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); - VdbeCoverage(v); + /* We know that the regSampleRowid row exists because it was read by + ** the previous loop. Thus the not-found jump of seekOp will never + ** be taken */ + VdbeCoverageNeverTaken(v); #ifdef SQLITE_ENABLE_STAT3 sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pIdx->aiColumn[0], regSample); @@ -1192,7 +1195,7 @@ static void analyzeOneTable( sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeAddOp2(v, OP_Goto, 1, addrNext); /* P1==1 for end-of-loop */ sqlite3VdbeJumpHere(v, addrIsNull); } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ diff --git a/src/shell.c b/src/shell.c index f3f80bfd71..2cff102e91 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1184,6 +1184,7 @@ static int str_in_array(const char *zStr, const char **azArray){ ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: ** Yield SeekGt SeekLt RowSetRead Rewind +** or if the P1 parameter is one instead of zero, ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ @@ -1231,7 +1232,9 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ if( str_in_array(zOp, azNext) ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent && abYield[p2op] ){ + if( str_in_array(zOp, azGoto) && p2opnIndent + && (abYield[p2op] || sqlite3_column_int(pSql, 2)) + ){ for(i=p2op+1; iaiIndent[i] += 2; } } diff --git a/src/vdbe.c b/src/vdbe.c index 4c36e12090..c58b5d9fda 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -671,6 +671,11 @@ int sqlite3VdbeExec( ** The next instruction executed will be ** the one at index P2 from the beginning of ** the program. +** +** The P1 parameter is not actually used by this opcode. However, it +** is sometimes set to 1 instead of 0 as a hint to the command-line shell +** that this Goto is the bottom of a loop and that the lines from P2 down +** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ pc = pOp->p2 - 1; From f0ec2a5ea71bf981b32c61856e08917950473a1e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Mar 2014 18:16:45 +0000 Subject: [PATCH 063/177] Do not run corruptH.test in mmap mode. FossilOrigin-Name: c0d54b4e41cba84dc5934e0fcd03fe422fe5c92b --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/corruptH.test | 24 +++++++++++++----------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index c1695f1700..a96f29287e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sEXPLAIN\sindentation\sof\sa\sloop\sin\sthe\sANALYZE\slogic\sfor\sSTAT4.\nMark\sthe\snot-found\sjump\sof\sa\sseek\soperation\sin\sthat\sloop\sas\snever\staken. -D 2014-03-06T13:38:37.861 +C Do\snot\srun\scorruptH.test\sin\smmap\smode. +D 2014-03-06T18:16:45.962 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -403,7 +403,7 @@ F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d -F test/corruptH.test 9d8186f6f8751efdfd445d8546fd98f073499039 +F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test 88886ec9cd1bdba835263566bbf60ee009c6ea09 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4a4997221f3d61c1411a993053b40c787beea736 -R 6c77c921c71ee968c47f10dbbbd337e5 -U drh -Z 43128f0b9615c8a6ed241332450c076a +P 0a4200f95cf46ad620b9fd91f4444114a0c74730 +R e9933db135410760cbb03eb238614bcf +U dan +Z b325d7c68ad2fff1170ad7d6e80e5338 diff --git a/manifest.uuid b/manifest.uuid index de8a5a7e02..8adb79e711 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a4200f95cf46ad620b9fd91f4444114a0c74730 \ No newline at end of file +c0d54b4e41cba84dc5934e0fcd03fe422fe5c92b \ No newline at end of file diff --git a/test/corruptH.test b/test/corruptH.test index 775c30d2a1..ee2bb1ee48 100644 --- a/test/corruptH.test +++ b/test/corruptH.test @@ -20,6 +20,18 @@ set testprefix corruptH do_not_use_codec database_may_be_corrupt +# The corruption migrations tested by the code in this file are not detected +# mmap mode. +# +# The reason is that in mmap mode, the different queries may use different +# PgHdr objects for the same page (same data, but different PgHdr container +# objects). And so the corruption is not detected. +# +if {[permutation]=="mmap"} { + finish_test + return +} + # Initialize the database. # do_execsql_test 1.1 { @@ -52,6 +64,7 @@ do_test 1.2 { } {} do_test 1.3 { +breakpoint db eval { PRAGMA secure_delete=1 } list [catch { db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { @@ -97,9 +110,6 @@ do_test 2.2 { } {} -# The corruption migration caused by the test case below does not -# cause corruption to be detected in mmap mode. -# # The trick here is that the root page of the tree scanned by the outer # query is also currently on the free-list. So while the first seek on # the table (for a==1) works, by the time the second is attempted The @@ -109,15 +119,7 @@ do_test 2.2 { # the cursor (as it is now marked with PgHdr.intKey==0) and returns # SQLITE_CORRUPT. # -# However, in mmap mode, the outer query and the inner queries use -# different PgHdr objects (same data, but different PgHdr container -# objects). And so the corruption is not detected. Instead, the second -# seek fails to find anything and only a single row is returned. -# set res23 {1 {database disk image is malformed}} -if {[permutation]=="mmap"} { - set res23 {0 one} -} do_test 2.3 { list [catch { set res [list] From 8879481868f56a18f53aad35b2e9b384c52601d1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 7 Mar 2014 02:29:56 +0000 Subject: [PATCH 064/177] Avoid calling sqlite3OsDelete() on a file that is open, since this causes Windows to run *very* slowly. Comes up on error recovery in journal_mode=PERSIST. FossilOrigin-Name: fdc651e2ec7a0babee669e24fd56632e7cd5f0e9 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 12 ++++++------ test/uri.test | 2 ++ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index a96f29287e..88bbb69aa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\scorruptH.test\sin\smmap\smode. -D 2014-03-06T18:16:45.962 +C Avoid\scalling\ssqlite3OsDelete()\son\sa\sfile\sthat\sis\sopen,\ssince\sthis\scauses\sWindows\sto\srun\s*very*\sslowly.\s\sComes\sup\son\serror\srecovery\sin\sjournal_mode=PERSIST. +D 2014-03-07T02:29:56.179 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c 0ffa313a30ed6d061d9c6601b7b175cc50a1cab7 +F src/pager.c f5805e0e8c271d37e812402c98b50b9bfb1b196b F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1020,7 +1020,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb -F test/uri.test 63e03df051620a18f794b4f4adcdefb3c23b6751 +F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test af432e6e3bfc0ea20a80cb86a03c7d9876d38324 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0a4200f95cf46ad620b9fd91f4444114a0c74730 -R e9933db135410760cbb03eb238614bcf -U dan -Z b325d7c68ad2fff1170ad7d6e80e5338 +P c0d54b4e41cba84dc5934e0fcd03fe422fe5c92b +R 8e1ba1c1988955cecc40c9b987fa270a +U mistachkin +Z a001c15cb2d0783379551661d5488666 diff --git a/manifest.uuid b/manifest.uuid index 8adb79e711..28d5ef59d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0d54b4e41cba84dc5934e0fcd03fe422fe5c92b \ No newline at end of file +fdc651e2ec7a0babee669e24fd56632e7cd5f0e9 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index b6cd93b723..6a4746e340 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4889,15 +4889,15 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ - /* Check the size of the database file. If it consists of 0 pages, - ** then delete the journal file. See the header comment above for - ** the reasoning here. Delete the obsolete journal file under - ** a RESERVED lock to avoid race conditions and to avoid violating - ** [H33020]. + /* Check the size of the database file. If it consists of 0 pages + ** and the journal is not being persisted, then delete the journal + ** file. See the header comment above for the reasoning here. + ** Delete the obsolete journal file under a RESERVED lock to avoid + ** race conditions and to avoid violating [H33020]. */ rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ - if( nPage==0 ){ + if( nPage==0 && pPager->journalMode!=PAGER_JOURNALMODE_PERSIST ){ sqlite3BeginBenignMalloc(); if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); diff --git a/test/uri.test b/test/uri.test index af1ad67250..dd78783ddd 100644 --- a/test/uri.test +++ b/test/uri.test @@ -238,12 +238,14 @@ ifcapable wal { tvfs1 script tvfs1_callback proc tvfs1_callback {method filename args} { set ::T1([file tail $filename]) 1 + return SQLITE_OK } testvfs tvfs2 tvfs2 filter {xOpen xDelete xAccess xFullPathname} tvfs2 script tvfs2_callback proc tvfs2_callback {method filename args} { set ::T2([file tail $filename]) 1 + return SQLITE_OK } catch {db close} From eb443925ef31cbdcfb42655f7a9aa44c65e4c102 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 7 Mar 2014 03:31:35 +0000 Subject: [PATCH 065/177] Revise change from the previous check-in to clarify the situation when handling open journal files, regardless of journal mode. FossilOrigin-Name: 1c318ef3b76e9a9a5ff2f156a9acddfc1bda0949 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 88bbb69aa0..9e997b1eb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3OsDelete()\son\sa\sfile\sthat\sis\sopen,\ssince\sthis\scauses\sWindows\sto\srun\s*very*\sslowly.\s\sComes\sup\son\serror\srecovery\sin\sjournal_mode=PERSIST. -D 2014-03-07T02:29:56.179 +C Revise\schange\sfrom\sthe\sprevious\scheck-in\sto\sclarify\sthe\ssituation\swhen\shandling\sopen\sjournal\sfiles,\sregardless\sof\sjournal\smode. +D 2014-03-07T03:31:35.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c f5805e0e8c271d37e812402c98b50b9bfb1b196b +F src/pager.c ab9b5331e402f438effb02575955ffea673246cd F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c0d54b4e41cba84dc5934e0fcd03fe422fe5c92b -R 8e1ba1c1988955cecc40c9b987fa270a +P fdc651e2ec7a0babee669e24fd56632e7cd5f0e9 +R dd6c54a8aa52f6e2a8e41a668b913ede U mistachkin -Z a001c15cb2d0783379551661d5488666 +Z 7a2dfaabd9df7aaa8958f21cc25c2459 diff --git a/manifest.uuid b/manifest.uuid index 28d5ef59d6..13a19cc1e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdc651e2ec7a0babee669e24fd56632e7cd5f0e9 \ No newline at end of file +1c318ef3b76e9a9a5ff2f156a9acddfc1bda0949 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6a4746e340..e2486caa0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4897,7 +4897,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ */ rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ - if( nPage==0 && pPager->journalMode!=PAGER_JOURNALMODE_PERSIST ){ + if( nPage==0 && !jrnlOpen ){ sqlite3BeginBenignMalloc(); if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ sqlite3OsDelete(pVfs, pPager->zJournal, 0); From f3ccc38a697da306aee63e66c22c2d480d292875 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Mar 2014 14:57:07 +0000 Subject: [PATCH 066/177] Improved comment on the pager.c PERSIST rollback journal delete logic inside of hasHotJournal(). No changes to code. FossilOrigin-Name: e5b17a9d07a35c9b44ff977ba81b93d745d26a11 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 14 ++++++++------ 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 9e997b1eb1..f1ed65c66d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\schange\sfrom\sthe\sprevious\scheck-in\sto\sclarify\sthe\ssituation\swhen\shandling\sopen\sjournal\sfiles,\sregardless\sof\sjournal\smode. -D 2014-03-07T03:31:35.143 +C Improved\scomment\son\sthe\spager.c\sPERSIST\srollback\sjournal\sdelete\slogic\ninside\sof\shasHotJournal().\s\sNo\schanges\sto\scode. +D 2014-03-07T14:57:07.028 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -204,7 +204,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c ab9b5331e402f438effb02575955ffea673246cd +F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fdc651e2ec7a0babee669e24fd56632e7cd5f0e9 -R dd6c54a8aa52f6e2a8e41a668b913ede -U mistachkin -Z 7a2dfaabd9df7aaa8958f21cc25c2459 +P 1c318ef3b76e9a9a5ff2f156a9acddfc1bda0949 +R 061864752fb659e79b9068736cf1ac45 +U drh +Z 9721365445b2c154201b5f1346741e86 diff --git a/manifest.uuid b/manifest.uuid index 13a19cc1e7..e76365c166 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c318ef3b76e9a9a5ff2f156a9acddfc1bda0949 \ No newline at end of file +e5b17a9d07a35c9b44ff977ba81b93d745d26a11 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e2486caa0a..c6485a4d45 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4889,14 +4889,16 @@ static int hasHotJournal(Pager *pPager, int *pExists){ if( rc==SQLITE_OK && !locked ){ Pgno nPage; /* Number of pages in database file */ - /* Check the size of the database file. If it consists of 0 pages - ** and the journal is not being persisted, then delete the journal - ** file. See the header comment above for the reasoning here. - ** Delete the obsolete journal file under a RESERVED lock to avoid - ** race conditions and to avoid violating [H33020]. - */ rc = pagerPagecount(pPager, &nPage); if( rc==SQLITE_OK ){ + /* If the database is zero pages in size, that means that either (1) the + ** journal is a remnant from a prior database with the same name where + ** the database file but not the journal was deleted, or (2) the initial + ** transaction that populates a new database is being rolled back. + ** In either case, the journal file can be deleted. However, take care + ** not to delete the journal file if it is already open due to + ** journal_mode=PERSIST. + */ if( nPage==0 && !jrnlOpen ){ sqlite3BeginBenignMalloc(); if( pagerLockDb(pPager, RESERVED_LOCK)==SQLITE_OK ){ From b72cad14d03c6198e88b90005a2e0318702c0610 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 8 Mar 2014 19:07:03 +0000 Subject: [PATCH 067/177] Fix a bug causing "SELECT char()" to return SQLITE_NOMEM. FossilOrigin-Name: ba39df9d4f7ffc6475ae0dc794f7d3f58c486de8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 2 +- test/func.test | 5 +++++ 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f1ed65c66d..95666febf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomment\son\sthe\spager.c\sPERSIST\srollback\sjournal\sdelete\slogic\ninside\sof\shasHotJournal().\s\sNo\schanges\sto\scode. -D 2014-03-07T14:57:07.028 +C Fix\sa\sbug\scausing\s"SELECT\schar()"\sto\sreturn\sSQLITE_NOMEM. +D 2014-03-08T19:07:03.394 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf -F src/func.c f4499b39d66b71825514334ce67b32ff14bd19f5 +F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -573,7 +573,7 @@ F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test 00667bbeac044d007f6f021af1b9f6150f0c7ff8 +F test/func.test a21814945d32137412b553d98ad2107f9b2173a9 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1c318ef3b76e9a9a5ff2f156a9acddfc1bda0949 -R 061864752fb659e79b9068736cf1ac45 -U drh -Z 9721365445b2c154201b5f1346741e86 +P e5b17a9d07a35c9b44ff977ba81b93d745d26a11 +R f73df97bd275791734e233575a443767 +U dan +Z 7931bffb2eb7579d713d462f61979bc1 diff --git a/manifest.uuid b/manifest.uuid index e76365c166..afb40009da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5b17a9d07a35c9b44ff977ba81b93d745d26a11 \ No newline at end of file +ba39df9d4f7ffc6475ae0dc794f7d3f58c486de8 \ No newline at end of file diff --git a/src/func.c b/src/func.c index e657558ba2..6be963580c 100644 --- a/src/func.c +++ b/src/func.c @@ -1017,7 +1017,7 @@ static void charFunc( ){ unsigned char *z, *zOut; int i; - zOut = z = sqlite3_malloc( argc*4 ); + zOut = z = sqlite3_malloc( argc*4+1 ); if( z==0 ){ sqlite3_result_error_nomem(context); return; diff --git a/test/func.test b/test/func.test index edec591eae..0fafd1828d 100644 --- a/test/func.test +++ b/test/func.test @@ -1361,4 +1361,9 @@ for {set i 65536} {$i<=0x10ffff} {incr i 139} { do_execsql_test func-30.5.$i {SELECT unicode(char($i))} $i } +# Test char(). +# +do_execsql_test func-31.1 { + SELECT char(), length(char()), typeof(char()) +} {{} 0 text} finish_test From 55aeee4b48d99d97c9108a385318c54292895410 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Mar 2014 12:20:37 +0000 Subject: [PATCH 068/177] Version 3.8.4 FossilOrigin-Name: 530a1ee7dc2435f80960ce4710a3c2d2bfaaccc5 --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 95666febf7..f4391b3680 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\scausing\s"SELECT\schar()"\sto\sreturn\sSQLITE_NOMEM. -D 2014-03-08T19:07:03.394 +C Version\s3.8.4 +D 2014-03-10T12:20:37.813 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1155,7 +1155,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e5b17a9d07a35c9b44ff977ba81b93d745d26a11 +P ba39df9d4f7ffc6475ae0dc794f7d3f58c486de8 R f73df97bd275791734e233575a443767 -U dan -Z 7931bffb2eb7579d713d462f61979bc1 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.4 * +U drh +Z 82f901d4413784b1ea9af3cba9303801 diff --git a/manifest.uuid b/manifest.uuid index afb40009da..6e62e57b7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba39df9d4f7ffc6475ae0dc794f7d3f58c486de8 \ No newline at end of file +530a1ee7dc2435f80960ce4710a3c2d2bfaaccc5 \ No newline at end of file From b06a4ec1f01f6140206374a58a240abdefc9a356 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Mar 2014 18:03:09 +0000 Subject: [PATCH 069/177] Attempt to work around MSVC's treatment of __LINE__ as a non-constant value in "Edit and Continue" mode by avoiding the use of __LINE__ when SQLITE_VDBE_COVERAGE is not defined. FossilOrigin-Name: 0a5318bc272b844e937cb1df3a07224034bc3450 --- manifest | 21 +++++++++------------ manifest.uuid | 2 +- src/pragma.c | 6 +++--- src/trigger.c | 2 +- src/vdbe.h | 2 ++ src/vdbeblob.c | 2 +- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index f4391b3680..f54d726bd9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.4 -D 2014-03-10T12:20:37.813 +C Attempt\sto\swork\saround\sMSVC's\streatment\sof\s__LINE__\sas\sa\snon-constant\svalue\nin\s"Edit\sand\sContinue"\smode\sby\savoiding\sthe\suse\sof\s__LINE__\swhen\nSQLITE_VDBE_COVERAGE\sis\snot\sdefined. +D 2014-03-10T18:03:09.341 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -210,7 +210,7 @@ F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c a46ee83671f5c95f53d2ceeb5e1a818d7b1df99a +F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -272,17 +272,17 @@ F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 -F src/trigger.c a80036fcbd992729adc7cd34a875d59a71fa10cc +F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 5f0fffa9bf49a90c05dc3d46d8217603fd0ee00e -F src/vdbe.h d189f92468a17a6f04daeec9df3b767f50557b21 +F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 -F src/vdbeblob.c d939997de046b8fcc607cfee4248f3d33dbcca50 +F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1155,10 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ba39df9d4f7ffc6475ae0dc794f7d3f58c486de8 -R f73df97bd275791734e233575a443767 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.4 * +P 530a1ee7dc2435f80960ce4710a3c2d2bfaaccc5 +R 760c031eed969eb2e4721ef1a6cbabbc U drh -Z 82f901d4413784b1ea9af3cba9303801 +Z a6c38cee4adc797ddcb5e254b94d119b diff --git a/manifest.uuid b/manifest.uuid index 6e62e57b7b..b4216cd100 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -530a1ee7dc2435f80960ce4710a3c2d2bfaaccc5 \ No newline at end of file +0a5318bc272b844e937cb1df3a07224034bc3450 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index a211633f23..fcecad269a 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -824,7 +824,7 @@ void sqlite3Pragma( ** size of historical compatibility. */ case PragTyp_DEFAULT_CACHE_SIZE: { - static const int iLn = __LINE__+2; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ @@ -1087,7 +1087,7 @@ void sqlite3Pragma( ** file. Before writing to meta[6], check that meta[3] indicates ** that this really is an auto-vacuum capable database. */ - static const int iLn = __LINE__+2; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList setMeta6[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_LARGEST_ROOT_PAGE}, @@ -1790,7 +1790,7 @@ void sqlite3Pragma( ** messages have been generated, output OK. Otherwise output the ** error message */ - static const int iLn = __LINE__+2; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ { OP_IfNeg, 1, 0, 0}, /* 1 */ diff --git a/src/trigger.c b/src/trigger.c index dcbaf5d332..01f7b21f74 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -566,7 +566,7 @@ void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ int base; - static const int iLn = __LINE__+2; + static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList dropTrigger[] = { { OP_Rewind, 0, ADDR(9), 0}, { OP_String8, 0, 1, 0}, /* 1 */ diff --git a/src/vdbe.h b/src/vdbe.h index 66ca602ab7..8e300b88a6 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -272,11 +272,13 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); # define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) # define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); # define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) #else # define VdbeCoverage(v) # define VdbeCoverageIf(v,x) # define VdbeCoverageAlwaysTaken(v) # define VdbeCoverageNeverTaken(v) +# define VDBE_OFFSET_LINENO(x) 0 #endif #endif diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 4b34f2c295..cf7495e5b4 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -133,7 +133,7 @@ int sqlite3_blob_open( ** which closes the b-tree cursor and (possibly) commits the ** transaction. */ - static const int iLn = __LINE__+4; + static const int iLn = VDBE_OFFSET_LINENO(4); static const VdbeOpList openBlob[] = { /* {OP_Transaction, 0, 0, 0}, // 0: Inserted separately */ {OP_TableLock, 0, 0, 0}, /* 1: Acquire a read or write lock */ From 89212fb9f9f0c35bb4725bfc8b85eb96ad751939 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Mar 2014 20:12:31 +0000 Subject: [PATCH 070/177] Adjustments to the cost computation for the skip-scan query plan, to take into account the fact that a seek has to occur on each skip. FossilOrigin-Name: 0769eebd028ce31375cf93509a1d3687f7b117eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 +++ test/skipscan1.test | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f54d726bd9..37e199a7ca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\swork\saround\sMSVC's\streatment\sof\s__LINE__\sas\sa\snon-constant\svalue\nin\s"Edit\sand\sContinue"\smode\sby\savoiding\sthe\suse\sof\s__LINE__\swhen\nSQLITE_VDBE_COVERAGE\sis\snot\sdefined. -D 2014-03-10T18:03:09.341 +C Adjustments\sto\sthe\scost\scomputation\sfor\sthe\sskip-scan\squery\splan,\sto\stake\ninto\saccount\sthe\sfact\sthat\sa\sseek\shas\sto\soccur\son\seach\sskip. +D 2014-03-10T20:12:31.746 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 95d30485879122acc5284d4523e01399b23b6148 +F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -810,7 +810,7 @@ F test/shell5.test bb755ea9144b8078a752fc56223582627070b5f1 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 8c777ffd9dad6ee6d2568160cb2158f0b5cd9dd2 +F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 @@ -1155,7 +1155,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 530a1ee7dc2435f80960ce4710a3c2d2bfaaccc5 -R 760c031eed969eb2e4721ef1a6cbabbc +P 0a5318bc272b844e937cb1df3a07224034bc3450 +R e70fe0ad6a94758a25d6a041a31ecb69 U drh -Z a6c38cee4adc797ddcb5e254b94d119b +Z 344f247bc956540503c85d9ccd8fecb9 diff --git a/manifest.uuid b/manifest.uuid index b4216cd100..ed97f1ebb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a5318bc272b844e937cb1df3a07224034bc3450 \ No newline at end of file +0769eebd028ce31375cf93509a1d3687f7b117eb \ No newline at end of file diff --git a/src/where.c b/src/where.c index 963878d009..6cd9c167a1 100644 --- a/src/where.c +++ b/src/where.c @@ -3974,7 +3974,10 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + pNew->rRun = rLogSize + nIter; + pNew->nOut += nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; diff --git a/test/skipscan1.test b/test/skipscan1.test index b9eaf2729b..81cbc1dbc5 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -209,4 +209,42 @@ do_execsql_test skipscan1-4.1 { SELECT i FROM t4 WHERE h=8; } {9 9 9 9 9 9 9 9} +# Make sure skip-scan cost computation in the query planner takes into +# account the fact that the seek must occur multiple times. +# +# Prior to 2014-03-10, the costs were computed incorrectly which would +# cause index t5i2 to be used instead of t5i1 on the skipscan1-5.3. +# +do_execsql_test skipscan1-5.1 { + CREATE TABLE t5( + id INTEGER PRIMARY KEY, + loc TEXT, + lang INTEGER, + utype INTEGER, + xa INTEGER, + xd INTEGER, + xh INTEGER + ); + CREATE INDEX t5i1 on t5(loc, xh, xa, utype, lang); + CREATE INDEX t5i2 ON t5(xd,loc,utype,lang); + EXPLAIN QUERY PLAN + SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N'; +} {/.*COVERING INDEX t5i1 .*/} +do_execsql_test skipscan1-5.2 { + ANALYZE; + DELETE FROM sqlite_stat1; + DROP TABLE IF EXISTS sqlite_stat4; + DROP TABLE IF EXISTS sqlite_stat3; + INSERT INTO sqlite_stat1 VALUES('t5','t5i1','2702931 3 2 2 2 2'); + INSERT INTO sqlite_stat1 VALUES('t5','t5i2','2702931 686 2 2 2'); + ANALYZE sqlite_master; +} {} +db cache flush +do_execsql_test skipscan1-5.3 { + EXPLAIN QUERY PLAN + SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N'; +} {/.*COVERING INDEX t5i1 .*/} + + + finish_test From 3afca7d18cd4953c73e39f47e249a983000be97e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Mar 2014 09:19:26 +0000 Subject: [PATCH 071/177] Add a test to see what happens if an rtree constructor encounters an SQLITE_BUSY error. No changes to code. FossilOrigin-Name: 5527e23636eee192030f6a3c41628030b6c50bb8 --- ext/rtree/rtreeD.test | 57 +++++++++++++++++++++++++++++++++++++++++++ manifest | 13 +++++----- manifest.uuid | 2 +- 3 files changed, 65 insertions(+), 7 deletions(-) create mode 100644 ext/rtree/rtreeD.test diff --git a/ext/rtree/rtreeD.test b/ext/rtree/rtreeD.test new file mode 100644 index 0000000000..c4a7d22e2e --- /dev/null +++ b/ext/rtree/rtreeD.test @@ -0,0 +1,57 @@ +# 2014 March 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Miscellaneous tests for errors in the rtree constructor. +# + + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] rtree_util.tcl] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +ifcapable !rtree { + finish_test + return +} +set testprefix rtreeD + +#------------------------------------------------------------------------- +# Test that if an SQLITE_BUSY is encountered within the vtable +# constructor, a relevant error message is returned. +# +do_multiclient_test tn { + do_test 1.$tn.1 { + sql1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1,2); + CREATE VIRTUAL TABLE rt USING rtree(id, minx, maxx, miny, maxy); + INSERT INTO rt VALUES(1,2,3,4,5); + } + } {} + + do_test 1.$tn.2 { + sql2 { SELECT * FROM t1; } + } {1 2} + + do_test 1.$tn.3 { + sql1 { BEGIN EXCLUSIVE; INSERT INTO t1 VALUES(3, 4); } + } {} + + do_test 1.$tn.4 { + list [catch { sql2 { SELECT * FROM rt } } msg] $msg + } {1 {database is locked}} +} + +finish_test + + diff --git a/manifest b/manifest index 37e199a7ca..92cf570548 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustments\sto\sthe\scost\scomputation\sfor\sthe\sskip-scan\squery\splan,\sto\stake\ninto\saccount\sthe\sfact\sthat\sa\sseek\shas\sto\soccur\son\seach\sskip. -D 2014-03-10T20:12:31.746 +C Add\sa\stest\sto\ssee\swhat\shappens\sif\san\srtree\sconstructor\sencounters\san\sSQLITE_BUSY\serror.\sNo\schanges\sto\scode. +D 2014-03-11T09:19:26.952 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -134,6 +134,7 @@ F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 F ext/rtree/rtreeC.test 03975565f40a0bee165f613143e4dec716dd5a59 +F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h c34c1e41d1ab80bb8ad09aae402c9c956871a765 @@ -1155,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0a5318bc272b844e937cb1df3a07224034bc3450 -R e70fe0ad6a94758a25d6a041a31ecb69 -U drh -Z 344f247bc956540503c85d9ccd8fecb9 +P 0769eebd028ce31375cf93509a1d3687f7b117eb +R 69f07d6dc46567fe52be3d501cb07a93 +U dan +Z a17034cb7e3a0293ec242be23e8c481f diff --git a/manifest.uuid b/manifest.uuid index ed97f1ebb6..1d58945e9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0769eebd028ce31375cf93509a1d3687f7b117eb \ No newline at end of file +5527e23636eee192030f6a3c41628030b6c50bb8 \ No newline at end of file From 8073c70b332914ad61dc0309f0414306991ee120 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Mar 2014 15:25:43 +0000 Subject: [PATCH 072/177] Update version number to 3.8.4.1 FossilOrigin-Name: 33f5694fa42d9a521ca081de4ddd1e9dc8952b16 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index ff313b8c21..a93056bbe2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.4 +3.8.4.1 diff --git a/configure b/configure index c182a4a92d..56d58c7b73 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.4. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.4.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.4' -PACKAGE_STRING='sqlite 3.8.4' +PACKAGE_VERSION='3.8.4.1' +PACKAGE_STRING='sqlite 3.8.4.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.4 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.4.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.4:";; + short | recursive ) echo "Configuration of sqlite 3.8.4.1:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.4 +sqlite configure 3.8.4.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.4, which was +It was created by sqlite $as_me 3.8.4.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.4, which was +This file was extended by sqlite $as_me 3.8.4.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.4 +sqlite config.status 3.8.4.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 92cf570548..47a8571eef 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\sa\stest\sto\ssee\swhat\shappens\sif\san\srtree\sconstructor\sencounters\san\sSQLITE_BUSY\serror.\sNo\schanges\sto\scode. -D 2014-03-11T09:19:26.952 +C Update\sversion\snumber\sto\s3.8.4.1 +D 2014-03-11T15:25:43.839 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc fdacba6fb574868c71fde6db6b77789a383a4c19 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 0dc30ad5cf90736d5fd9e540c9f05c542658abe7 +F VERSION c3b0d47c3c5cf25c5bd4ff9e6f3af2f9d7934ea6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure e12e1a4f031354f9af735bcb377941a72dc2cca5 x +F configure 6a7a482649dd20b7ee31424bf6553c6f20ca7936 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0769eebd028ce31375cf93509a1d3687f7b117eb -R 69f07d6dc46567fe52be3d501cb07a93 -U dan -Z a17034cb7e3a0293ec242be23e8c481f +P 5527e23636eee192030f6a3c41628030b6c50bb8 +R 581994aed374f15c1c6bac5b0a9a5df5 +U drh +Z 8230a7efb9bcd6e2bb88b717c62ea7a8 diff --git a/manifest.uuid b/manifest.uuid index 1d58945e9c..0453f16ea5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5527e23636eee192030f6a3c41628030b6c50bb8 \ No newline at end of file +33f5694fa42d9a521ca081de4ddd1e9dc8952b16 \ No newline at end of file From 468108628c0399f0bc52f639b93317565badb3aa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Mar 2014 15:27:36 +0000 Subject: [PATCH 073/177] Version 3.8.4.1 FossilOrigin-Name: 018d317b1257ce68a92908b05c9c7cf1494050d0 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 47a8571eef..219399a93b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sversion\snumber\sto\s3.8.4.1 -D 2014-03-11T15:25:43.839 +C Version\s3.8.4.1 +D 2014-03-11T15:27:36.923 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5527e23636eee192030f6a3c41628030b6c50bb8 +P 33f5694fa42d9a521ca081de4ddd1e9dc8952b16 R 581994aed374f15c1c6bac5b0a9a5df5 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.4.1 * U drh -Z 8230a7efb9bcd6e2bb88b717c62ea7a8 +Z c1ec45dc8a92b80bfa16f4c74ccfeec8 diff --git a/manifest.uuid b/manifest.uuid index 0453f16ea5..1f0b42178b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33f5694fa42d9a521ca081de4ddd1e9dc8952b16 \ No newline at end of file +018d317b1257ce68a92908b05c9c7cf1494050d0 \ No newline at end of file From 5a500afd571a5f1219d6e840e8ab41ba772eb616 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Mar 2014 20:33:04 +0000 Subject: [PATCH 074/177] Enable the b-tree cursor objects overflow page-number cache, which is normally enabled only for incr-blob cursors, for all cursors. FossilOrigin-Name: da59198505990a4fe832be7932117c7e014955b7 --- manifest | 26 +++++++------- manifest.uuid | 2 +- src/btree.c | 98 +++++++++++++++++++++++--------------------------- src/btree.h | 2 +- src/btreeInt.h | 2 ++ src/vdbeblob.c | 4 +-- 6 files changed, 63 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index 219399a93b..17e141330e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.4.1 -D 2014-03-11T15:27:36.923 +C Enable\sthe\sb-tree\scursor\sobjects\soverflow\spage-number\scache,\swhich\sis\snormally\senabled\sonly\sfor\sincr-blob\scursors,\sfor\sall\scursors. +D 2014-03-11T20:33:04.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,9 +164,9 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4d28fb15543f0e071b1780b2af8cd2ee489de32d -F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f -F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 +F src/btree.c c748d8387789d01769e442b9da65e52ed61f408e +F src/btree.h 6b0c1a3f0937f9852ec727c820e71dbdd4bd0b27 +F src/btreeInt.h 22a8f07968ec821910ab93cc822d7f0c404c74d1 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac @@ -283,7 +283,7 @@ F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 -F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa +F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 @@ -1156,10 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 33f5694fa42d9a521ca081de4ddd1e9dc8952b16 -R 581994aed374f15c1c6bac5b0a9a5df5 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.4.1 * -U drh -Z c1ec45dc8a92b80bfa16f4c74ccfeec8 +P 018d317b1257ce68a92908b05c9c7cf1494050d0 +R 9660042129e37a760044c3b5edf13974 +T *branch * overflow-pgno-cache +T *sym-overflow-pgno-cache * +T -sym-trunk * +U dan +Z 262878375cf61ff20b91f83b47f52bc2 diff --git a/manifest.uuid b/manifest.uuid index 1f0b42178b..6b4cb8a17f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -018d317b1257ce68a92908b05c9c7cf1494050d0 \ No newline at end of file +da59198505990a4fe832be7932117c7e014955b7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 29ead1c67a..5716a5199a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -446,16 +446,11 @@ static int cursorHoldsMutex(BtCursor *p){ } #endif - -#ifndef SQLITE_OMIT_INCRBLOB /* -** Invalidate the overflow page-list cache for cursor pCur, if any. +** Invalidate the overflow cache of the cursor passed as the first argument. +** on the shared btree structure pBt. */ -static void invalidateOverflowCache(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - sqlite3_free(pCur->aOverflow); - pCur->aOverflow = 0; -} +#define invalidateOverflowCache(pCur) (pCur->bOvflValid = 0) /* ** Invalidate the overflow page-list cache for all cursors opened @@ -469,6 +464,7 @@ static void invalidateAllOverflowCache(BtShared *pBt){ } } +#ifndef SQLITE_OMIT_INCRBLOB /* ** This function is called before modifying the contents of a table ** to invalidate any incrblob cursors that are open on the @@ -498,9 +494,7 @@ static void invalidateIncrblobCursors( } #else - /* Stub functions when INCRBLOB is omitted */ - #define invalidateOverflowCache(x) - #define invalidateAllOverflowCache(x) + /* Stub function when INCRBLOB is omitted */ #define invalidateIncrblobCursors(x,y,z) #endif /* SQLITE_OMIT_INCRBLOB */ @@ -3694,7 +3688,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - invalidateOverflowCache(pCur); + sqlite3DbFree(pBtree->db, pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -3915,10 +3909,12 @@ static int copyPayload( /* ** This function is used to read or overwrite payload information -** for the entry that the pCur cursor is pointing to. If the eOp -** parameter is 0, this is a read operation (data copied into -** buffer pBuf). If it is non-zero, a write (data copied from -** buffer pBuf). +** for the entry that the pCur cursor is pointing to. The eOp +** argument is interpreted as follows: +** +** 0: The operation is a read. Populate the overflow cache. +** 1: The operation is a write. Populate the overflow cache. +** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -3926,11 +3922,11 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the BtCursor.isIncrblobHandle flag is set, and the current -** cursor entry uses one or more overflow pages, this function -** allocates space for and lazily popluates the overflow page-list -** cache array (BtCursor.aOverflow). Subsequent calls use this -** cache to make seeking to the supplied offset more efficient. +** If the current cursor entry uses one or more overflow pages and the +** eOp argument is not 2, this function may allocate space for and lazily +** popluates the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset +** more efficient. ** ** Once an overflow page-list cache has been allocated, it may be ** invalidated if some other cursor writes to the same table, or if @@ -3977,7 +3973,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -3991,7 +3987,6 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); -#ifndef SQLITE_OMIT_INCRBLOB /* If the isIncrblobHandle flag is set and the BtCursor.aOverflow[] ** has not been allocated, allocate it now. The array is sized at ** one entry for each overflow page in the overflow chain. The @@ -3999,13 +3994,22 @@ static int accessPayload( ** etc. A value of 0 in the aOverflow[] array means "not yet known" ** (the cache is lazily populated). */ - if( pCur->isIncrblobHandle && !pCur->aOverflow ){ + if( eOp!=2 && !pCur->bOvflValid ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - pCur->aOverflow = (Pgno *)sqlite3MallocZero(sizeof(Pgno)*nOvfl); - /* nOvfl is always positive. If it were zero, fetchPayload would have - ** been used instead of this routine. */ - if( ALWAYS(nOvfl) && !pCur->aOverflow ){ - rc = SQLITE_NOMEM; + if( nOvfl>pCur->nOvflAlloc ){ + Pgno *aNew = (Pgno*)sqlite3DbRealloc( + pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pCur->nOvflAlloc = nOvfl*2; + pCur->aOverflow = aNew; + } + } + if( rc==SQLITE_OK ){ + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->bOvflValid = 1; } } @@ -4013,22 +4017,19 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( pCur->aOverflow && pCur->aOverflow[offset/ovflSize] ){ + if( pCur->bOvflValid && pCur->aOverflow[offset/ovflSize] ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); } -#endif for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ -#ifndef SQLITE_OMIT_INCRBLOB /* If required, populate the overflow page-list cache. */ - if( pCur->aOverflow ){ + if( pCur->bOvflValid ){ assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); pCur->aOverflow[iIdx] = nextPage; } -#endif if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -4037,11 +4038,9 @@ static int accessPayload( ** page-list cache, if any, then fall back to the getOverflowPage() ** function. */ -#ifndef SQLITE_OMIT_INCRBLOB - if( pCur->aOverflow && pCur->aOverflow[iIdx+1] ){ + if( pCur->bOvflValid && pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; } else -#endif rc = getOverflowPage(pBt, nextPage, 0, &nextPage); offset -= ovflSize; }else{ @@ -4069,7 +4068,7 @@ static int accessPayload( ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( eOp==0 /* (1) */ + if( (eOp&0x01)==0 /* (1) */ && offset==0 /* (2) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ @@ -4087,12 +4086,12 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, - (eOp==0 ? PAGER_GET_READONLY : 0) + ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } @@ -4337,6 +4336,7 @@ static int moveToRoot(BtCursor *pCur){ assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); + invalidateOverflowCache(pCur); if( pCur->eState>=CURSOR_REQUIRESEEK ){ if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); @@ -4686,7 +4686,7 @@ int sqlite3BtreeMovetoUnpacked( goto moveto_finish; } pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); if( rc ){ sqlite3_free(pCellKey); goto moveto_finish; @@ -4775,6 +4775,7 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + invalidateOverflowCache(pCur); if( pCur->eState!=CURSOR_VALID ){ rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ @@ -4870,6 +4871,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + invalidateOverflowCache(pCur); pCur->atLast = 0; if( pCur->eState!=CURSOR_VALID ){ if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ @@ -8424,19 +8426,9 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ } /* -** Set a flag on this cursor to cache the locations of pages from the -** overflow list for the current row. This is used by cursors opened -** for incremental blob IO only. -** -** This function sets a flag only. The actual page location cache -** (stored in BtCursor.aOverflow[]) is allocated and used by function -** accessPayload() (the worker function for sqlite3BtreeData() and -** sqlite3BtreePutData()). +** Mark this cursor as an incremental blob cursor. */ -void sqlite3BtreeCacheOverflow(BtCursor *pCur){ - assert( cursorHoldsMutex(pCur) ); - assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); - invalidateOverflowCache(pCur); +void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ pCur->isIncrblobHandle = 1; } #endif diff --git a/src/btree.h b/src/btree.h index eda7bef70a..bedbef692f 100644 --- a/src/btree.h +++ b/src/btree.h @@ -187,7 +187,7 @@ char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); struct Pager *sqlite3BtreePager(Btree*); int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); -void sqlite3BtreeCacheOverflow(BtCursor *); +void sqlite3BtreeIncrblobCursor(BtCursor *); void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); diff --git a/src/btreeInt.h b/src/btreeInt.h index 87b4181f82..a41d8ebb55 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -497,6 +497,8 @@ struct BtCursor { BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ #ifndef SQLITE_OMIT_INCRBLOB + int nOvflAlloc; /* Allocated size of aOverflow[] array */ + u8 bOvflValid; /* True if size and contents of aOverflow[] valid */ Pgno *aOverflow; /* Cache of overflow page locations */ #endif Pgno pgnoRoot; /* The root page of this tree */ diff --git a/src/vdbeblob.c b/src/vdbeblob.c index cf7495e5b4..083f3f413c 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -77,9 +77,7 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ p->iOffset = pC->aType[p->iCol + pC->nField]; p->nByte = sqlite3VdbeSerialTypeLen(type); p->pCsr = pC->pCursor; - sqlite3BtreeEnterCursor(p->pCsr); - sqlite3BtreeCacheOverflow(p->pCsr); - sqlite3BtreeLeaveCursor(p->pCsr); + sqlite3BtreeIncrblobCursor(p->pCsr); } } From 036dbec01c083e196725e7fff530e8f8de61914d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Mar 2014 23:40:44 +0000 Subject: [PATCH 075/177] Combine the various boolean fields of the BtCursor object into a single bit-vector. This allows setting or clearing more than one boolean at a time and makes the overflow-pgno-cache branch faster than trunk on speedtest1. FossilOrigin-Name: 968fec44d7fde3adbd3e9603e4282351f0d4bda1 --- manifest | 21 ++++++------ manifest.uuid | 2 +- src/btree.c | 84 +++++++++++++++++++++++++----------------------- src/btreeInt.h | 27 ++++++++-------- src/test_btree.c | 2 +- 5 files changed, 69 insertions(+), 67 deletions(-) diff --git a/manifest b/manifest index 17e141330e..24561347ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\sb-tree\scursor\sobjects\soverflow\spage-number\scache,\swhich\sis\snormally\senabled\sonly\sfor\sincr-blob\scursors,\sfor\sall\scursors. -D 2014-03-11T20:33:04.219 +C Combine\sthe\svarious\sboolean\sfields\sof\sthe\sBtCursor\sobject\sinto\sa\ssingle\nbit-vector.\s\sThis\sallows\ssetting\sor\sclearing\smore\sthan\sone\sboolean\sat\sa\ntime\sand\smakes\sthe\soverflow-pgno-cache\sbranch\sfaster\sthan\strunk\son\s\nspeedtest1. +D 2014-03-11T23:40:44.961 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,9 +164,9 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c748d8387789d01769e442b9da65e52ed61f408e +F src/btree.c 225a3f4d5511d4f5eab38a686f6c7e22290739f3 F src/btree.h 6b0c1a3f0937f9852ec727c820e71dbdd4bd0b27 -F src/btreeInt.h 22a8f07968ec821910ab93cc822d7f0c404c74d1 +F src/btreeInt.h d1784d1e17d08d29e890190dbb9836fa64573381 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac @@ -239,7 +239,7 @@ F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e -F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 +F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc @@ -1156,10 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 018d317b1257ce68a92908b05c9c7cf1494050d0 -R 9660042129e37a760044c3b5edf13974 -T *branch * overflow-pgno-cache -T *sym-overflow-pgno-cache * -T -sym-trunk * -U dan -Z 262878375cf61ff20b91f83b47f52bc2 +P da59198505990a4fe832be7932117c7e014955b7 +R a9a97bb8415d8a76adacb051af2ccc88 +U drh +Z 90b8a93448c14945f126312bc79eea6f diff --git a/manifest.uuid b/manifest.uuid index 6b4cb8a17f..6cfc27f694 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da59198505990a4fe832be7932117c7e014955b7 \ No newline at end of file +968fec44d7fde3adbd3e9603e4282351f0d4bda1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5716a5199a..bd1a0e1c75 100644 --- a/src/btree.c +++ b/src/btree.c @@ -450,7 +450,7 @@ static int cursorHoldsMutex(BtCursor *p){ ** Invalidate the overflow cache of the cursor passed as the first argument. ** on the shared btree structure pBt. */ -#define invalidateOverflowCache(pCur) (pCur->bOvflValid = 0) +#define invalidateOverflowCache(pCur) (pCur->curFlags &= ~BTCF_ValidOvfl) /* ** Invalidate the overflow page-list cache for all cursors opened @@ -487,7 +487,7 @@ static void invalidateIncrblobCursors( BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p->isIncrblobHandle && (isClearTable || p->info.nKey==iRow) ){ + if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ p->eState = CURSOR_INVALID; } } @@ -2543,7 +2543,8 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ BtCursor *pCur; int r = 0; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ - if( (wrOnly==0 || pCur->wrFlag) && pCur->eState!=CURSOR_FAULT ) r++; + if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) + && pCur->eState!=CURSOR_FAULT ) r++; } return r; } @@ -3618,7 +3619,7 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->wrFlag = (u8)wrFlag; + pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; pCur->pNext = pBt->pCursor; if( pCur->pNext ){ pCur->pNext->pPrev = pCur; @@ -3727,7 +3728,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ if( pCur->info.nSize==0 ){ int iPage = pCur->iPage; btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); - pCur->validNKey = 1; + pCur->curFlags |= BTCF_ValidNKey; }else{ assertCellInfo(pCur); } @@ -3737,8 +3738,8 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ #define getCellInfo(pCur) \ if( pCur->info.nSize==0 ){ \ int iPage = pCur->iPage; \ - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ - pCur->validNKey = 1; \ + btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); \ + pCur->curFlags |= BTCF_ValidNKey; \ }else{ \ assertCellInfo(pCur); \ } @@ -3987,14 +3988,14 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); - /* If the isIncrblobHandle flag is set and the BtCursor.aOverflow[] + /* If the BTCF_Incrblob flag is set and the BtCursor.aOverflow[] ** has not been allocated, allocate it now. The array is sized at ** one entry for each overflow page in the overflow chain. The ** page number of the first overflow page is stored in aOverflow[0], ** etc. A value of 0 in the aOverflow[] array means "not yet known" ** (the cache is lazily populated). */ - if( eOp!=2 && !pCur->bOvflValid ){ + if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ Pgno *aNew = (Pgno*)sqlite3DbRealloc( @@ -4009,7 +4010,7 @@ static int accessPayload( } if( rc==SQLITE_OK ){ memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); - pCur->bOvflValid = 1; + pCur->curFlags |= BTCF_ValidOvfl; } } @@ -4017,7 +4018,7 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( pCur->bOvflValid && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -4026,7 +4027,7 @@ static int accessPayload( for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ /* If required, populate the overflow page-list cache. */ - if( pCur->bOvflValid ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ assert(!pCur->aOverflow[iIdx] || pCur->aOverflow[iIdx]==nextPage); pCur->aOverflow[iIdx] = nextPage; } @@ -4038,7 +4039,7 @@ static int accessPayload( ** page-list cache, if any, then fall back to the getOverflowPage() ** function. */ - if( pCur->bOvflValid && pCur->aOverflow[iIdx+1] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; } else rc = getOverflowPage(pBt, nextPage, 0, &nextPage); @@ -4239,14 +4240,14 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ return SQLITE_CORRUPT_BKPT; } rc = getAndInitPage(pBt, newPgno, &pNewPage, - pCur->wrFlag==0 ? PAGER_GET_READONLY : 0); + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; pCur->iPage++; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( pNewPage->nCell<1 || pNewPage->intKey!=pCur->apPage[i]->intKey ){ return SQLITE_CORRUPT_BKPT; } @@ -4304,7 +4305,7 @@ static void moveToParent(BtCursor *pCur){ releasePage(pCur->apPage[pCur->iPage]); pCur->iPage--; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); } /* @@ -4336,7 +4337,7 @@ static int moveToRoot(BtCursor *pCur){ assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); - invalidateOverflowCache(pCur); +// invalidateOverflowCache(pCur); if( pCur->eState>=CURSOR_REQUIRESEEK ){ if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); @@ -4352,7 +4353,7 @@ static int moveToRoot(BtCursor *pCur){ return SQLITE_OK; }else{ rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - pCur->wrFlag==0 ? PAGER_GET_READONLY : 0); + (pCur->curFlags & BTCF_WriteFlag)==0 ? PAGER_GET_READONLY : 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -4379,8 +4380,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->aiIdx[0] = 0; pCur->info.nSize = 0; - pCur->atLast = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); if( pRoot->nCell>0 ){ pCur->eState = CURSOR_VALID; @@ -4443,7 +4443,7 @@ static int moveToRightmost(BtCursor *pCur){ if( rc==SQLITE_OK ){ pCur->aiIdx[pCur->iPage] = pPage->nCell-1; pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~BTCF_ValidNKey; } return rc; } @@ -4482,7 +4482,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ - if( CURSOR_VALID==pCur->eState && pCur->atLast ){ + if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ #ifdef SQLITE_DEBUG /* This block serves to assert() that the cursor really does point ** to the last entry in the b-tree. */ @@ -4505,7 +4505,12 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ assert( pCur->eState==CURSOR_VALID ); *pRes = 0; rc = moveToRightmost(pCur); - pCur->atLast = rc==SQLITE_OK ?1:0; + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + } } return rc; @@ -4556,14 +4561,14 @@ int sqlite3BtreeMovetoUnpacked( /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && pCur->validNKey + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 && pCur->apPage[0]->intKey ){ if( pCur->info.nKey==intKey ){ *pRes = 0; return SQLITE_OK; } - if( pCur->atLast && pCur->info.nKeycurFlags & BTCF_AtLast)!=0 && pCur->info.nKeyupr ){ c = +1; break; } }else{ assert( nCellKey==intKey ); - pCur->validNKey = 1; + pCur->curFlags |= BTCF_ValidNKey; pCur->info.nKey = nCellKey; pCur->aiIdx[pCur->iPage] = (u16)idx; if( !pPage->leaf ){ @@ -4731,7 +4736,7 @@ moveto_next_layer: } moveto_finish: pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); return rc; } @@ -4775,8 +4780,8 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - invalidateOverflowCache(pCur); if( pCur->eState!=CURSOR_VALID ){ + invalidateOverflowCache(pCur); rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ *pRes = 0; @@ -4810,7 +4815,7 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ testcase( idx>pPage->nCell ); pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); @@ -4871,8 +4876,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ assert( pRes!=0 ); assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - invalidateOverflowCache(pCur); - pCur->atLast = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); if( pCur->eState!=CURSOR_VALID ){ if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ rc = btreeRestoreCursorPosition(pCur); @@ -4917,7 +4921,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ moveToParent(pCur); } pCur->info.nSize = 0; - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -6942,7 +6946,7 @@ int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( pCur->wrFlag && pBt->inTransaction==TRANS_WRITE + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -6975,7 +6979,7 @@ int sqlite3BtreeInsert( /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() ** call */ - if( pCur->validNKey && nKey>0 && pCur->info.nKey==nKey-1 ){ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ loc = -1; } } @@ -7028,7 +7032,7 @@ int sqlite3BtreeInsert( /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move - ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey + ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey ** variables. ** ** Previous versions of SQLite called moveToRoot() to move the cursor @@ -7048,7 +7052,7 @@ int sqlite3BtreeInsert( */ pCur->info.nSize = 0; if( rc==SQLITE_OK && pPage->nOverflow ){ - pCur->validNKey = 0; + pCur->curFlags &= ~(BTCF_ValidNKey); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() @@ -7080,7 +7084,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); assert( (pBt->btsFlags & BTS_READ_ONLY)==0 ); - assert( pCur->wrFlag ); + assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); @@ -8384,7 +8388,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ int rc; assert( cursorHoldsMutex(pCsr) ); assert( sqlite3_mutex_held(pCsr->pBtree->db->mutex) ); - assert( pCsr->isIncrblobHandle ); + assert( pCsr->curFlags & BTCF_Incrblob ); rc = restoreCursorPosition(pCsr); if( rc!=SQLITE_OK ){ @@ -8413,7 +8417,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** (d) there are no conflicting read-locks, and ** (e) the cursor points at a valid row of an intKey table. */ - if( !pCsr->wrFlag ){ + if( (pCsr->curFlags & BTCF_WriteFlag)==0 ){ return SQLITE_READONLY; } assert( (pCsr->pBt->btsFlags & BTS_READ_ONLY)==0 @@ -8429,7 +8433,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** Mark this cursor as an incremental blob cursor. */ void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ - pCur->isIncrblobHandle = 1; + pCur->curFlags |= BTCF_Incrblob; } #endif diff --git a/src/btreeInt.h b/src/btreeInt.h index a41d8ebb55..dff07f145e 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -496,29 +496,30 @@ struct BtCursor { BtShared *pBt; /* The BtShared this cursor points to */ BtCursor *pNext, *pPrev; /* Forms a linked list of all cursors */ struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ -#ifndef SQLITE_OMIT_INCRBLOB - int nOvflAlloc; /* Allocated size of aOverflow[] array */ - u8 bOvflValid; /* True if size and contents of aOverflow[] valid */ Pgno *aOverflow; /* Cache of overflow page locations */ -#endif - Pgno pgnoRoot; /* The root page of this tree */ CellInfo info; /* A parse of the cell we are pointing at */ - i64 nKey; /* Size of pKey, or last integer key */ - void *pKey; /* Saved key that was cursor's last known position */ + i64 nKey; /* Size of pKey, or last integer key */ + void *pKey; /* Saved key that was cursor last known position */ + Pgno pgnoRoot; /* The root page of this tree */ + int nOvflAlloc; /* Allocated size of aOverflow[] array */ int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ - u8 wrFlag; /* True if writable */ - u8 atLast; /* Cursor pointing to the last entry */ - u8 validNKey; /* True if info.nKey is valid */ + u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ -#ifndef SQLITE_OMIT_INCRBLOB - u8 isIncrblobHandle; /* True if this cursor is an incr. io handle */ -#endif u8 hints; /* As configured by CursorSetHints() */ i16 iPage; /* Index of current page in apPage */ u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ }; +/* +** Legal values for BtCursor.curFlags +*/ +#define BTCF_ValidNKey 0x01 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x02 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x04 /* Cursor is pointing ot the last entry */ +#define BTCF_WriteFlag 0x08 /* True if a write cursor */ +#define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ + /* ** Potential values for BtCursor.eState. ** diff --git a/src/test_btree.c b/src/test_btree.c index db72889b2a..dfe77051b6 100644 --- a/src/test_btree.c +++ b/src/test_btree.c @@ -51,7 +51,7 @@ void sqlite3BtreeCursorList(Btree *p){ BtShared *pBt = p->pBt; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ MemPage *pPage = pCur->apPage[pCur->iPage]; - char *zMode = pCur->wrFlag ? "rw" : "ro"; + char *zMode = (pCur->curFlags & BTCF_WriteFlag) ? "rw" : "ro"; sqlite3DebugPrintf("CURSOR %p rooted at %4d(%s) currently at %d.%d%s\n", pCur, pCur->pgnoRoot, zMode, pPage ? pPage->pgno : 0, pCur->aiIdx[pCur->iPage], From 926345e02761db4fc92b08568cc4c3a80c07953c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Mar 2014 23:44:10 +0000 Subject: [PATCH 076/177] Remove a stray C99-style comment. FossilOrigin-Name: f500e87d6ec6fa43c13f4e94edf32789f36e4233 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 24561347ec..ff09351dd2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Combine\sthe\svarious\sboolean\sfields\sof\sthe\sBtCursor\sobject\sinto\sa\ssingle\nbit-vector.\s\sThis\sallows\ssetting\sor\sclearing\smore\sthan\sone\sboolean\sat\sa\ntime\sand\smakes\sthe\soverflow-pgno-cache\sbranch\sfaster\sthan\strunk\son\s\nspeedtest1. -D 2014-03-11T23:40:44.961 +C Remove\sa\sstray\sC99-style\scomment. +D 2014-03-11T23:44:10.719 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 225a3f4d5511d4f5eab38a686f6c7e22290739f3 +F src/btree.c 33b5ad54a14c2b62e8adbc7656dec2b7cd760cbf F src/btree.h 6b0c1a3f0937f9852ec727c820e71dbdd4bd0b27 F src/btreeInt.h d1784d1e17d08d29e890190dbb9836fa64573381 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P da59198505990a4fe832be7932117c7e014955b7 -R a9a97bb8415d8a76adacb051af2ccc88 +P 968fec44d7fde3adbd3e9603e4282351f0d4bda1 +R c766c2eb192c274bbd3dc0af3849118d U drh -Z 90b8a93448c14945f126312bc79eea6f +Z 264233dee2a1cd950306c0e6dc59ee29 diff --git a/manifest.uuid b/manifest.uuid index 6cfc27f694..4c7b90b821 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -968fec44d7fde3adbd3e9603e4282351f0d4bda1 \ No newline at end of file +f500e87d6ec6fa43c13f4e94edf32789f36e4233 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index bd1a0e1c75..045e8ba2fd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4337,7 +4337,6 @@ static int moveToRoot(BtCursor *pCur){ assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); -// invalidateOverflowCache(pCur); if( pCur->eState>=CURSOR_REQUIRESEEK ){ if( pCur->eState==CURSOR_FAULT ){ assert( pCur->skipNext!=SQLITE_OK ); From 87af14a639aa677047f75e008aa7d107343a7510 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 12:44:46 +0000 Subject: [PATCH 077/177] Prevent the rtree module from reading sqlite_stat1 data from the wrong database. FossilOrigin-Name: 7ce03c1b5552d830300575c5b41a874db7a2ec77 --- ext/rtree/rtree.c | 36 ++++++++++-------- ext/rtree/rtreeC.test | 86 +++++++++++++++++++++++++++++++++++++++++++ manifest | 19 ++++------ manifest.uuid | 2 +- 4 files changed, 116 insertions(+), 27 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 577e19d4c6..cd652a3936 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -2947,26 +2947,32 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ ** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. */ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ - const char *zSql = "SELECT stat FROM sqlite_stat1 WHERE tbl= ? || '_rowid'"; + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; sqlite3_stmt *p; int rc; i64 nRow = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_text(p, 1, pRtree->zName, -1, SQLITE_STATIC); - if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); - rc = sqlite3_finalize(p); - }else if( rc!=SQLITE_NOMEM ){ - rc = SQLITE_OK; - } - - if( rc==SQLITE_OK ){ - if( nRow==0 ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - }else{ - pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); } return rc; diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index 2e5bedec96..b72007245f 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -158,5 +158,91 @@ do_execsql_test 4.3 { SELECT b, a FROM t2 LEFT JOIN t1 ON (+a = +b); } {1 1 3 {}} +#-------------------------------------------------------------------- +# Test that the sqlite_stat1 data is used correctly. +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(x PRIMARY KEY, y); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + + INSERT INTO t1(x) VALUES(1); + INSERT INTO t1(x) SELECT x+1 FROM t1; -- 2 + INSERT INTO t1(x) SELECT x+2 FROM t1; -- 4 + INSERT INTO t1(x) SELECT x+4 FROM t1; -- 8 + INSERT INTO t1(x) SELECT x+8 FROM t1; -- 16 + INSERT INTO t1(x) SELECT x+16 FROM t1; -- 32 + INSERT INTO t1(x) SELECT x+32 FROM t1; -- 64 + INSERT INTO t1(x) SELECT x+64 FROM t1; -- 128 + INSERT INTO t1(x) SELECT x+128 FROM t1; -- 256 + INSERT INTO t1(x) SELECT x+256 FROM t1; -- 512 + INSERT INTO t1(x) SELECT x+512 FROM t1; --1024 + + INSERT INTO rt SELECT x, x, x+1 FROM t1 WHERE x<=5; +} + +# First test a query with no ANALYZE data at all. The outer loop is +# real table "t1". +# +do_eqp_test 5.2 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# Now create enough ANALYZE data to tell SQLite that virtual table "rt" +# contains very few rows. This causes it to move "rt" to the outer loop. +# +do_execsql_test 5.3 { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; +} +db close +sqlite3 db test.db +do_eqp_test 5.4 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:} + 0 1 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (x=?)} +} + +# Delete the ANALYZE data. "t1" should be the outer loop again. +# +do_execsql_test 5.5 { DROP TABLE sqlite_stat1; } +db close +sqlite3 db test.db +do_eqp_test 5.6 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# This time create and attach a database that contains ANALYZE data for +# tables of the same names as those used internally by virtual table +# "rt". Check that the rtree module is not fooled into using this data. +# Table "t1" should remain the outer loop. +# +do_test 5.7 { + db backup test.db2 + sqlite3 db2 test.db2 + db2 eval { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; + } + db2 close + db close + sqlite3 db test.db + execsql { ATTACH 'test.db2' AS aux; } +} {} +do_eqp_test 5.8 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + + finish_test diff --git a/manifest b/manifest index 219399a93b..8dd646f40a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.4.1 -D 2014-03-11T15:27:36.923 +C Prevent\sthe\srtree\smodule\sfrom\sreading\ssqlite_stat1\sdata\sfrom\sthe\swrong\sdatabase. +D 2014-03-12T12:44:46.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,7 +120,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c a4e18b2c150adad20aecbeb3408cd235a0a57441 +F ext/rtree/rtree.c d963b961f7ebb3842a6dbc959ed8c3f3534206c4 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -133,7 +133,7 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test 03975565f40a0bee165f613143e4dec716dd5a59 +F ext/rtree/rtreeC.test afe930ac948c79fd12e48691c478d3f3bfc763d8 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea @@ -1156,10 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 33f5694fa42d9a521ca081de4ddd1e9dc8952b16 -R 581994aed374f15c1c6bac5b0a9a5df5 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.4.1 * -U drh -Z c1ec45dc8a92b80bfa16f4c74ccfeec8 +P 018d317b1257ce68a92908b05c9c7cf1494050d0 +R 771148b6f32577641865e57d520a643c +U dan +Z 84dfe23ccda2d3e1d846c6ac2ce685a0 diff --git a/manifest.uuid b/manifest.uuid index 1f0b42178b..b36a223a45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -018d317b1257ce68a92908b05c9c7cf1494050d0 \ No newline at end of file +7ce03c1b5552d830300575c5b41a874db7a2ec77 \ No newline at end of file From 82d25da5b08cc1358b1b19fdb625575dc923879e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 19:20:36 +0000 Subject: [PATCH 078/177] Changes to FTS to ensure that it does not access the database from within the xConnect method. FossilOrigin-Name: c67a52c356416cc200f51f9168d942e78db11d34 --- ext/fts3/fts3.c | 44 ++++++++++++++++++++++++++++++++++++------- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 4 ++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/fts3d.test | 11 +++++++++++ 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 44b7f431df..f7b2232902 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1410,10 +1410,7 @@ static int fts3InitVtab( ** addition of a %_stat table so that it can use incremental merge. */ if( !isFts4 && !isCreate ){ - int rc2 = SQLITE_OK; - fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", - p->zDb, p->zName); - if( rc2==SQLITE_OK ) p->bHasStat = 1; + p->bHasStat = 2; } /* Figure out the page-size for the database. This is required in order to @@ -3320,7 +3317,34 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ } /* -** Implementation of xBegin() method. This is a no-op. +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; @@ -3331,7 +3355,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); p->nLeafAdd = 0; - return SQLITE_OK; + return fts3SetHasStat(p); } /* @@ -3580,6 +3604,10 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + /* As it happens, the pending terms table is always empty here. This is ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction ** always opens a savepoint transaction. And the xSavepoint() method @@ -3587,7 +3615,9 @@ static int fts3RenameMethod( ** PendingTermsFlush() in in case that changes. */ assert( p->nPendingData==0 ); - rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 7f71736888..a5bb2f0041 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -223,7 +223,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3564cf8206..90d1609226 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -5271,6 +5271,10 @@ int sqlite3Fts3UpdateMethod( int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + assert( p->pSegments==0 ); assert( nArg==1 /* DELETE operations */ diff --git a/manifest b/manifest index 8dd646f40a..313f580784 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sthe\srtree\smodule\sfrom\sreading\ssqlite_stat1\sdata\sfrom\sthe\swrong\sdatabase. -D 2014-03-12T12:44:46.777 +C Changes\sto\sFTS\sto\sensure\sthat\sit\sdoes\snot\saccess\sthe\sdatabase\sfrom\swithin\sthe\sxConnect\smethod. +D 2014-03-12T19:20:36.904 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 3fe91e36a0304ad4b35020f0e22ff37e95873166 +F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 +F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 5165c365cb5a035f5be8bb296f7aa3211d43e4ac F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c cdebe72171a217b1465032dec103f8d17f7dad4d +F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -535,7 +535,7 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 -F test/fts3d.test c8a193513a269ec4c205ca09645e26e0bc71b860 +F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 F test/fts3defer2.test a3b6cbeabaf28c9398652a4d101ea224d9358479 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 018d317b1257ce68a92908b05c9c7cf1494050d0 -R 771148b6f32577641865e57d520a643c +P 7ce03c1b5552d830300575c5b41a874db7a2ec77 +R deefc2d49a0ad1f14f370fa09b6f4e09 U dan -Z 84dfe23ccda2d3e1d846c6ac2ce685a0 +Z 287a2bd2233940be3dc25c35e9cb956e diff --git a/manifest.uuid b/manifest.uuid index b36a223a45..1a5538a0a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce03c1b5552d830300575c5b41a874db7a2ec77 \ No newline at end of file +c67a52c356416cc200f51f9168d942e78db11d34 \ No newline at end of file diff --git a/test/fts3d.test b/test/fts3d.test index 818456bfae..2914818d4e 100644 --- a/test/fts3d.test +++ b/test/fts3d.test @@ -354,6 +354,17 @@ do_test fts3d-6.5 { SELECT name FROM sqlite_master WHERE name GLOB '???_*' ORDER BY 1; } } {xyz_content xyz_segdir xyz_segments} + +# ALTER TABLE RENAME on an FTS3 table following an incr-merge op. +# +do_test fts3d-6.6 { + execsql { INSERT INTO xyz(xyz) VALUES('merge=2,2') } + sqlite3 db test.db + execsql { + ALTER TABLE xyz RENAME TO ott; + SELECT name FROM sqlite_master WHERE name GLOB '???_*' ORDER BY 1; + } +} {ott_content ott_segdir ott_segments ott_stat} finish_test From d88e521f591a447cc13b377fcbd078ed435ebdb8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 19:38:38 +0000 Subject: [PATCH 079/177] Fix an obscure bug causing sqlite3_close() to fail if there are virtual tables on the disconnect list when it is called. FossilOrigin-Name: 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 --- ext/rtree/rtree.c | 2 ++ ext/rtree/rtreeC.test | 26 ++++++++++++++++++++++++ manifest | 20 +++++++++--------- manifest.uuid | 2 +- src/main.c | 1 + test/fts3ao.test | 28 ++++++++++++++++++++++++++ test/vtab_shared.test | 47 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 115 insertions(+), 11 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index cd652a3936..cefb9a8b2a 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3241,6 +3241,8 @@ static int rtreeInit( if( rc==SQLITE_OK ){ *ppVtab = (sqlite3_vtab *)pRtree; }else{ + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); rtreeRelease(pRtree); } return rc; diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index b72007245f..23dc607841 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -243,6 +243,32 @@ do_eqp_test 5.8 { 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} } +#-------------------------------------------------------------------- +# Test that having a second connection drop the sqlite_stat1 table +# before it is required by rtreeConnect() does not cause problems. +# +ifcapable rtree { + reset_db + do_execsql_test 6.1 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1); + INSERT INTO rt VALUES(1,2,3); + ANALYZE; + } + db close + sqlite3 db test.db + do_execsql_test 6.2 { SELECT * FROM t1 } {1} + + do_test 6.3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE sqlite_stat1 } + db2 close + execsql { SELECT * FROM rt } + } {1 2.0 3.0} + db close +} + finish_test diff --git a/manifest b/manifest index 313f580784..66dfec84b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sFTS\sto\sensure\sthat\sit\sdoes\snot\saccess\sthe\sdatabase\sfrom\swithin\sthe\sxConnect\smethod. -D 2014-03-12T19:20:36.904 +C Fix\san\sobscure\sbug\scausing\ssqlite3_close()\sto\sfail\sif\sthere\sare\svirtual\stables\son\sthe\sdisconnect\slist\swhen\sit\sis\scalled. +D 2014-03-12T19:38:38.004 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,7 +120,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d963b961f7ebb3842a6dbc959ed8c3f3534206c4 +F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -133,7 +133,7 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test afe930ac948c79fd12e48691c478d3f3bfc763d8 +F ext/rtree/rtreeC.test 16d7aa86ecb6a876d2a38cf590a1471a41b3a46d F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c e054917b1beb3081b0f23e8bdd3d6c0e12933dd3 +F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -523,7 +523,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 -F test/fts3ao.test 71b0675e3df5c512a5a03daaa95ee1916de23dda +F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2 F test/fts3atoken.test fca30fd86db9241d571c637751e9a8a2f50f1451 F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9 F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221 @@ -1047,7 +1047,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 -F test/vtab_shared.test 6acafaae7126c9f49be72c2f57eb8bef3024f1cb +F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 40073e54359d43354925b2b700b7eebd5e207285 F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7ce03c1b5552d830300575c5b41a874db7a2ec77 -R deefc2d49a0ad1f14f370fa09b6f4e09 +P c67a52c356416cc200f51f9168d942e78db11d34 +R 675a28e896dc8931c7ccad371f04b002 U dan -Z 287a2bd2233940be3dc25c35e9cb956e +Z 867ad6b65f1e7902a3a764dd985f4266 diff --git a/manifest.uuid b/manifest.uuid index 1a5538a0a1..7a533b2f8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c67a52c356416cc200f51f9168d942e78db11d34 \ No newline at end of file +6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 8b7495dd49..9e83d4963a 100644 --- a/src/main.c +++ b/src/main.c @@ -800,6 +800,7 @@ static void disconnectAllVtab(sqlite3 *db){ } } } + sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else UNUSED_PARAMETER(db); diff --git a/test/fts3ao.test b/test/fts3ao.test index 91693094db..60f0aa7d90 100644 --- a/test/fts3ao.test +++ b/test/fts3ao.test @@ -219,4 +219,32 @@ do_execsql_test 5.2 { SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; } {0 6} +# At one point this was causing a memory leak. +# +foreach {tn sql} { + 1 {} + 2 { INSERT INTO ft(ft) VALUES('merge=2,2'); } +} { + reset_db + do_execsql_test 6.$tn.1 " + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('hello world'); + $sql + " + + db close + sqlite3 db test.db + do_execsql_test 6.$tn.2 { SELECT * FROM t1 } {} + + do_test 6.$tn.3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE t1 } + db2 close + set stmt [sqlite3_prepare db { SELECT * FROM ft } -1 dummy] + sqlite3_finalize $stmt + } {SQLITE_OK} + db close +} + finish_test diff --git a/test/vtab_shared.test b/test/vtab_shared.test index 13237a8d20..3473992900 100644 --- a/test/vtab_shared.test +++ b/test/vtab_shared.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix vtab_shared ifcapable !vtab||!shared_cache { finish_test @@ -228,5 +229,51 @@ do_test vtab_shared_1.15.3 { db close db2 close + +#--------------------------------------------------------------- +# Test calling sqlite3_close() with vtabs on the disconnect list. +# +ifcapable rtree { + reset_db + do_test 2.1.1 { + sqlite3 db test.db + sqlite3 db2 test.db + + # Create a virtual table using [db]. + execsql { + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO rt VALUES(1, 2 ,3); + SELECT * FROM rt; + } + + # Drop the virtual table using [db2]. The sqlite3_vtab object belonging + # to [db] is moved to the sqlite3.pDisconnect list. + execsql { DROP TABLE rt } db2 + + # Immediately close [db]. At one point this would fail due to the + # unfinalized statements held by the un-xDisconnect()ed sqlite3_vtab. + db close + } {} + db2 close +} + +ifcapable fts3 { + # Same test as above, except using fts3 instead of rtree. + reset_db + do_test 2.2.1 { + sqlite3 db test.db + sqlite3 db2 test.db + execsql { + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('hello world'); + SELECT * FROM ft; + } + execsql { DROP TABLE ft } db2 + db close + } {} + db2 close +} + sqlite3_enable_shared_cache 0 finish_test + From 18c7e40ca3de1a1d10548f07d7cdb73db8abbbf1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Mar 2014 11:46:10 +0000 Subject: [PATCH 080/177] Fix a harmless compiler warning that crops up with SQLITE_MAX_MMAP_SIZE=0. FossilOrigin-Name: 1277932b7e8bb36c7070ffafbf110a8e128c267b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 ++ src/btree.h | 4 +++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 66dfec84b5..60f7a4b93f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sbug\scausing\ssqlite3_close()\sto\sfail\sif\sthere\sare\svirtual\stables\son\sthe\sdisconnect\slist\swhen\sit\sis\scalled. -D 2014-03-12T19:38:38.004 +C Fix\sa\sharmless\scompiler\swarning\sthat\scrops\sup\swith\sSQLITE_MAX_MMAP_SIZE=0. +D 2014-03-14T11:46:10.508 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,8 +164,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4d28fb15543f0e071b1780b2af8cd2ee489de32d -F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f +F src/btree.c 2a0e73f26c329f49e426237e71a879ffd205d778 +F src/btree.h da1b69b441ecee21f8b34ba73f0ae339540c4025 F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c67a52c356416cc200f51f9168d942e78db11d34 -R 675a28e896dc8931c7ccad371f04b002 -U dan -Z 867ad6b65f1e7902a3a764dd985f4266 +P 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 +R afd560ffe96625fe4d1d6971544be730 +U drh +Z 413087b4f5e9f622f8abe560a87a95ed diff --git a/manifest.uuid b/manifest.uuid index 7a533b2f8c..04c2fe3e5e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 \ No newline at end of file +1277932b7e8bb36c7070ffafbf110a8e128c267b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 29ead1c67a..9ca600e8cb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2161,6 +2161,7 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be ** memory mapped. @@ -2173,6 +2174,7 @@ int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ sqlite3BtreeLeave(p); return SQLITE_OK; } +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** Change the way data is synced to disk in order to increase or decrease diff --git a/src/btree.h b/src/btree.h index eda7bef70a..0c3fdfad63 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,7 +63,9 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); -int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#if SQLITE_MAX_MMAP_SIZE>0 + int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif int sqlite3BtreeSetPagerFlags(Btree*,unsigned); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); From a70a073b3fb789be80ec0c6bbbaeba2e67ee9a22 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Mar 2014 14:24:27 +0000 Subject: [PATCH 081/177] Performance enhancements and size reduction for sqlite3VXPrintf() FossilOrigin-Name: eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 --- manifest | 12 ++-- manifest.uuid | 2 +- src/printf.c | 155 ++++++++++++++++++++++++++------------------------ 3 files changed, 89 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 60f7a4b93f..7a2c6867be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning\sthat\scrops\sup\swith\sSQLITE_MAX_MMAP_SIZE=0. -D 2014-03-14T11:46:10.508 +C Performance\senhancements\sand\ssize\sreduction\sfor\ssqlite3VXPrintf() +D 2014-03-17T14:24:27.523 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 +F src/printf.c d95ea1b8a1e825a2f4756322f2ce35f91289fa1a F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 -R afd560ffe96625fe4d1d6971544be730 +P 1277932b7e8bb36c7070ffafbf110a8e128c267b +R 6d92934790c660d014aba176737edbd6 U drh -Z 413087b4f5e9f622f8abe560a87a95ed +Z c25cfd9c876d8aae4bc2be28c38a9b40 diff --git a/manifest.uuid b/manifest.uuid index 04c2fe3e5e..5f5b8ba0b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1277932b7e8bb36c7070ffafbf110a8e128c267b \ No newline at end of file +eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 8cfa542b41..a5a3fbbe0c 100644 --- a/src/printf.c +++ b/src/printf.c @@ -134,20 +134,6 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ -/* -** Append N space characters to the given string buffer. -*/ -void sqlite3AppendSpace(StrAccum *pAccum, int N){ - static const char zSpaces[] = " "; - while( N>=(int)sizeof(zSpaces)-1 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1); - N -= sizeof(zSpaces)-1; - } - if( N>0 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, N); - } -} - /* ** Set the StrAccum object to an error mode. */ @@ -237,11 +223,9 @@ void sqlite3VXPrintf( } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ - int amt; bufpt = (char *)fmt; - amt = 1; - while( (c=(*++fmt))!='%' && c!=0 ) amt++; - sqlite3StrAccumAppend(pAccum, bufpt, amt); + while( (c=(*++fmt))!='%' && c!=0 ){}; + sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); if( c==0 ) break; } if( (c=(*++fmt))==0 ){ @@ -729,29 +713,89 @@ void sqlite3VXPrintf( ** "length" characters long. The field width is "width". Do ** the output. */ - if( !flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } - if( length>0 ){ - sqlite3StrAccumAppend(pAccum, bufpt, length); - } - if( flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } + width -= length; + if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + sqlite3StrAccumAppend(pAccum, bufpt, length); + if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( zExtra ) sqlite3_free(zExtra); }/* End for loop over the format string */ } /* End of function */ /* -** Append N bytes of text from z to the StrAccum object. +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==STRACCUM_TOOBIG); + testcase(p->accError==STRACCUM_NOMEM); + return 0; + } + if( !p->useMalloc ){ + N = p->nAlloc - p->nChar - 1; + setStrAccumError(p, STRACCUM_TOOBIG); + return N; + }else{ + char *zOld = (p->zText==p->zBase ? 0 : p->zText); + i64 szNew = p->nChar; + szNew += N + 1; + if( szNew > p->mxAlloc ){ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->useMalloc==1 ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3_realloc(zOld, p->nAlloc); + } + if( zNew ){ + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + }else{ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N space characters to the given string buffer. +*/ +void sqlite3AppendSpace(StrAccum *p, int N){ + if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + while( (N--)>0 ) p->zText[p->nChar++] = ' '; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3StrAccumAppend() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3StrAccumAppend() routine can use fast calling semantics. +*/ +static void enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. */ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( z!=0 ); @@ -759,43 +803,8 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( N>=0 ); assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ - char *zNew; - if( p->accError ){ - testcase(p->accError==STRACCUM_TOOBIG); - testcase(p->accError==STRACCUM_NOMEM); - return; - } - if( !p->useMalloc ){ - N = p->nAlloc - p->nChar - 1; - setStrAccumError(p, STRACCUM_TOOBIG); - if( N<=0 ){ - return; - } - }else{ - char *zOld = (p->zText==p->zBase ? 0 : p->zText); - i64 szNew = p->nChar; - szNew += N + 1; - if( szNew > p->mxAlloc ){ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_TOOBIG); - return; - }else{ - p->nAlloc = (int)szNew; - } - if( p->useMalloc==1 ){ - zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); - }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); - } - if( zNew ){ - if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); - p->zText = zNew; - }else{ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_NOMEM); - return; - } - } + enlargeAndAppend(p,z,N); + return; } assert( p->zText ); memcpy(&p->zText[p->nChar], z, N); From 0e682099a1ee1bf0b63937834b952f77ec55f1b5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Mar 2014 15:06:57 +0000 Subject: [PATCH 082/177] Clean up some obsolete "register" declarations in printf.c. FossilOrigin-Name: ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7a2c6867be..bb21639801 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sand\ssize\sreduction\sfor\ssqlite3VXPrintf() -D 2014-03-17T14:24:27.523 +C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. +D 2014-03-17T15:06:57.359 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c d95ea1b8a1e825a2f4756322f2ce35f91289fa1a +F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1277932b7e8bb36c7070ffafbf110a8e128c267b -R 6d92934790c660d014aba176737edbd6 +P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 +R f35820126ad6a1c360258186873a2e79 U drh -Z c25cfd9c876d8aae4bc2be28c38a9b40 +Z b6382160ccdf74a44bc9653763bb82ef diff --git a/manifest.uuid b/manifest.uuid index 5f5b8ba0b3..3e3e1d6ac7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 \ No newline at end of file +ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index a5a3fbbe0c..01209c042d 100644 --- a/src/printf.c +++ b/src/printf.c @@ -406,10 +406,8 @@ void sqlite3VXPrintf( *(--bufpt) = zOrd[x*2]; } { - register const char *cset; /* Use registers for speed */ - register int base; - cset = &aDigits[infop->charset]; - base = infop->base; + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; do{ /* Convert to ascii */ *(--bufpt) = cset[longvalue%base]; longvalue = longvalue/base; From 0401acec910236ec9a7d9758e00a62cdebf684b8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Mar 2014 15:30:27 +0000 Subject: [PATCH 083/177] Experiments with the optimization of ORDER BY and GROUP BY clauses. FossilOrigin-Name: b150902579d708b454efd5f8750e26a816f7f1a6 --- manifest | 17 +++++++----- manifest.uuid | 2 +- src/where.c | 72 ++++++++++++++++++++++++-------------------------- src/whereInt.h | 5 ++-- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index bb21639801..6f347f2fef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. -D 2014-03-17T15:06:57.359 +C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses. +D 2014-03-18T15:30:27.660 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 -F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 +F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2 +F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 -R f35820126ad6a1c360258186873a2e79 +P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 +R 8abedb680a805ab9b5e400a1ac18119b +T *branch * orderby-planning +T *sym-orderby-planning * +T -sym-trunk * U drh -Z b6382160ccdf74a44bc9653763bb82ef +Z 6c19349f1c38f37cb3b2bffb6095d2a0 diff --git a/manifest.uuid b/manifest.uuid index 3e3e1d6ac7..26699b8468 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file +b150902579d708b454efd5f8750e26a816f7f1a6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6cd9c167a1..bed58ac752 100644 --- a/src/where.c +++ b/src/where.c @@ -4506,8 +4506,8 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) - && pIdxInfo->orderByConsumed); + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -4668,11 +4668,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 5th ** parameters) to see if it outputs rows in the requested ORDER BY -** (or GROUP BY) without requiring a separate sort operation. Return: +** (or GROUP BY) without requiring a separate sort operation. Return N: ** -** 0: ORDER BY is not satisfied. Sorting required -** 1: ORDER BY is satisfied. Omit sorting -** -1: Unknown at this time +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. ** ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** strict. With GROUP BY and DISTINCT the only requirement is that @@ -4682,7 +4682,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ -static int wherePathSatisfiesOrderBy( +static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ @@ -4915,7 +4915,7 @@ static int wherePathSatisfiesOrderBy( } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return 1; + if( obSat==obDone ) return nOrderBy; if( !isOrderDistinct ) return 0; return -1; } @@ -5001,12 +5001,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** to sqlite3WhereBegin() was concerned about sorting */ rSortCost = 0; if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - aFrom[0].isOrderedValid = 1; + aFrom[0].isOrdered = 0; }else{ /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the ** number of output rows. The 48 is the expected size of a row to sort. ** FIXME: compute a better estimate of the 48 multiplier based on the ** result set expressions. */ + aFrom[0].isOrdered = -1; rSortCost = nRowEst + estLog(nRowEst); WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } @@ -5020,8 +5021,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; Bitmask revMask = 0; - u8 isOrderedValid = pFrom->isOrderedValid; - u8 isOrdered = pFrom->isOrdered; + i8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. @@ -5030,21 +5030,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; - if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, - iLoop, pWLoop, &revMask) ){ - case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ - isOrdered = 1; - isOrderedValid = 1; - break; - case 0: /* No. pFrom+pWLoop will require a separate sort */ - isOrdered = 0; - isOrderedValid = 1; - rCost = sqlite3LogEstAdd(rCost, rSortCost); - break; - default: /* Cannot tell yet. Try again on the next iteration */ - break; + iLoop, pWLoop, &revMask); + if( isOrdered==0 ){ + rCost = sqlite3LogEstAdd(rCost, rSortCost); } }else{ revMask = pFrom->revLoop; @@ -5052,7 +5043,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Check to see if pWLoop should be added to the mxChoice best so far */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && pTo->isOrderedValid==isOrderedValid + && ((pTo->isOrdered^isOrdered)&80)==0 && ((pTo->rCost<=rCost && pTo->nRow<=nOut) || (pTo->rCost>=rCost && pTo->nRow>=nOut)) ){ @@ -5066,7 +5057,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif continue; @@ -5084,7 +5075,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif }else{ @@ -5094,10 +5085,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ sqlite3DebugPrintf( "Skip %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif testcase( pTo->rCost==rCost ); @@ -5110,10 +5101,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ sqlite3DebugPrintf( "Update %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif } @@ -5122,7 +5113,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; - pTo->isOrderedValid = isOrderedValid; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; @@ -5147,8 +5137,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); - if( pTo->isOrderedValid && pTo->isOrdered ){ + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); }else{ sqlite3DebugPrintf("\n"); @@ -5191,9 +5181,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ Bitmask notUsed; int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } } - if( pFrom->isOrdered ){ + if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; }else{ @@ -5386,7 +5378,7 @@ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ @@ -5408,6 +5400,10 @@ WhereInfo *sqlite3WhereBegin( /* Variable initialization */ db = pParse->db; memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; sWLB.pOrderBy = pOrderBy; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via diff --git a/src/whereInt.h b/src/whereInt.h index 23f929c461..233b331d42 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -121,7 +121,7 @@ struct WhereLoop { struct { /* Information for virtual tables */ int idxNum; /* Index number */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */ - u8 isOrdered; /* True if satisfies ORDER BY */ + i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ } vtab; @@ -183,8 +183,7 @@ struct WherePath { Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ - u8 isOrdered; /* True if this path satisfies ORDER BY */ - u8 isOrderedValid; /* True if the isOrdered field is valid */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; From d2de861cbece884fe9eb9f033308f6ca21cd7bab Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Mar 2014 18:59:07 +0000 Subject: [PATCH 084/177] Adjust the query planner to keep track of the number of ORDER BY terms satisfied. Still doesn't do anything with this information. Some tests fail after this check-in, but all failures are believed to be benign. The failures will be addressed at a later stage. FossilOrigin-Name: 59d49b7fc49fa290e04a02653e7268c85836b27e --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 30 ++++++++++++++++++++---------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 6f347f2fef..cf41f09f0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses. -D 2014-03-18T15:30:27.660 +C Adjust\sthe\squery\splanner\sto\skeep\strack\sof\sthe\snumber\sof\sORDER\sBY\sterms\nsatisfied.\s\sStill\sdoesn't\sdo\sanything\swith\sthis\sinformation.\s\sSome\stests\nfail\safter\sthis\scheck-in,\sbut\sall\sfailures\sare\sbelieved\sto\sbe\sbenign.\s\sThe\nfailures\swill\sbe\saddressed\sat\sa\slater\sstage. +D 2014-03-18T18:59:07.798 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2 +F src/where.c 81eea5ced2a2586ccd7ebd1d5bed7d34f4a2e99c F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,10 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 8abedb680a805ab9b5e400a1ac18119b -T *branch * orderby-planning -T *sym-orderby-planning * -T -sym-trunk * +P b150902579d708b454efd5f8750e26a816f7f1a6 +R 179b4f0ee34acb2c0f356485d1e56bc3 U drh -Z 6c19349f1c38f37cb3b2bffb6095d2a0 +Z adae3618c374e493bb2483c0db46dcf7 diff --git a/manifest.uuid b/manifest.uuid index 26699b8468..6d8c042ba1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b150902579d708b454efd5f8750e26a816f7f1a6 \ No newline at end of file +59d49b7fc49fa290e04a02653e7268c85836b27e \ No newline at end of file diff --git a/src/where.c b/src/where.c index bed58ac752..a766def035 100644 --- a/src/where.c +++ b/src/where.c @@ -4916,7 +4916,13 @@ static i8 wherePathSatisfiesOrderBy( } } /* End the loop over all WhereLoops from outer-most down to inner-most */ if( obSat==obDone ) return nOrderBy; - if( !isOrderDistinct ) return 0; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } return -1; } @@ -4953,11 +4959,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ LogEst rCost; /* Cost of a path */ LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ LogEst mxOut = 0; /* Maximum nOut value on the set of paths */ - LogEst rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -4999,17 +5005,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ - rSortCost = 0; if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrdered = 0; + nOrderBy = 0; }else{ - /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the - ** number of output rows. The 48 is the expected size of a row to sort. - ** FIXME: compute a better estimate of the 48 multiplier based on the - ** result set expressions. */ aFrom[0].isOrdered = -1; - rSortCost = nRowEst + estLog(nRowEst); - WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); + nOrderBy = pWInfo->pOrderBy->nExpr; } /* Compute successively longer WherePaths using the previous generation @@ -5034,7 +5035,16 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); - if( isOrdered==0 ){ + if( isOrdered>=0 && isOrdered Date: Tue, 18 Mar 2014 20:33:42 +0000 Subject: [PATCH 085/177] Make the partial-ORDER-BY information in the query planner available to the SELECT code generator. Still doesn't make a difference in the generated code. FossilOrigin-Name: e258df236b7de70087c8227cb209080e55b9bf9c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 25 ++++++++++++++----------- src/where.c | 17 ++++++++++------- src/whereInt.h | 2 +- 5 files changed, 34 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index cf41f09f0d..d76e572699 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\squery\splanner\sto\skeep\strack\sof\sthe\snumber\sof\sORDER\sBY\sterms\nsatisfied.\s\sStill\sdoesn't\sdo\sanything\swith\sthis\sinformation.\s\sSome\stests\nfail\safter\sthis\scheck-in,\sbut\sall\sfailures\sare\sbelieved\sto\sbe\sbenign.\s\sThe\nfailures\swill\sbe\saddressed\sat\sa\slater\sstage. -D 2014-03-18T18:59:07.798 +C Make\sthe\spartial-ORDER-BY\sinformation\sin\sthe\squery\splanner\savailable\sto\sthe\nSELECT\scode\sgenerator.\s\sStill\sdoesn't\smake\sa\sdifference\sin\sthe\sgenerated\scode. +D 2014-03-18T20:33:42.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 +F src/select.c 41df0e4dd1d167dc8b39047c84e1b8b51516a237 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 81eea5ced2a2586ccd7ebd1d5bed7d34f4a2e99c -F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f +F src/where.c 3f5de18e74b0f6da3218c2c609616a2f9e2c404a +F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b150902579d708b454efd5f8750e26a816f7f1a6 -R 179b4f0ee34acb2c0f356485d1e56bc3 +P 59d49b7fc49fa290e04a02653e7268c85836b27e +R 2b69bb33bc67b11eec1f6a4a37941d3e U drh -Z adae3618c374e493bb2483c0db46dcf7 +Z 43f4f2d39f5cfe334420891461bd8dda diff --git a/manifest.uuid b/manifest.uuid index 6d8c042ba1..4ef359f854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59d49b7fc49fa290e04a02653e7268c85836b27e \ No newline at end of file +e258df236b7de70087c8227cb209080e55b9bf9c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 850bc6a902..613ea25e26 100644 --- a/src/select.c +++ b/src/select.c @@ -562,6 +562,7 @@ static void selectInnerLoop( ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ + int nOBSat, /* Terms of ORDER BY already satisfied */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ @@ -1054,10 +1055,10 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; @@ -1918,7 +1919,7 @@ static void generateWithRecursiveQuery( addrCont = sqlite3VdbeMakeLabel(v); codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, - 0, 0, pDest, addrCont, addrBreak); + 0, 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); VdbeCoverage(v); @@ -2192,7 +2193,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, - 0, 0, &dest, iCont, iBreak); + 0, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -2270,7 +2271,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, - 0, 0, &dest, iCont, iBreak); + 0, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -4730,6 +4731,7 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + int nOBSat; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, @@ -4741,7 +4743,8 @@ int sqlite3Select( if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } - if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0; + nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( pOrderBy && nOBSat==pOrderBy->nExpr ){ pOrderBy = 0; nOBSat = 0; } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -4753,7 +4756,7 @@ int sqlite3Select( } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, nOBSat, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4875,7 +4878,7 @@ int sqlite3Select( pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY, 0); if( pWInfo==0 ) goto select_end; - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo @@ -5026,7 +5029,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, 0, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5158,7 +5161,7 @@ int sqlite3Select( } updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -5169,7 +5172,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -5186,7 +5189,7 @@ int sqlite3Select( */ if( pOrderBy ){ explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, v, pEList->nExpr, pDest); + generateSortTail(pParse, p, pEList->nExpr, pDest); } /* Jump here to skip this query diff --git a/src/where.c b/src/where.c index a766def035..9d83636f4b 100644 --- a/src/where.c +++ b/src/where.c @@ -39,7 +39,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** Return FALSE if the output needs to be sorted. */ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->bOBSat!=0; + return pWInfo->nOBSat; } /* @@ -3036,8 +3036,11 @@ static Bitmask codeOneLoopStart( ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pWInfo->bOBSat!=0) + && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ assert( pLoop->u.btree.nSkip==0 ); @@ -5199,7 +5202,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; }else{ - pWInfo->bOBSat = 1; + pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; } } @@ -5284,7 +5287,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; - if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5492,7 +5495,7 @@ WhereInfo *sqlite3WhereBegin( /* Special case: No FROM clause */ if( nTabList==0 ){ - if( pOrderBy ) pWInfo->bOBSat = 1; + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; if( wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5603,8 +5606,8 @@ WhereInfo *sqlite3WhereBegin( if( sqlite3WhereTrace ){ int ii; sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); - if( pWInfo->bOBSat ){ - sqlite3DebugPrintf(" ORDERBY=0x%llx", pWInfo->revMask); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); } switch( pWInfo->eDistinct ){ case WHERE_DISTINCT_UNIQUE: { diff --git a/src/whereInt.h b/src/whereInt.h index 233b331d42..419bb81c0e 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -397,7 +397,7 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 bOBSat; /* ORDER BY satisfied by indices */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ From 079a3072590186c50ad93c4b4dfd911c2f2e08f5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 14:10:55 +0000 Subject: [PATCH 086/177] First attempt at getting block-sort to work. This is an incremental check-in. There are many problems still to be worked out. FossilOrigin-Name: 59742dd4c5259883850044d0938248b009ebd045 --- manifest | 30 +++--- manifest.uuid | 2 +- src/btree.c | 9 ++ src/btree.h | 1 + src/expr.c | 6 +- src/pragma.c | 2 +- src/select.c | 249 +++++++++++++++++++++++++++++++----------------- src/sqliteInt.h | 7 +- src/vdbe.c | 32 +++++-- src/vdbeInt.h | 1 + src/vdbeaux.c | 4 +- src/where.c | 6 +- 12 files changed, 225 insertions(+), 124 deletions(-) diff --git a/manifest b/manifest index d76e572699..a367d8b9b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\spartial-ORDER-BY\sinformation\sin\sthe\squery\splanner\savailable\sto\sthe\nSELECT\scode\sgenerator.\s\sStill\sdoesn't\smake\sa\sdifference\sin\sthe\sgenerated\scode. -D 2014-03-18T20:33:42.067 +C First\sattempt\sat\sgetting\sblock-sort\sto\swork.\s\sThis\sis\san\sincremental\scheck-in.\nThere\sare\smany\sproblems\sstill\sto\sbe\sworked\sout. +D 2014-03-19T14:10:55.625 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,8 +164,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 2a0e73f26c329f49e426237e71a879ffd205d778 -F src/btree.h da1b69b441ecee21f8b34ba73f0ae339540c4025 +F src/btree.c 029cec7b98fe0a985922c03f101630391044c4ad +F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 +F src/expr.c e33fa25d2279c692ad899695cfb93db840498b65 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -211,18 +211,18 @@ F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 +F src/pragma.c 10f169b9650f0930a7a6df67e1387a4c2c449f38 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 41df0e4dd1d167dc8b39047c84e1b8b51516a237 +F src/select.c bd6ac5dc27046877f53065ea25f36ca90db4c594 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fa7161b3de18a9c355d4148233f3563c92311fcc +F src/sqliteInt.h 03f7d4deeaa1a9558df62f6df7bc907bb473fcea F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -278,11 +278,11 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5f0fffa9bf49a90c05dc3d46d8217603fd0ee00e +F src/vdbe.c e0fe939465f101a6f30f1864813698edc57f610e F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b +F src/vdbeInt.h aa7ea20696609966c71a4b58c6a4fdb974c5e877 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 +F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3f5de18e74b0f6da3218c2c609616a2f9e2c404a +F src/where.c d4739c3900e4cb99dfd7d19853e6b4fb63c9d342 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59d49b7fc49fa290e04a02653e7268c85836b27e -R 2b69bb33bc67b11eec1f6a4a37941d3e +P e258df236b7de70087c8227cb209080e55b9bf9c +R edda8149383dd364d2984f7033840ec9 U drh -Z 43f4f2d39f5cfe334420891461bd8dda +Z c306e7ec68619896c760c46adb63dbc2 diff --git a/manifest.uuid b/manifest.uuid index 4ef359f854..837615cb34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e258df236b7de70087c8227cb209080e55b9bf9c \ No newline at end of file +59742dd4c5259883850044d0938248b009ebd045 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9ca600e8cb..486cf8e6d5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7425,6 +7425,15 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ return rc; } +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + /* ** Erase all information in a table and add the root of the table to ** the freelist. Except, the root of the principle table (the one on diff --git a/src/btree.h b/src/btree.h index 0c3fdfad63..a62c3c60d4 100644 --- a/src/btree.h +++ b/src/btree.h @@ -115,6 +115,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); +int sqlite3BtreeClearTableOfCursor(BtCursor*); void sqlite3BtreeTripAllCursors(Btree*, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); diff --git a/src/expr.c b/src/expr.c index 722a77db73..2d7c95a682 100644 --- a/src/expr.c +++ b/src/expr.c @@ -949,7 +949,6 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ if( p==0 ) return 0; pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; - pNew->iECursor = 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); @@ -1062,7 +1061,6 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); return pNew; @@ -2335,7 +2333,7 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ int i; struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); for(i=0, p=pParse->aColCache; iiReg; if( x>=iFrom && xaDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_Move, 2, 4); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); diff --git a/src/select.c b/src/select.c index 613ea25e26..b3f7f861bc 100644 --- a/src/select.c +++ b/src/select.c @@ -14,6 +14,34 @@ */ #include "sqliteInt.h" +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure but do not deallocate @@ -87,7 +115,6 @@ Select *sqlite3SelectNew( assert( pOffset==0 || pLimit!=0 ); pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; if( db->mallocFailed ) { clearSelect(db, pNew); if( pNew!=&standin ) sqlite3DbFree(db, pNew); @@ -419,32 +446,69 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } +/* Forward reference */ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +); + /* -** Insert code into "v" that will push the record on the top of the -** stack into the sorter. +** Insert code into "v" that will push the record in register regData +** into the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ - ExprList *pOrderBy, /* The ORDER BY clause */ + SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData /* Register holding data to be sorted */ ){ Vdbe *v = pParse->pVdbe; - int nExpr = pOrderBy->nExpr; + int nExpr = pSort->pOrderBy->nExpr; int regBase = sqlite3GetTempRange(pParse, nExpr+2); int regRecord = sqlite3GetTempReg(pParse); + int nOBSat = pSort->nOBSat; int op; sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - if( pSelect->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->opcode = OP_OpenEphemeral; + pSort->sortFlags &= ~SORTFLAG_UseSorter; + nKey = nExpr - pSort->nOBSat + 1; + pOp->p2 = nKey + 1; + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + sqlite3VdbeAddOp4(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat, + (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ClearEphem, pSort->iECursor); + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } - sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); + sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); if( pSelect->iLimit ){ @@ -459,8 +523,8 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr2); } } @@ -534,19 +598,6 @@ static int checkForMultiColumnSelectError( } #endif -/* -** An instance of the following object is used to record information about -** how to process the DISTINCT keyword, to simplify passing that information -** into the selectInnerLoop() routine. -*/ -typedef struct DistinctCtx DistinctCtx; -struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ - u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ - int tabTnct; /* Ephemeral table used for DISTINCT processing */ - int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ -}; - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -561,8 +612,7 @@ static void selectInnerLoop( Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int nOBSat, /* Terms of ORDER BY already satisfied */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ @@ -579,7 +629,8 @@ static void selectInnerLoop( assert( v ); assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; - if( pOrderBy==0 && !hasDistinct ){ + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ codeOffset(v, p->iOffset, iContinue); } @@ -669,7 +720,7 @@ static void selectInnerLoop( break; } } - if( pOrderBy==0 ){ + if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } } @@ -717,11 +768,11 @@ static void selectInnerLoop( int addr = sqlite3VdbeCurrentAddr(v) + 4; sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); - assert( pOrderBy==0 ); + assert( pSort==0 ); } #endif - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, r1); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, r1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -742,12 +793,12 @@ static void selectInnerLoop( assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); - if( pOrderBy ){ + if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -772,8 +823,8 @@ static void selectInnerLoop( */ case SRT_Mem: { assert( nResultCol==1 ); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, regResult); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -786,10 +837,10 @@ static void selectInnerLoop( case SRT_Output: { /* Return the results */ testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); - if( pOrderBy ){ + if( pSort ){ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pOrderBy, p, r1); + pushOntoSorter(pParse, pSort, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); @@ -866,7 +917,7 @@ static void selectInnerLoop( ** there is a sorter, in which case the sorter has already limited ** the output for us. */ - if( pOrderBy==0 && p->iLimit ){ + if( pSort==0 && p->iLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } } @@ -937,7 +988,12 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; @@ -945,10 +1001,10 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); - for(i=0, pItem=pList->a; ia; ipExpr); if( !pColl ) pColl = db->pDfltColl; @@ -1055,6 +1111,7 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ + SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ @@ -1062,17 +1119,23 @@ static void generateSortTail( int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; + int addrOnce = 0; int iTab; int pseudoTab = 0; - ExprList *pOrderBy = p->pOrderBy; - + ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; - int regRow; int regRowid; + int nKey; - iTab = pOrderBy->iECursor; + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ pseudoTab = pParse->nTab++; @@ -1081,20 +1144,23 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - if( p->selFlags & SF_UseSorter ){ + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); } switch( eDest ){ case SRT_Table: @@ -1149,11 +1215,12 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - if( p->selFlags & SF_UseSorter ){ + if( pSort->sortFlags & SORTFLAG_UseSorter ){ sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); @@ -1919,7 +1986,7 @@ static void generateWithRecursiveQuery( addrCont = sqlite3VdbeMakeLabel(v); codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, - 0, 0, 0, pDest, addrCont, addrBreak); + 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); VdbeCoverage(v); @@ -2193,7 +2260,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, - 0, 0, 0, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -2271,7 +2338,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, - 0, 0, 0, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -4310,7 +4377,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -4465,12 +4532,11 @@ int sqlite3Select( ExprList *pEList; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ - ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ int rc = 1; /* Value to return from this function */ - int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -4497,7 +4563,8 @@ int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - pOrderBy = p->pOrderBy; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ @@ -4619,7 +4686,7 @@ int sqlite3Select( pParse->nHeight -= sqlite3SelectExprHeight(p); pTabList = p->pSrc; if( !IgnorableOrderby(pDest) ){ - pOrderBy = p->pOrderBy; + sSort.pOrderBy = p->pOrderBy; } } pEList = p->pEList; @@ -4646,9 +4713,9 @@ int sqlite3Select( ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER ** to disable this optimization for testing purposes. */ - if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy, -1)==0 + if( sqlite3ExprListCompare(p->pGroupBy, sSort.pOrderBy, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ - pOrderBy = 0; + sSort.pOrderBy = 0; } /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and @@ -4667,12 +4734,12 @@ int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - pOrderBy = 0; + sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ @@ -4686,16 +4753,16 @@ int sqlite3Select( ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); - pOrderBy->iECursor = pParse->nTab++; - p->addrOpenEphm[2] = addrSortIndex = + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+2, 0, + sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); }else{ - addrSortIndex = -1; + sSort.addrSortIndex = -1; } /* If the output is destined for a temporary table, open that table. @@ -4709,9 +4776,9 @@ int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); - if( p->iLimit==0 && addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; - p->selFlags |= SF_UseSorter; + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sSort.sortFlags |= SORTFLAG_UseSorter; } /* Open a virtual index to use for the distinct set. @@ -4720,7 +4787,7 @@ int sqlite3Select( sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList, 0), + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; @@ -4731,11 +4798,10 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); - int nOBSat; /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, - wctrlFlags, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -4743,20 +4809,23 @@ int sqlite3Select( if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } - nOBSat = sqlite3WhereIsOrdered(pWInfo); - if( pOrderBy && nOBSat==pOrderBy->nExpr ){ pOrderBy = 0; nOBSat = 0; } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ - if( addrSortIndex>=0 && pOrderBy==0 ){ - sqlite3VdbeChangeToNoop(v, addrSortIndex); - p->addrOpenEphm[2] = -1; + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, pOrderBy, nOBSat, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4812,7 +4881,7 @@ int sqlite3Select( sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); - sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } @@ -4846,7 +4915,7 @@ int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -4875,7 +4944,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -5029,7 +5098,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, 0, + selectInnerLoop(pParse, p, p->pEList, -1, &sSort, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5170,9 +5239,9 @@ int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); } - pOrderBy = 0; + sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -5187,9 +5256,9 @@ int sqlite3Select( /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, pEList->nExpr, pDest); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } /* Jump here to skip this query diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ee39e5278..54aec9ccaa 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1958,7 +1958,6 @@ struct Expr { */ struct ExprList { int nExpr; /* Number of expressions on the list */ - int iECursor; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The list of expressions */ char *zName; /* Token associated with this expression */ @@ -2182,7 +2181,7 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ - int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -2206,9 +2205,9 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x0040 /* Sort using a sorter */ + /* 0x0040 NOT USED */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ -#define SF_Materialize 0x0100 /* NOT USED */ + /* 0x0100 NOT USED */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ diff --git a/src/vdbe.c b/src/vdbe.c index c58b5d9fda..9f02fa796c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1080,10 +1080,11 @@ case OP_Variable: { /* out2-prerelease */ /* Opcode: Move P1 P2 P3 * * ** Synopsis: r[P2@P3]=r[P1@P3] ** -** Move the values in register P1..P1+P3 over into -** registers P2..P2+P3. Registers P1..P1+P3 are +** Move the P3 values in register P1..P1+P3-1 over into +** registers P2..P2+P3-1. Registers P1..P1+P3-1 are ** left holding a NULL. It is an error for register ranges -** P1..P1+P3 and P2..P2+P3 to overlap. +** P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error +** for P3 to be less than 1. */ case OP_Move: { char *zMalloc; /* Holding variable for allocated memory */ @@ -1094,7 +1095,7 @@ case OP_Move: { n = pOp->p3; p1 = pOp->p1; p2 = pOp->p2; - assert( n>=0 && p1>0 && p2>0 ); + assert( n>0 && p1>0 && p2>0 ); assert( p1+n<=p2 || p2+n<=p1 ); pIn1 = &aMem[p1]; @@ -1118,7 +1119,7 @@ case OP_Move: { REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; - }while( n-- ); + }while( --n ); break; } @@ -1995,6 +1996,7 @@ case OP_Permutation: { } /* Opcode: Compare P1 P2 P3 P4 P5 +** Synopsis: r[P1@P3] <-> r[P2@P3] ** ** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this ** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of @@ -3330,6 +3332,7 @@ case OP_OpenEphemeral: { pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; + pCx->isEphemeral = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ @@ -3820,7 +3823,7 @@ case OP_NotExists: { /* jump, in3 */ } /* Opcode: Sequence P1 P2 * * * -** Synopsis: r[P2]=rowid +** Synopsis: r[P2]=cursor[P1].ctr++ ** ** Find the next available sequence number for cursor P1. ** Write the sequence number into register P2. @@ -4869,6 +4872,23 @@ case OP_Clear: { break; } +/* Opcode: ClearEphem P1 * * * * +** +** Delete all contents from the ephemeral table that is open on cursor P1. +** +** See also: Clear, Destroy +*/ +case OP_ClearEphem: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + break; +} + /* Opcode: CreateTable P1 P2 * * * ** Synopsis: r[P2]=root iDb=P1 ** diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 0394782f73..dc40187833 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -72,6 +72,7 @@ struct VdbeCursor { u8 nullRow; /* True if pointing to a row with no data */ u8 rowidIsValid; /* True if lastRowid is valid */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ Bool isTable:1; /* True if a table requiring integer keys */ Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a641a72ec8..6f1fa2670b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -783,7 +783,9 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED || pOp->p4type==P4_INT32 ); + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_INT32 + || pOp->p4type==P4_KEYINFO ); freeP4(db, pOp->p4type, pOp->p4.p); pOp->p4.p = 0; if( n==P4_INT32 ){ diff --git a/src/where.c b/src/where.c index 9d83636f4b..a0bc027d97 100644 --- a/src/where.c +++ b/src/where.c @@ -5198,9 +5198,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } - if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + if( pWInfo->pOrderBy ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } }else{ pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; From ea6c36e2041410d64292bdc3493124dd457eb20c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 14:30:55 +0000 Subject: [PATCH 087/177] Make sure the where.c query planner never reports that the number of ORDER BY terms that are satisfied by indices is negative. FossilOrigin-Name: b186d8d15a10d5e8fcae9ae72e8624d71831f5bd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a367d8b9b9..cdb6bd8483 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sattempt\sat\sgetting\sblock-sort\sto\swork.\s\sThis\sis\san\sincremental\scheck-in.\nThere\sare\smany\sproblems\sstill\sto\sbe\sworked\sout. -D 2014-03-19T14:10:55.625 +C Make\ssure\sthe\swhere.c\squery\splanner\snever\sreports\sthat\sthe\snumber\sof\sORDER\sBY\nterms\sthat\sare\ssatisfied\sby\sindices\sis\snegative. +D 2014-03-19T14:30:55.491 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4739c3900e4cb99dfd7d19853e6b4fb63c9d342 +F src/where.c 1757f31c7f1c59a35ac8ffe931d22d66b43f9f1f F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e258df236b7de70087c8227cb209080e55b9bf9c -R edda8149383dd364d2984f7033840ec9 +P 59742dd4c5259883850044d0938248b009ebd045 +R 48f9fc76ec5916ff8df0bfaab631c0e0 U drh -Z c306e7ec68619896c760c46adb63dbc2 +Z 42480fcdcdf9fedc12ae658e080be0a9 diff --git a/manifest.uuid b/manifest.uuid index 837615cb34..ee59a2b175 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59742dd4c5259883850044d0938248b009ebd045 \ No newline at end of file +b186d8d15a10d5e8fcae9ae72e8624d71831f5bd \ No newline at end of file diff --git a/src/where.c b/src/where.c index a0bc027d97..740e1607e0 100644 --- a/src/where.c +++ b/src/where.c @@ -5205,6 +5205,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } }else{ pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; } } From ba194369add5ddbbacda3edb43116960b59e301d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 15:17:24 +0000 Subject: [PATCH 088/177] Avoid reusing temporary registers in co-routines when doing block-sorting. FossilOrigin-Name: 463d921b519b08648a715e987c430f06b06bae84 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cdb6bd8483..34c7b8395d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\swhere.c\squery\splanner\snever\sreports\sthat\sthe\snumber\sof\sORDER\sBY\nterms\sthat\sare\ssatisfied\sby\sindices\sis\snegative. -D 2014-03-19T14:30:55.491 +C Avoid\sreusing\stemporary\sregisters\sin\sco-routines\swhen\sdoing\nblock-sorting. +D 2014-03-19T15:17:24.171 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bd6ac5dc27046877f53065ea25f36ca90db4c594 +F src/select.c 45d3ac1273b15e98f94c261bd40eb54a8eff0ac7 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59742dd4c5259883850044d0938248b009ebd045 -R 48f9fc76ec5916ff8df0bfaab631c0e0 +P b186d8d15a10d5e8fcae9ae72e8624d71831f5bd +R b0910f2ab36ae08486561cabd81d63b0 U drh -Z 42480fcdcdf9fedc12ae658e080be0a9 +Z 0471d95e83a3c5ea30582909f120da3f diff --git a/manifest.uuid b/manifest.uuid index ee59a2b175..6cd642341c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b186d8d15a10d5e8fcae9ae72e8624d71831f5bd \ No newline at end of file +463d921b519b08648a715e987c430f06b06bae84 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b3f7f861bc..087cd401f0 100644 --- a/src/select.c +++ b/src/select.c @@ -509,8 +509,10 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + if( nOBSat==0 ){ + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; From 26d7e7c6b7dd234b5ae4cf6dd4ce96ed98b078c7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 16:56:58 +0000 Subject: [PATCH 089/177] Fix a problem in the block-sort logic that can arise if the VDBE opcode array is resized while the block-sort logic is being coded. "make test" now runs to completion, though there are still 17 failures. FossilOrigin-Name: 62f3a22050e0f11c789374f8a3785df0a84c24cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 34c7b8395d..2fa2da7098 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sreusing\stemporary\sregisters\sin\sco-routines\swhen\sdoing\nblock-sorting. -D 2014-03-19T15:17:24.171 +C Fix\sa\sproblem\sin\sthe\sblock-sort\slogic\sthat\scan\sarise\sif\sthe\sVDBE\sopcode\narray\sis\sresized\swhile\sthe\sblock-sort\slogic\sis\sbeing\scoded.\s\s"make\stest"\nnow\sruns\sto\scompletion,\sthough\sthere\sare\sstill\s17\sfailures. +D 2014-03-19T16:56:58.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 45d3ac1273b15e98f94c261bd40eb54a8eff0ac7 +F src/select.c 0c5df3eb627983064207c543db7d46fc960ca3a4 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b186d8d15a10d5e8fcae9ae72e8624d71831f5bd -R b0910f2ab36ae08486561cabd81d63b0 +P 463d921b519b08648a715e987c430f06b06bae84 +R 0046fa449ec804e57106e3417f064519 U drh -Z 0471d95e83a3c5ea30582909f120da3f +Z 208b34a8fc9197aeefd7f546d5d68bb0 diff --git a/manifest.uuid b/manifest.uuid index 6cd642341c..9945baac14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -463d921b519b08648a715e987c430f06b06bae84 \ No newline at end of file +62f3a22050e0f11c789374f8a3785df0a84c24cb \ No newline at end of file diff --git a/src/select.c b/src/select.c index 087cd401f0..ca54ae635f 100644 --- a/src/select.c +++ b/src/select.c @@ -482,16 +482,16 @@ static void pushOntoSorter( VdbeOp *pOp; /* Opcode that opens the sorter */ int nKey; /* Number of sorting key columns, including OP_Sequence */ + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + 1; addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); pOp->opcode = OP_OpenEphemeral; pSort->sortFlags &= ~SORTFLAG_UseSorter; - nKey = nExpr - pSort->nOBSat + 1; pOp->p2 = nKey + 1; - regPrevKey = pParse->nMem+1; - pParse->nMem += pSort->nOBSat; - sqlite3VdbeAddOp4(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat, - (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); From 65ea12cb4aef6f33970e74d4de705c5a7628f622 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 17:41:36 +0000 Subject: [PATCH 090/177] Make it possible for block-sort to use the OP_SorterOpen sorter in addition to a generic OP_OpenEphemeral. FossilOrigin-Name: 7ce2daafd39a117041bfdd0a7132e2764fe7a74b --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 4 +--- src/vdbe.c | 18 ++++++++++++------ src/vdbeInt.h | 1 + src/vdbesort.c | 39 ++++++++++++++++++++++++++++----------- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 2fa2da7098..d2192d8683 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sblock-sort\slogic\sthat\scan\sarise\sif\sthe\sVDBE\sopcode\narray\sis\sresized\swhile\sthe\sblock-sort\slogic\sis\sbeing\scoded.\s\s"make\stest"\nnow\sruns\sto\scompletion,\sthough\sthere\sare\sstill\s17\sfailures. -D 2014-03-19T16:56:58.204 +C Make\sit\spossible\sfor\sblock-sort\sto\suse\sthe\sOP_SorterOpen\ssorter\sin\saddition\nto\sa\sgeneric\sOP_OpenEphemeral. +D 2014-03-19T17:41:36.148 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0c5df3eb627983064207c543db7d46fc960ca3a4 +F src/select.c f481ab847cfb3ad2b2dce671a4dcbfdb50f8e666 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -278,14 +278,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c e0fe939465f101a6f30f1864813698edc57f610e +F src/vdbe.c 37cfae03a0c40515304fb0a8a1a97ac3aa965e87 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h aa7ea20696609966c71a4b58c6a4fdb974c5e877 +F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 553537f632e8925bb8a2563692d0ee6e2175018c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 463d921b519b08648a715e987c430f06b06bae84 -R 0046fa449ec804e57106e3417f064519 +P 62f3a22050e0f11c789374f8a3785df0a84c24cb +R 83d570cd19fdb25aa0d409b1a1586b4b U drh -Z 208b34a8fc9197aeefd7f546d5d68bb0 +Z 5d298c6da64eb9d6d4284253fdadfab0 diff --git a/manifest.uuid b/manifest.uuid index 9945baac14..88b98e78a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62f3a22050e0f11c789374f8a3785df0a84c24cb \ No newline at end of file +7ce2daafd39a117041bfdd0a7132e2764fe7a74b \ No newline at end of file diff --git a/src/select.c b/src/select.c index ca54ae635f..6ea206d3f0 100644 --- a/src/select.c +++ b/src/select.c @@ -488,8 +488,6 @@ static void pushOntoSorter( addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); - pOp->opcode = OP_OpenEphemeral; - pSort->sortFlags &= ~SORTFLAG_UseSorter; pOp->p2 = nKey + 1; sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); @@ -498,7 +496,7 @@ static void pushOntoSorter( pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp1(v, OP_ClearEphem, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); sqlite3VdbeJumpHere(v, addrFirst); sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); diff --git a/src/vdbe.c b/src/vdbe.c index 9f02fa796c..7104c6d017 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4872,20 +4872,26 @@ case OP_Clear: { break; } -/* Opcode: ClearEphem P1 * * * * +/* Opcode: ResetSorter P1 * * * * ** -** Delete all contents from the ephemeral table that is open on cursor P1. +** Delete all contents from the ephemeral table or sorter +** that is open on cursor P1. ** -** See also: Clear, Destroy +** This opcode only works for cursors used for sorting and +** opened with OP_OpenEphemeral or OP_SorterOpen. */ -case OP_ClearEphem: { +case OP_ResetSorter: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->isEphemeral ); - rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + if( pC->pSorter ){ + sqlite3VdbeSorterReset(db, pC->pSorter); + }else{ + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + } break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index dc40187833..b752478035 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -438,6 +438,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..8b6ecc5e0c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -504,23 +504,40 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } } +/* +** Reset a sorting cursor back to its original empty state. +*/ +void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ + if( pSorter->aIter ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(db, &pSorter->aIter[i]); + } + sqlite3DbFree(db, pSorter->aIter); + pSorter->aIter = 0; + } + if( pSorter->pTemp1 ){ + sqlite3OsCloseFree(pSorter->pTemp1); + pSorter->pTemp1 = 0; + } + vdbeSorterRecordFree(db, pSorter->pRecord); + pSorter->pRecord = 0; + pSorter->iWriteOff = 0; + pSorter->iReadOff = 0; + pSorter->nInMemory = 0; + pSorter->nTree = 0; + pSorter->nPMA = 0; + pSorter->aTree = 0; +} + + /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); - } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - } - vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; From 344510e62b113a570f1b2bc6c1a59c5de8ccd51a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Mar 2014 20:01:25 +0000 Subject: [PATCH 091/177] Avoid some unnecessary calls to sqlite3VdbeRecordUnpack() that were being made when merging data from two or more temp files together in vdbesort.c FossilOrigin-Name: 707ea170b3e26965b7e3982f7554d122d130b9a6 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/vdbesort.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bb21639801..8ad063c938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. -D 2014-03-17T15:06:57.359 +C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c +D 2014-03-19T20:01:25.712 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 0a9e5e63d7ce196651610d1a264eeb0b2152d3ba F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 -R f35820126ad6a1c360258186873a2e79 -U drh -Z b6382160ccdf74a44bc9653763bb82ef +P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 +R 597feeb3c9b32d5be1e5c696f1b077da +U dan +Z 03e3772c0627172d43f05f231b29ede3 diff --git a/manifest.uuid b/manifest.uuid index 3e3e1d6ac7..5d0bb22e62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file +707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..f971c2d933 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -956,14 +956,55 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( pSorter->aTree ){ int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pCsr, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pSorter->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pSorter->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. */ + if( iRes<=0 ){ + pSorter->aTree[i] = (pIter1 - pSorter->aIter); + pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pSorter->aTree[i] = (pIter2 - pSorter->aIter); + pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + } + + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + } }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; From 6284db908a0249feef9fba346e2a6bab57603861 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 23:24:49 +0000 Subject: [PATCH 092/177] Fix query planner weights associated with choosing block-sorting. Fix block sorting of tables with collating functions. Fix various test cases. All "veryquick" tests are now passing, though more tests need to be added. FossilOrigin-Name: 01afbf97c0ff29667806e9a7c4d74ca717819de5 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/select.c | 8 ++++---- src/where.c | 12 +++++++++--- test/distinct.test | 2 +- test/orderby5.test | 22 ++++++++++++++++++++-- test/whereG.test | 4 ++-- 7 files changed, 47 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d2192d8683..f766ef6eb9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sit\spossible\sfor\sblock-sort\sto\suse\sthe\sOP_SorterOpen\ssorter\sin\saddition\nto\sa\sgeneric\sOP_OpenEphemeral. -D 2014-03-19T17:41:36.148 +C Fix\squery\splanner\sweights\sassociated\swith\schoosing\sblock-sorting.\s\sFix\nblock\ssorting\sof\stables\swith\scollating\sfunctions.\s\sFix\svarious\stest\scases.\nAll\s"veryquick"\stests\sare\snow\spassing,\sthough\smore\stests\sneed\sto\sbe\sadded. +D 2014-03-19T23:24:49.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c f481ab847cfb3ad2b2dce671a4dcbfdb50f8e666 +F src/select.c 12534449e77ff54fca5d3e9fcddc10031b13fbc1 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 1757f31c7f1c59a35ac8ffe931d22d66b43f9f1f +F src/where.c 7c74debe2e76711082d6cb503aa89dd2afaf84cf F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -431,7 +431,7 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test c7b194ef95dbddb32d77acbbab2e023c6eed0cb2 +F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test ee95d48664503d40f6cc9ef4a7d03216188e2ada F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -722,7 +722,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 0eb82d5890c3f3d0563966560cfdc984ea69e30c +F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1089,7 +1089,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2a3d5181decc801b36600fa1c40b0dad2ccc267f +F test/whereG.test eb3a46b3eaf38e25e3013433b2db8a25a866c215 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 62f3a22050e0f11c789374f8a3785df0a84c24cb -R 83d570cd19fdb25aa0d409b1a1586b4b +P 7ce2daafd39a117041bfdd0a7132e2764fe7a74b +R f3bf694b0186fa15aaedfb1a5982c72f U drh -Z 5d298c6da64eb9d6d4284253fdadfab0 +Z 15ce545d4c6482a35d82cbe6b958fe2e diff --git a/manifest.uuid b/manifest.uuid index 88b98e78a6..a006d6c80d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce2daafd39a117041bfdd0a7132e2764fe7a74b \ No newline at end of file +01afbf97c0ff29667806e9a7c4d74ca717819de5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 6ea206d3f0..bb72e1ae70 100644 --- a/src/select.c +++ b/src/select.c @@ -1004,12 +1004,12 @@ static KeyInfo *keyInfoFromExprList( pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); - for(i=iStart, pItem=pList->a; ia+iStart; ipExpr); if( !pColl ) pColl = db->pDfltColl; - pInfo->aColl[i] = pColl; - pInfo->aSortOrder[i] = pItem->sortOrder; + pInfo->aColl[i-iStart] = pColl; + pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; @@ -5257,7 +5257,7 @@ int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, "ORDER BY"); + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } diff --git a/src/where.c b/src/where.c index 740e1607e0..ee608dd83f 100644 --- a/src/where.c +++ b/src/where.c @@ -5039,11 +5039,17 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrderedwctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost, diff --git a/test/distinct.test b/test/distinct.test index 5198113920..78c2c1df36 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -162,7 +162,7 @@ do_execsql_test 2.0 { foreach {tn sql temptables res} { 1 "a, b FROM t1" {} {A B a b} 2 "b, a FROM t1" {} {B A b a} - 3 "a, b, c FROM t1" {hash} {a b c A B C} + 3 "a, b, c FROM t1" {hash} {A B C a b c} 4 "a, b, c FROM t1 ORDER BY a, b, c" {btree} {A B C a b c} 5 "b FROM t1 WHERE a = 'a'" {} {b} 6 "b FROM t1 ORDER BY +b COLLATE binary" {btree hash} {B b} diff --git a/test/orderby5.test b/test/orderby5.test index 6da0dd4e76..bccd469f25 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -64,10 +64,28 @@ do_execsql_test 1.7 { EXPLAIN QUERY PLAN SELECT DISTINCT c, b, a FROM t1 WHERE +a=0; } {/B-TREE/} -do_execsql_test 2.1 { + +# In some cases, it is faster to do repeated index lookups than it is to +# sort. But in other cases, it is faster to sort than to do repeated index +# lookups. +# +do_execsql_test 2.1a { + CREATE TABLE t2(a,b,c); + CREATE INDEX t2bc ON t2(b,c); + ANALYZE; + INSERT INTO sqlite_stat1 VALUES('t1','t1bc','1000000 10 9'); + INSERT INTO sqlite_stat1 VALUES('t2','t2bc','100 10 5'); + ANALYZE sqlite_master; + + EXPLAIN QUERY PLAN + SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; +} {~/B-TREE/} +do_execsql_test 2.1b { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; -} {~/B-TREE/} +} {/B-TREE/} + + do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; diff --git a/test/whereG.test b/test/whereG.test index 1b6a861855..490fffe64e 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -95,7 +95,7 @@ do_eqp_test whereG-1.5 { WHERE cname LIKE '%bach%' AND composer.cid=track.cid AND album.aid=track.aid; -} {/.*track.*composer.*album.*/} +} {/.*track.*(composer.*album|album.*composer).*/} do_execsql_test whereG-1.6 { SELECT DISTINCT aname FROM album, composer, track @@ -110,7 +110,7 @@ do_eqp_test whereG-1.7 { WHERE cname LIKE '%bach%' AND unlikely(composer.cid=track.cid) AND unlikely(album.aid=track.aid); -} {/.*track.*composer.*album.*/} +} {/.*track.*(composer.*album|album.*composer).*/} do_execsql_test whereG-1.8 { SELECT DISTINCT aname FROM album, composer, track From 781e34cde3908108e2cadf2c727e8fe0e8743961 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Mar 2014 08:59:47 +0000 Subject: [PATCH 093/177] Add an experimental fix to avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. FossilOrigin-Name: 6f3a5c24d254fc6faf607b505bdef4a7aafc21af --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/os_unix.c | 71 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 8ad063c938..0276975e81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c -D 2014-03-19T20:01:25.712 +C Add\san\sexperimental\sfix\sto\savoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. +D 2014-03-20T08:59:47.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 +F src/os_unix.c 7e2f6348e99bd215d36cb5d40161b06456089e21 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 597feeb3c9b32d5be1e5c696f1b077da +P 707ea170b3e26965b7e3982f7554d122d130b9a6 +R df2b4ee35d794d54be353b7886f4773f +T *branch * shm-mapping-fix +T *sym-shm-mapping-fix * +T -sym-trunk * U dan -Z 03e3772c0627172d43f05f231b29ede3 +Z 8f1167a615e2deebf3a41d21381f8bfc diff --git a/manifest.uuid b/manifest.uuid index 5d0bb22e62..b57a947167 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file +6f3a5c24d254fc6faf607b505bdef4a7aafc21af \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index deb9e51d07..151ed890ae 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4105,6 +4105,33 @@ static int unixShmSystemLock( return rc; } +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = unixGetPagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ + int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->mutex); - for(i=0; inRegion; i++){ + for(i=0; inRegion; i+=nShmPerMap){ if( p->h>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ @@ -4326,6 +4354,8 @@ static int unixShmMap( unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; /* If the shared-memory file has not yet been opened, open it now. */ if( pDbFd->pShm==0 ){ @@ -4341,9 +4371,12 @@ static int unixShmMap( assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); - if( pShmNode->nRegion<=iRegion ){ + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; @@ -4392,17 +4425,19 @@ static int unixShmMap( /* Map the requested memory region into this processes address space. */ apNew = (char **)sqlite3_realloc( - pShmNode->apRegion, (iRegion+1)*sizeof(char *) + pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->apRegion = apNew; - while(pShmNode->nRegion<=iRegion){ + while( pShmNode->nRegionh>=0 ){ - pMem = osMmap(0, szRegion, + pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4418,8 +4453,11 @@ static int unixShmMap( } memset(pMem, 0, szRegion); } - pShmNode->apRegion[pShmNode->nRegion] = pMem; - pShmNode->nRegion++; + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; } } @@ -4633,19 +4671,6 @@ static void unixUnmapfile(unixFile *pFd){ } } -/* -** Return the system page size. -*/ -static int unixGetPagesize(void){ -#if HAVE_MREMAP - return 512; -#elif defined(_BSD_SOURCE) - return getpagesize(); -#else - return (int)sysconf(_SC_PAGESIZE); -#endif -} - /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4682,8 +4707,12 @@ static void unixRemapfile( if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; if( pOrig ){ +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else const int szSyspage = unixGetPagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif u8 *pReq = &pOrig[nReuse]; /* Unmap any pages of the existing mapping that cannot be reused. */ From bc76063cd2d0c7bdf59965af73a67a2ed1181641 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Mar 2014 09:42:09 +0000 Subject: [PATCH 094/177] Add a test to ensure os_unix.c works with 64KiB OS pages. FossilOrigin-Name: e3d2be3ba47cdaafd26347620ae3bc2813203f16 --- manifest | 20 +++++++++---------- manifest.uuid | 2 +- src/os_unix.c | 15 ++++++++++---- src/test_syscall.c | 49 +++++++++++++++++++++++++++++++++++++++++++++- test/syscall.test | 1 + test/wal64k.test | 47 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 test/wal64k.test diff --git a/manifest b/manifest index 0276975e81..5dfca569cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sfix\sto\savoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. -D 2014-03-20T08:59:47.455 +C Add\sa\stest\sto\sensure\sos_unix.c\sworks\swith\s64KiB\sOS\spages. +D 2014-03-20T09:42:09.744 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 7e2f6348e99bd215d36cb5d40161b06456089e21 +F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -266,7 +266,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c 9898687a6c2beca733b0dd6fe19163d987826d31 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c 16dbe79fb320fadb5acd7a0a59f49e52ab2d2091 +F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 @@ -835,7 +835,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 +F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 F test/table.test 580d23530187026d4502fae74a490f0408cf2cc7 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -1054,6 +1054,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e +F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 @@ -1156,10 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 707ea170b3e26965b7e3982f7554d122d130b9a6 -R df2b4ee35d794d54be353b7886f4773f -T *branch * shm-mapping-fix -T *sym-shm-mapping-fix * -T -sym-trunk * +P 6f3a5c24d254fc6faf607b505bdef4a7aafc21af +R 20782a7f4b1e1e0590156d829fb7cb71 U dan -Z 8f1167a615e2deebf3a41d21381f8bfc +Z 523769a43e90bd1d8c3037ea221853f3 diff --git a/manifest.uuid b/manifest.uuid index b57a947167..40e6d382ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f3a5c24d254fc6faf607b505bdef4a7aafc21af \ No newline at end of file +e3d2be3ba47cdaafd26347620ae3bc2813203f16 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 151ed890ae..fc320a4926 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -323,6 +323,7 @@ static int posixFchown(int fd, uid_t uid, gid_t gid){ /* Forward reference */ static int openDirectory(const char*, int*); +static int unixGetpagesize(void); /* ** Many system calls are accessed through pointer-to-functions so that @@ -446,6 +447,9 @@ static struct unix_syscall { #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) #endif + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4107,8 +4111,11 @@ static int unixShmSystemLock( /* ** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). */ -static int unixGetPagesize(void){ +static int unixGetpagesize(void){ #if defined(_BSD_SOURCE) return getpagesize(); #else @@ -4127,7 +4134,7 @@ static int unixGetPagesize(void){ */ static int unixShmRegionPerMap(void){ int shmsz = 32*1024; /* SHM region size */ - int pgsz = unixGetPagesize(); /* System page size */ + int pgsz = osGetpagesize(); /* System page size */ assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ if( pgszmmapSize; #else - const int szSyspage = unixGetPagesize(); + const int szSyspage = osGetpagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); #endif u8 *pReq = &pOrig[nReuse]; @@ -7458,7 +7465,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==24 ); + assert( ArraySize(aSyscall)==25 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/test_syscall.c b/src/test_syscall.c index 7c0873c16d..0dac2e897e 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -67,6 +67,11 @@ ** test_syscall list ** Return a list of all system calls. The list is constructed using ** the xNextSystemCall() VFS method. +** +** test_syscall pagesize PGSZ +** If PGSZ is a power of two greater than 256, install a wrapper around +** OS function getpagesize() that reports the system page size as PGSZ. +** Or, if PGSZ is less than zero, remove any wrapper already installed. */ #include "sqliteInt.h" @@ -89,7 +94,9 @@ static struct TestSyscallGlobal { int bPersist; /* 1 for persistent errors, 0 for transient */ int nCount; /* Fail after this many more calls */ int nFail; /* Number of failures that have occurred */ -} gSyscall = { 0, 0 }; + int pgsz; + sqlite3_syscall_ptr orig_getpagesize; +} gSyscall = { 0, 0, 0, 0, 0 }; static int ts_open(const char *, int, int); static int ts_close(int fd); @@ -650,6 +657,45 @@ static int test_syscall_defaultvfs( return TCL_OK; } +static int ts_getpagesize(void){ + return gSyscall.pgsz; +} + +static int test_syscall_pagesize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + int pgsz; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "PGSZ"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &pgsz) ){ + return TCL_ERROR; + } + + if( pgsz<0 ){ + if( gSyscall.orig_getpagesize ){ + pVfs->xSetSystemCall(pVfs, "getpagesize", gSyscall.orig_getpagesize); + } + }else{ + if( pgsz<512 || (pgsz & (pgsz-1)) ){ + Tcl_AppendResult(interp, "pgsz out of range", 0); + return TCL_ERROR; + } + gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize"); + gSyscall.pgsz = pgsz; + pVfs->xSetSystemCall( + pVfs, "getpagesize", (sqlite3_syscall_ptr)ts_getpagesize + ); + } + + return TCL_OK; +} + static int test_syscall( void * clientData, Tcl_Interp *interp, @@ -668,6 +714,7 @@ static int test_syscall( { "exists", test_syscall_exists }, { "list", test_syscall_list }, { "defaultvfs", test_syscall_defaultvfs }, + { "pagesize", test_syscall_pagesize }, { 0, 0 } }; int iCmd; diff --git a/test/syscall.test b/test/syscall.test index 5bf1225867..c2d9979031 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -61,6 +61,7 @@ foreach s { fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown umask mmap munmap mremap + getpagesize } { if {[test_syscall exists $s]} {lappend syscall_list $s} } diff --git a/test/wal64k.test b/test/wal64k.test new file mode 100644 index 0000000000..afd820778e --- /dev/null +++ b/test/wal64k.test @@ -0,0 +1,47 @@ +# 2010 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix wal64k + +ifcapable !wal {finish_test ; return } + +db close +test_syscall pagesize 65536 +sqlite3 db test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + CREATE INDEX i1 ON t1(x); +} {wal} +do_test 1.1 { file size test.db-shm } {65536} + +do_test 1.2 { + execsql BEGIN + while {[file size test.db-shm]==65536} { + execsql { INSERT INTO t1 VALUES( randstr(900,1100) ) } + } + execsql COMMIT + file size test.db-shm +} {131072} + +integrity_check 1.3 + +db close +test_syscall pagesize -1 +finish_test + From 0c8a934cb44f262ee60535db0a0167cb546b2aef Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 12:17:35 +0000 Subject: [PATCH 095/177] Fix an unnecessarily obtuse use of a bitmask flag. FossilOrigin-Name: ca3140813198ab7ce470cf86334e55207f3461f9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8ad063c938..41cb481776 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c -D 2014-03-19T20:01:25.712 +C Fix\san\sunnecessarily\sobtuse\suse\sof\sa\sbitmask\sflag. +D 2014-03-20T12:17:35.051 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 +F src/where.c e433accd201ca482c761e679f4a2ce2f6a7348e6 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 597feeb3c9b32d5be1e5c696f1b077da -U dan -Z 03e3772c0627172d43f05f231b29ede3 +P 707ea170b3e26965b7e3982f7554d122d130b9a6 +R 829b1d227d22721b3fefbad8e2ead1b7 +U drh +Z fcfbdaafabd38e7e728b08b0509fc847 diff --git a/manifest.uuid b/manifest.uuid index 5d0bb22e62..444963914a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file +ca3140813198ab7ce470cf86334e55207f3461f9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6cd9c167a1..d13258ad66 100644 --- a/src/where.c +++ b/src/where.c @@ -3208,8 +3208,7 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - assert( (WHERE_UNQ_WANTED>>16)==1 ); - pLevel->p3 = (pLoop->wsFlags>>16)&1; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ From 2b59b3a4c80207e0c4692a8bbbcac5a0dce0c9d5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 13:26:47 +0000 Subject: [PATCH 096/177] =?UTF-8?q?Convert=20expressions=20of=20the=20form?= =?UTF-8?q?=20"X=20IN=20(=3F)"=20with=20exactly=20one=20value=20on=20the?= =?UTF-8?q?=20RHS=20of=20the=20IN=20into=20equality=20tests:=20=20"X=3D=3F?= =?UTF-8?q?".=20=20Add=20test=20cases=20to=20verify=20that=20statements=20?= =?UTF-8?q?work=20correctly=20on=20this=20corner=20case.=20Fix=20for=20tic?= =?UTF-8?q?ket=20[e39d032577df6942].?= FossilOrigin-Name: e68b427afbc82e201c64474117851aa4c9eb0c92 --- manifest | 16 +++++----- manifest.uuid | 2 +- src/parse.y | 15 +++++++++ src/where.c | 2 ++ test/in4.test | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 41cb481776..16ca144d3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sunnecessarily\sobtuse\suse\sof\sa\sbitmask\sflag. -D 2014-03-20T12:17:35.051 +C Convert\sexpressions\sof\sthe\sform\s"X\sIN\s(?)"\swith\sexactly\sone\svalue\son\sthe\nRHS\sof\sthe\sIN\sinto\sequality\stests:\s\s"X=?".\s\sAdd\stest\scases\sto\sverify\sthat\nstatements\swork\scorrectly\son\sthis\scorner\scase.\nFix\sfor\sticket\s[e39d032577df6942]. +D 2014-03-20T13:26:47.152 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 +F src/parse.y 52d749e7fd1289d9a576fe88782d9637a2f7bb6d F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e433accd201ca482c761e679f4a2ce2f6a7348e6 +F src/where.c a2f20bdc6565a3add38ac5e4d9e71523ed5b2f2b F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 +F test/in4.test ed42587bed8c8e2219a09a6a6e3675edffc011da F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 707ea170b3e26965b7e3982f7554d122d130b9a6 -R 829b1d227d22721b3fefbad8e2ead1b7 +P ca3140813198ab7ce470cf86334e55207f3461f9 +R abb091fc1f04a82c7540caf08b56e8dd U drh -Z fcfbdaafabd38e7e728b08b0509fc847 +Z 8ed34cb3369a2d9b188815f2f4b333aa diff --git a/manifest.uuid b/manifest.uuid index 444963914a..6fc3864b4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3140813198ab7ce470cf86334e55207f3461f9 \ No newline at end of file +e68b427afbc82e201c64474117851aa4c9eb0c92 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index ad9bedf0b4..1e6d859881 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1020,6 +1020,21 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { */ A.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[N]); sqlite3ExprDelete(pParse->db, X.pExpr); + }else if( Y->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + */ + Expr *pRHS = Y->a[0].pExpr; + Y->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, Y); + A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); if( A.pExpr ){ diff --git a/src/where.c b/src/where.c index d13258ad66..e3ce8b8697 100644 --- a/src/where.c +++ b/src/where.c @@ -4009,6 +4009,8 @@ static int whereLoopAddBtreeIndex( /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); } + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ pNew->rRun += nIn; pNew->u.btree.nEq++; pNew->nOut = nRowEst + nInMul + nIn; diff --git a/test/in4.test b/test/in4.test index 470f4f0e34..b8ae4d9825 100644 --- a/test/in4.test +++ b/test/in4.test @@ -159,4 +159,92 @@ do_test in4-3.12 { execsql { SELECT * FROM t3 WHERE x IN (1, 2) AND y IN ()} } {} +# Tests for "... IN (?)" and "... NOT IN (?)". In other words, tests +# for when the RHS of IN is a single expression. This should work the +# same as the == and <> operators. +# +do_execsql_test in4-3.21 { + SELECT * FROM t3 WHERE x=10 AND y IN (10); +} {10 10 10} +do_execsql_test in4-3.22 { + SELECT * FROM t3 WHERE x IN (10) AND y=10; +} {10 10 10} +do_execsql_test in4-3.23 { + SELECT * FROM t3 WHERE x IN (10) AND y IN (10); +} {10 10 10} +do_execsql_test in4-3.24 { + SELECT * FROM t3 WHERE x=1 AND y NOT IN (10); +} {1 1 1} +do_execsql_test in4-3.25 { + SELECT * FROM t3 WHERE x NOT IN (10) AND y=1; +} {1 1 1} +do_execsql_test in4-3.26 { + SELECT * FROM t3 WHERE x NOT IN (10) AND y NOT IN (10); +} {1 1 1} + +# The query planner recognizes that "x IN (?)" only generates a +# single match and can use this information to optimize-out ORDER BY +# clauses. +# +do_execsql_test in4-3.31 { + DROP INDEX t3i1; + CREATE UNIQUE INDEX t3xy ON t3(x,y); + + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x=10 AND A.y IN (10) + AND B.x=1 AND B.y IN (1); +} {10 10 10 1 1 1 |} +do_execsql_test in4-3.32 { + EXPLAIN QUERY PLAN + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x=10 AND A.y IN (10) + AND B.x=1 AND B.y IN (1); +} {~/B-TREE/} ;# No separate sorting pass +do_execsql_test in4-3.33 { + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x IN (10) AND A.y=10 + AND B.x IN (1) AND B.y=1; +} {10 10 10 1 1 1 |} +do_execsql_test in4-3.34 { + EXPLAIN QUERY PLAN + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x IN (10) AND A.y=10 + AND B.x IN (1) AND B.y=1; +} {~/B-TREE/} ;# No separate sorting pass + +# An expression of the form "x IN (?,?)" creates an ephemeral table to +# hold the list of values on the RHS. But "x IN (?)" does not create +# an ephemeral table. +# +do_execsql_test in4-3.41 { + SELECT * FROM t3 WHERE x IN (10,11); +} {10 10 10} +do_execsql_test in4-3.42 { + EXPLAIN + SELECT * FROM t3 WHERE x IN (10,11); +} {/OpenEphemeral/} +do_execsql_test in4-3.43 { + SELECT * FROM t3 WHERE x IN (10); +} {10 10 10} +do_execsql_test in4-3.44 { + EXPLAIN + SELECT * FROM t3 WHERE x IN (10); +} {~/OpenEphemeral/} +do_execsql_test in4-3.45 { + SELECT * FROM t3 WHERE x NOT IN (10,11); +} {1 1 1} +do_execsql_test in4-3.46 { + EXPLAIN + SELECT * FROM t3 WHERE x NOT IN (10,11); +} {/OpenEphemeral/} +do_execsql_test in4-3.47 { + SELECT * FROM t3 WHERE x NOT IN (10); +} {1 1 1} +do_execsql_test in4-3.48 { + EXPLAIN + SELECT * FROM t3 WHERE x NOT IN (10); +} {~/OpenEphemeral/} + + + finish_test From 0a8d14261c7b47208176f750f95b2472fecd63bf Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 14:56:47 +0000 Subject: [PATCH 097/177] =?UTF-8?q?Previous=20check-in=20is=20not=20quite?= =?UTF-8?q?=20correct.=20=20"x=20IN=20(=3F)"=20is=20not=20exactly=20the=20?= =?UTF-8?q?same=20as=20"x=3D=3D=3F"=20do=20to=20collation=20and=20affinity?= =?UTF-8?q?=20issues.=20=20The=20correct=20converstion=20should=20be=20to?= =?UTF-8?q?=20"x=3D=3D(+=3F=20COLLATE=20binary)".=20=20The=20current=20che?= =?UTF-8?q?ck-in=20fixes=20this=20problem=20and=20provides=20test=20cases.?= =?UTF-8?q?=20=20Ticket=20[e39d032577df69]?= FossilOrigin-Name: 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/parse.y | 8 ++++--- test/in4.test | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 16ca144d3e..5d2b629021 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sexpressions\sof\sthe\sform\s"X\sIN\s(?)"\swith\sexactly\sone\svalue\son\sthe\nRHS\sof\sthe\sIN\sinto\sequality\stests:\s\s"X=?".\s\sAdd\stest\scases\sto\sverify\sthat\nstatements\swork\scorrectly\son\sthis\scorner\scase.\nFix\sfor\sticket\s[e39d032577df6942]. -D 2014-03-20T13:26:47.152 +C Previous\scheck-in\sis\snot\squite\scorrect.\s\s"x\sIN\s(?)"\sis\snot\sexactly\sthe\ssame\nas\s"x==?"\sdo\sto\scollation\sand\saffinity\sissues.\s\sThe\scorrect\sconverstion\sshould\nbe\sto\s"x==(+?\sCOLLATE\sbinary)".\s\sThe\scurrent\scheck-in\sfixes\sthis\sproblem\sand\nprovides\stest\scases.\s\sTicket\s[e39d032577df69] +D 2014-03-20T14:56:47.071 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 52d749e7fd1289d9a576fe88782d9637a2f7bb6d +F src/parse.y c9189eca0888ecd2711c77a199ae7a7c86f57546 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test ed42587bed8c8e2219a09a6a6e3675edffc011da +F test/in4.test 18202389003284e8e019750c04e4bc6333df9c99 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ca3140813198ab7ce470cf86334e55207f3461f9 -R abb091fc1f04a82c7540caf08b56e8dd +P e68b427afbc82e201c64474117851aa4c9eb0c92 +R 8739dc742c6ff38e89fff08724b87c5b U drh -Z 8ed34cb3369a2d9b188815f2f4b333aa +Z c6097cc93dfeb105af183d7216a869ad diff --git a/manifest.uuid b/manifest.uuid index 6fc3864b4f..7bb62d8aab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e68b427afbc82e201c64474117851aa4c9eb0c92 \ No newline at end of file +2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 1e6d859881..24b8b98afc 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1028,12 +1028,14 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** ** with exactly one value on the RHS can be simplified to: ** - ** expr1 == ?1 - ** expr1 <> ?2 + ** expr1 == (+?1 COLLATE binary) + ** expr1 <> (+?2 COLLATE binary) */ - Expr *pRHS = Y->a[0].pExpr; + static const Token collBin = { "binary", 6 }; + Expr *pRHS = sqlite3ExprAddCollateToken(pParse, Y->a[0].pExpr, &collBin); Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); diff --git a/test/in4.test b/test/in4.test index b8ae4d9825..5828a83cc5 100644 --- a/test/in4.test +++ b/test/in4.test @@ -245,6 +245,65 @@ do_execsql_test in4-3.48 { SELECT * FROM t3 WHERE x NOT IN (10); } {~/OpenEphemeral/} +# Make sure that when "x IN (?)" is converted into "x==?" that collating +# sequence and affinity computations do not get messed up. +# +do_execsql_test in4-4.1 { + CREATE TABLE t4a(a TEXT, b TEXT COLLATE nocase, c); + INSERT INTO t4a VALUES('ABC','abc',1); + INSERT INTO t4a VALUES('def','xyz',2); + INSERT INTO t4a VALUES('ghi','ghi',3); + SELECT c FROM t4a WHERE a=b ORDER BY c; +} {3} +do_execsql_test in4-4.2 { + SELECT c FROM t4a WHERE b=a ORDER BY c; +} {1 3} +do_execsql_test in4-4.3 { + SELECT c FROM t4a WHERE (a||'')=b ORDER BY c; +} {1 3} +do_execsql_test in4-4.4 { + SELECT c FROM t4a WHERE (a||'')=(b||'') ORDER BY c; +} {3} +do_execsql_test in4-4.5 { + SELECT c FROM t4a WHERE a IN (b) ORDER BY c; +} {3} +do_execsql_test in4-4.6 { + SELECT c FROM t4a WHERE (a||'') IN (b) ORDER BY c; +} {3} + + +do_execsql_test in4-4.11 { + CREATE TABLE t4b(a TEXT, b NUMERIC, c); + INSERT INTO t4b VALUES('1.0',1,4); + SELECT c FROM t4b WHERE a=b; +} {4} +do_execsql_test in4-4.12 { + SELECT c FROM t4b WHERE b=a; +} {4} +do_execsql_test in4-4.13 { + SELECT c FROM t4b WHERE +a=b; +} {4} +do_execsql_test in4-4.14 { + SELECT c FROM t4b WHERE a=+b; +} {} +do_execsql_test in4-4.15 { + SELECT c FROM t4b WHERE +b=a; +} {} +do_execsql_test in4-4.16 { + SELECT c FROM t4b WHERE b=+a; +} {4} +do_execsql_test in4-4.17 { + SELECT c FROM t4b WHERE a IN (b); +} {} +do_execsql_test in4-4.18 { + SELECT c FROM t4b WHERE b IN (a); +} {4} +do_execsql_test in4-4.19 { + SELECT c FROM t4b WHERE +b IN (a); +} {} + + + finish_test From 4ef7efad5eccd9acc32c3ecce05026914beaa903 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 15:14:08 +0000 Subject: [PATCH 098/177] Fix harmless compiler warnings. FossilOrigin-Name: b1435f26b07b2208cfcca557f96342a5bd0d5328 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 6 +++++- src/parse.y | 3 +-- src/sqliteInt.h | 2 +- src/vdbesort.c | 4 ++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 5d2b629021..9590aa3ca2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Previous\scheck-in\sis\snot\squite\scorrect.\s\s"x\sIN\s(?)"\sis\snot\sexactly\sthe\ssame\nas\s"x==?"\sdo\sto\scollation\sand\saffinity\sissues.\s\sThe\scorrect\sconverstion\sshould\nbe\sto\s"x==(+?\sCOLLATE\sbinary)".\s\sThe\scurrent\scheck-in\sfixes\sthis\sproblem\sand\nprovides\stest\scases.\s\sTicket\s[e39d032577df69] -D 2014-03-20T14:56:47.071 +C Fix\sharmless\scompiler\swarnings. +D 2014-03-20T15:14:08.664 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 +F src/expr.c b74939e7935c4ad9e7f87b31ce05713fd5dafc3a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y c9189eca0888ecd2711c77a199ae7a7c86f57546 +F src/parse.y d21075457487f84a72f848c2545e3319d6452e6f F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -222,7 +222,7 @@ F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fa7161b3de18a9c355d4148233f3563c92311fcc +F src/sqliteInt.h db6d7cf6e44d1c862f4f5290716098a0b90f1310 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0a9e5e63d7ce196651610d1a264eeb0b2152d3ba +F src/vdbesort.c 0fef5fa890fda760f92f293a5b0f2ccfe5d2dc55 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e68b427afbc82e201c64474117851aa4c9eb0c92 -R 8739dc742c6ff38e89fff08724b87c5b +P 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 +R b4897cbb8b99db805aa2642c2b5c6a8f U drh -Z c6097cc93dfeb105af183d7216a869ad +Z 8f62a0caca3ff37dd6879b77616b6faf diff --git a/manifest.uuid b/manifest.uuid index 7bb62d8aab..82d7da5708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 \ No newline at end of file +b1435f26b07b2208cfcca557f96342a5bd0d5328 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 722a77db73..83b948e8e1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -65,7 +65,11 @@ char sqlite3ExprAffinity(Expr *pExpr){ ** If a memory allocation error occurs, that fact is recorded in pParse->db ** and the pExpr parameter is returned unchanged. */ -Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){ +Expr *sqlite3ExprAddCollateToken( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName /* Name of collating sequence */ +){ if( pCollName->n>0 ){ Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); if( pNew ){ diff --git a/src/parse.y b/src/parse.y index 24b8b98afc..aae30f5ee2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1031,8 +1031,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 == (+?1 COLLATE binary) ** expr1 <> (+?2 COLLATE binary) */ - static const Token collBin = { "binary", 6 }; - Expr *pRHS = sqlite3ExprAddCollateToken(pParse, Y->a[0].pExpr, &collBin); + Expr *pRHS = sqlite3ExprAddCollateString(pParse, Y->a[0].pExpr, "binary"); Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ee39e5278..0974db7c91 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3291,7 +3291,7 @@ int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, Token*); +Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); Expr *sqlite3ExprSkipCollate(Expr*); int sqlite3CheckCollSeq(Parse *, CollSeq *); diff --git a/src/vdbesort.c b/src/vdbesort.c index f971c2d933..839782b9c7 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -993,12 +993,12 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** a value equivalent to pIter2. So set pKey2 to NULL to prevent ** vdbeSorterCompare() from decoding pIter2 again. */ if( iRes<=0 ){ - pSorter->aTree[i] = (pIter1 - pSorter->aIter); + pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; pKey2 = pIter2->aKey; }else{ if( pIter1->pFile ) pKey2 = 0; - pSorter->aTree[i] = (pIter2 - pSorter->aIter); + pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; } From fbb24d1092c1c09a9493d382589176c1ba54bd52 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 17:03:30 +0000 Subject: [PATCH 099/177] The "x IN (?)" optimization in check-ins [2ff3b25f40] and [e68b427afb] is incorrect, as demonstrated by the in4-5.1 test case in this check-in. The "COLLATE binary" that was being added to the RHS of IN was overriding the implicit collating sequence of the LHS. This change defines the EP_Generic expression node property that blocks all affinity or collating sequence information in the expression subtree and adds that property to the expression taken from RHS of the IN operator. FossilOrigin-Name: 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/parse.y | 19 ++++++++++++++----- src/sqliteInt.h | 4 ++-- test/in4.test | 30 ++++++++++++++++++++++++++++++ 6 files changed, 58 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9590aa3ca2..8eaa4bfe54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-03-20T15:14:08.664 +C The\s"x\sIN\s(?)"\soptimization\sin\scheck-ins\s[2ff3b25f40]\sand\s[e68b427afb]\sis\nincorrect,\sas\sdemonstrated\sby\sthe\sin4-5.1\stest\scase\sin\sthis\scheck-in.\nThe\s"COLLATE\sbinary"\sthat\swas\sbeing\sadded\sto\sthe\sRHS\sof\sIN\swas\soverriding\nthe\simplicit\scollating\ssequence\sof\sthe\sLHS.\s\sThis\schange\sdefines\sthe\sEP_Generic\nexpression\snode\sproperty\sthat\sblocks\sall\saffinity\sor\scollating\ssequence\ninformation\sin\sthe\sexpression\ssubtree\sand\sadds\sthat\sproperty\sto\sthe\sexpression\ntaken\sfrom\sRHS\sof\sthe\sIN\soperator. +D 2014-03-20T17:03:30.667 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c b74939e7935c4ad9e7f87b31ce05713fd5dafc3a +F src/expr.c 16ea9cefe7c8f998816b4eb8b8e7a88f0d2d3797 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y d21075457487f84a72f848c2545e3319d6452e6f +F src/parse.y fb3280d85a103f623e5cf551b5b96b9df33151ac F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -222,7 +222,7 @@ F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h db6d7cf6e44d1c862f4f5290716098a0b90f1310 +F src/sqliteInt.h 42acfa3d3b793822915ceb7e83c0cbc774d37d66 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 18202389003284e8e019750c04e4bc6333df9c99 +F test/in4.test 41c1c031aa46b1eb4411df2687ed2ed498da23b5 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 -R b4897cbb8b99db805aa2642c2b5c6a8f +P b1435f26b07b2208cfcca557f96342a5bd0d5328 +R a33b75c3e29794aff199068a6fe53400 U drh -Z 8f62a0caca3ff37dd6879b77616b6faf +Z 9b4a07b9ddb6d1760a1f90c03adda56a diff --git a/manifest.uuid b/manifest.uuid index 82d7da5708..5b4998caed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1435f26b07b2208cfcca557f96342a5bd0d5328 \ No newline at end of file +2ea4a9f75f46eaa928ba17e9e91bc0432750d46d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 83b948e8e1..204d696179 100644 --- a/src/expr.c +++ b/src/expr.c @@ -33,6 +33,7 @@ char sqlite3ExprAffinity(Expr *pExpr){ int op; pExpr = sqlite3ExprSkipCollate(pExpr); + if( pExpr->flags & EP_Generic ) return SQLITE_AFF_NONE; op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -122,6 +123,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ Expr *p = pExpr; while( p ){ int op = p->op; + if( p->flags & EP_Generic ) break; if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; continue; diff --git a/src/parse.y b/src/parse.y index aae30f5ee2..ef00925254 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1026,15 +1026,24 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 IN (?1) ** expr1 NOT IN (?2) ** - ** with exactly one value on the RHS can be simplified to: + ** with exactly one value on the RHS can be simplified to something + ** like this: ** - ** expr1 == (+?1 COLLATE binary) - ** expr1 <> (+?2 COLLATE binary) + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. */ - Expr *pRHS = sqlite3ExprAddCollateString(pParse, Y->a[0].pExpr, "binary"); + Expr *pRHS = Y->a[0].pExpr; Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); - pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); + if( pRHS ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; + } A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0974db7c91..55b7e080ae 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1893,8 +1893,8 @@ struct Expr { #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE opeartor */ - /* unused 0x000200 */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ #define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ #define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ diff --git a/test/in4.test b/test/in4.test index 5828a83cc5..0a4a75008b 100644 --- a/test/in4.test +++ b/test/in4.test @@ -302,8 +302,38 @@ do_execsql_test in4-4.19 { SELECT c FROM t4b WHERE +b IN (a); } {} +do_execsql_test in4-5.1 { + CREATE TABLE t5(c INTEGER PRIMARY KEY, d TEXT COLLATE nocase); + INSERT INTO t5 VALUES(17, 'fuzz'); + SELECT 1 FROM t5 WHERE 'fuzz' IN (d); -- match + SELECT 2 FROM t5 WHERE 'FUZZ' IN (d); -- no match + SELECT 3 FROM t5 WHERE d IN ('fuzz'); -- match + SELECT 4 FROM t5 WHERE d IN ('FUZZ'); -- match +} {1 3 4} +# An expression of the form "x IN (y)" can be used as "x=y" by the +# query planner when computing transitive constraints or to run the +# query using an index on y. +# +do_execsql_test in4-6.1 { + CREATE TABLE t6a(a INTEGER PRIMARY KEY, b); + INSERT INTO t6a VALUES(1,2),(3,4),(5,6); + CREATE TABLE t6b(c INTEGER PRIMARY KEY, d); + INSERT INTO t6b VALUES(4,44),(5,55),(6,66); + SELECT * FROM t6a, t6b WHERE a=3 AND b IN (c); +} {3 4 4 44} +do_execsql_test in4-6.1-eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t6a, t6b WHERE a=3 AND b IN (c); +} {~/SCAN/} +do_execsql_test in4-6.2 { + SELECT * FROM t6a, t6b WHERE a=3 AND c IN (b); +} {3 4 4 44} +do_execsql_test in4-6.2-eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t6a, t6b WHERE a=3 AND c IN (b); +} {~/SCAN/} finish_test From 8ad0de3f725e9d32d624d3102f4e1b75780e7ee6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 18:45:27 +0000 Subject: [PATCH 100/177] Fix the EXPLAIN indenter in the command-line shell to correctly handle NextIfOpen and PrevIfOpen opcodes. FossilOrigin-Name: 01944c53f5c129f48ce8842faaedcf5607dd381e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8eaa4bfe54..b902e242a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\s"x\sIN\s(?)"\soptimization\sin\scheck-ins\s[2ff3b25f40]\sand\s[e68b427afb]\sis\nincorrect,\sas\sdemonstrated\sby\sthe\sin4-5.1\stest\scase\sin\sthis\scheck-in.\nThe\s"COLLATE\sbinary"\sthat\swas\sbeing\sadded\sto\sthe\sRHS\sof\sIN\swas\soverriding\nthe\simplicit\scollating\ssequence\sof\sthe\sLHS.\s\sThis\schange\sdefines\sthe\sEP_Generic\nexpression\snode\sproperty\sthat\sblocks\sall\saffinity\sor\scollating\ssequence\ninformation\sin\sthe\sexpression\ssubtree\sand\sadds\sthat\sproperty\sto\sthe\sexpression\ntaken\sfrom\sRHS\sof\sthe\sIN\soperator. -D 2014-03-20T17:03:30.667 +C Fix\sthe\sEXPLAIN\sindenter\sin\sthe\scommand-line\sshell\sto\scorrectly\nhandle\sNextIfOpen\sand\sPrevIfOpen\sopcodes. +D 2014-03-20T18:45:27.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 -F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa +F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b1435f26b07b2208cfcca557f96342a5bd0d5328 -R a33b75c3e29794aff199068a6fe53400 +P 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d +R 4c494d6d400709087b8c24162b58f63e U drh -Z 9b4a07b9ddb6d1760a1f90c03adda56a +Z f93731af177c8fb155193dbd5a919278 diff --git a/manifest.uuid b/manifest.uuid index 5b4998caed..80f6ef7e37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ea4a9f75f46eaa928ba17e9e91bc0432750d46d \ No newline at end of file +01944c53f5c129f48ce8842faaedcf5607dd381e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2cff102e91..1313112709 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,7 +1195,8 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "NextIfOpen", "PrevIfOpen", 0 }; const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; From 9bc21b535e9b5578fb4a8ef77f58d0f392a437af Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Mar 2014 18:56:35 +0000 Subject: [PATCH 101/177] Only use the direct-overflow-read optimization if all data from the overflow page in question is being read. FossilOrigin-Name: d8e1f75ddf10f3c0b21acd5455f90fdcea54a948 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 4 ++++ test/func.test | 12 +++++++----- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ff09351dd2..751024e857 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sstray\sC99-style\scomment. -D 2014-03-11T23:44:10.719 +C Only\suse\sthe\sdirect-overflow-read\soptimization\sif\sall\sdata\sfrom\sthe\soverflow\spage\sin\squestion\sis\sbeing\sread. +D 2014-03-20T18:56:35.309 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 33b5ad54a14c2b62e8adbc7656dec2b7cd760cbf +F src/btree.c 0384d399cfb12b3767edec355f00996658bb093e F src/btree.h 6b0c1a3f0937f9852ec727c820e71dbdd4bd0b27 F src/btreeInt.h d1784d1e17d08d29e890190dbb9836fa64573381 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -574,7 +574,7 @@ F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584 F test/fts4noti.test aed33ba44808852dcb24bf70fa132e7bf530f057 F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test a21814945d32137412b553d98ad2107f9b2173a9 +F test/func.test c2cbfc23d554c5bf8678d0fb271aa4f8ef94839c F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 968fec44d7fde3adbd3e9603e4282351f0d4bda1 -R c766c2eb192c274bbd3dc0af3849118d -U drh -Z 264233dee2a1cd950306c0e6dc59ee29 +P f500e87d6ec6fa43c13f4e94edf32789f36e4233 +R 97d65ef8b8853428f883a03b9234ff63 +U dan +Z c025d7be55b561d9bf0252bb6230f9e0 diff --git a/manifest.uuid b/manifest.uuid index 4c7b90b821..16aed852aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f500e87d6ec6fa43c13f4e94edf32789f36e4233 \ No newline at end of file +d8e1f75ddf10f3c0b21acd5455f90fdcea54a948 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 045e8ba2fd..cd2df05324 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3951,6 +3951,7 @@ static int accessPayload( int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ + int bEnd; /* True if reading to end of data */ assert( pPage ); assert( pCur->eState==CURSOR_VALID ); @@ -3960,6 +3961,7 @@ static int accessPayload( getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); + bEnd = (offset+amt==nKey+pCur->info.nData); if( NEVER(offset+amt > nKey+pCur->info.nData) || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] @@ -4064,6 +4066,7 @@ static int accessPayload( ** 3) the database is file-backed, and ** 4) there is no open write-transaction, and ** 5) the database is not a WAL database, + ** 6) all data from the page is being read. ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -4071,6 +4074,7 @@ static int accessPayload( */ if( (eOp&0x01)==0 /* (1) */ && offset==0 /* (2) */ + && (bEnd || a==ovflSize) /* (6) */ && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ && pBt->pPage1->aData[19]==0x01 /* (5) */ diff --git a/test/func.test b/test/func.test index 0fafd1828d..e3cec32153 100644 --- a/test/func.test +++ b/test/func.test @@ -1301,11 +1301,13 @@ do_test func-29.3 { db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} if {[permutation] != "mmap"} { - do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x - } {many} + ifcapable !direct_read { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} + } } do_test func-29.5 { db close From 5b1420e046aaf2aa3043fa1725b0ae57abe1578a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 19:04:56 +0000 Subject: [PATCH 102/177] Remove a testcase() that is now always true due to the "x IN (?)" optimization. Add an ALWAYS() around a conditional in the parser that cannot be false. FossilOrigin-Name: d5a1530bdc7ace053d05d1a037551110021d3758 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 - src/parse.y | 4 +++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b902e242a6..9c280ab5f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sEXPLAIN\sindenter\sin\sthe\scommand-line\sshell\sto\scorrectly\nhandle\sNextIfOpen\sand\sPrevIfOpen\sopcodes. -D 2014-03-20T18:45:27.340 +C Remove\sa\stestcase()\sthat\sis\snow\salways\strue\sdue\sto\sthe\s"x\sIN\s(?)"\soptimization.\nAdd\san\sALWAYS()\saround\sa\sconditional\sin\sthe\sparser\sthat\scannot\sbe\sfalse. +D 2014-03-20T19:04:56.106 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 16ea9cefe7c8f998816b4eb8b8e7a88f0d2d3797 +F src/expr.c 6e1e8bca468753f06758db7835a85e5408351e7a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y fb3280d85a103f623e5cf551b5b96b9df33151ac +F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d -R 4c494d6d400709087b8c24162b58f63e +P 01944c53f5c129f48ce8842faaedcf5607dd381e +R 608adcc59408221c1aef74de280cb552 U drh -Z f93731af177c8fb155193dbd5a919278 +Z 43de690b7d9666e906803652c8c9527b diff --git a/manifest.uuid b/manifest.uuid index 80f6ef7e37..416a2c393f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01944c53f5c129f48ce8842faaedcf5607dd381e \ No newline at end of file +d5a1530bdc7ace053d05d1a037551110021d3758 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 204d696179..e3997f00c3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1636,7 +1636,6 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ *prNotFound = rMayHaveNull = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ - testcase( pParse->nQueryLoop>0 ); pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; diff --git a/src/parse.y b/src/parse.y index ef00925254..dbc129ce63 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1040,7 +1040,9 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { Expr *pRHS = Y->a[0].pExpr; Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); - if( pRHS ){ + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } From e99b9cdc70213d8e7d59a4b4447b7c59b56a4869 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 15:24:07 +0000 Subject: [PATCH 103/177] New test case for block-sorting. FossilOrigin-Name: e70cfa28aa393661ccc742ecd5e672d807bdd0a9 --- manifest | 11 +-- manifest.uuid | 2 +- test/orderby6.test | 183 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 test/orderby6.test diff --git a/manifest b/manifest index 6bb5c6bb73..e0157af624 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sfor\s"x\sIN\s(?)"\shandling. -D 2014-03-20T20:56:49.315 +C New\stest\scase\sfor\sblock-sorting. +D 2014-03-21T15:24:07.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -723,6 +723,7 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1156,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e4bfffb988283c077778c60696be0d285ad66c3c d5a1530bdc7ace053d05d1a037551110021d3758 -R a45d230e7f98fec92659279ce8b3552c +P eca35871c34374ca9189c7c9b6d490ac3c30357f +R f6c9f43506e02e19d33da5c08cba37a5 U drh -Z 027a45adb600d40b2564e06474072c41 +Z 983388db59b02f32fb79fe85b7106067 diff --git a/manifest.uuid b/manifest.uuid index 8a32b61e54..fe6f27d9c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eca35871c34374ca9189c7c9b6d490ac3c30357f \ No newline at end of file +e70cfa28aa393661ccc742ecd5e672d807bdd0a9 \ No newline at end of file diff --git a/test/orderby6.test b/test/orderby6.test new file mode 100644 index 0000000000..403250ddb5 --- /dev/null +++ b/test/orderby6.test @@ -0,0 +1,183 @@ +# 2014-03-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the block-sort optimization. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby6 + +# Run all tests twice. Once with a normal table and a second time +# with a WITHOUT ROWID table +# +foreach {tn rowidclause} {1 {} 2 {WITHOUT ROWID}} { + + # Construct a table with 1000 rows and a split primary key + # + reset_db + do_test $tn.1 { + db eval "CREATE TABLE t1(a,b,c,PRIMARY KEY(b,c)) $rowidclause;" + db eval { + WITH RECURSIVE + cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<1000) + INSERT INTO t1 SELECT x, x%40, x/40 FROM cnt; + } + } {} + + # Run various ORDER BY queries that can benefit from block-sort. + # Compare the output to the same output using a full-sort enforced + # by adding + to each term of the ORDER BY clause. + # + do_execsql_test $tn.2 { + SELECT b,a,c FROM t1 ORDER BY b,a,c; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b,+a,+c}] + do_execsql_test $tn.3 { + SELECT b,a,c FROM t1 ORDER BY b,c DESC,a; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b,+c DESC,+a}] + do_execsql_test $tn.4 { + SELECT b,a,c FROM t1 ORDER BY b DESC,c,a; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b DESC,+c,+a}] + do_execsql_test $tn.5 { + SELECT b,a,c FROM t1 ORDER BY b DESC,a,c; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b DESC,+a,+c}] + + # LIMIT and OFFSET clauses on block-sort queries. + # + do_execsql_test $tn.11 { + SELECT a FROM t1 ORDER BY b, a LIMIT 10 OFFSET 20; + } {840 880 920 960 1000 1 41 81 121 161} + do_execsql_test $tn.11x { + SELECT a FROM t1 ORDER BY +b, a LIMIT 10 OFFSET 20; + } {840 880 920 960 1000 1 41 81 121 161} + + do_execsql_test $tn.12 { + SELECT a FROM t1 ORDER BY b DESC, a LIMIT 10 OFFSET 20; + } {839 879 919 959 999 38 78 118 158 198} + do_execsql_test $tn.12 { + SELECT a FROM t1 ORDER BY +b DESC, a LIMIT 10 OFFSET 20; + } {839 879 919 959 999 38 78 118 158 198} + + do_execsql_test $tn.13 { + SELECT a FROM t1 ORDER BY b, a DESC LIMIT 10 OFFSET 45; + } {161 121 81 41 1 962 922 882 842 802} + do_execsql_test $tn.13x { + SELECT a FROM t1 ORDER BY +b, a DESC LIMIT 10 OFFSET 45; + } {161 121 81 41 1 962 922 882 842 802} + + do_execsql_test $tn.14 { + SELECT a FROM t1 ORDER BY b DESC, a LIMIT 10 OFFSET 45; + } {838 878 918 958 998 37 77 117 157 197} + do_execsql_test $tn.14x { + SELECT a FROM t1 ORDER BY +b DESC, a LIMIT 10 OFFSET 45; + } {838 878 918 958 998 37 77 117 157 197} + + # Many test cases where the LIMIT+OFFSET window is in various + # alignments with block-sort boundaries. + # + foreach {tx limit offset orderby} { + 1 10 24 {+b,+a} + 2 10 25 {+b,+a} + 3 10 26 {+b,+a} + 4 10 39 {+b,+a} + 5 10 40 {+b,+a} + 6 10 41 {+b,+a} + 7 27 24 {+b,+a} + 8 27 49 {+b,+a} + 11 10 24 {+b DESC,+a} + 12 10 25 {+b DESC,+a} + 13 10 26 {+b DESC,+a} + 14 10 39 {+b DESC,+a} + 15 10 40 {+b DESC,+a} + 16 10 41 {+b DESC,+a} + 17 27 24 {+b DESC,+a} + 18 27 49 {+b DESC,+a} + 21 10 24 {+b,+a DESC} + 22 10 25 {+b,+a DESC} + 23 10 26 {+b,+a DESC} + 24 10 39 {+b,+a DESC} + 25 10 40 {+b,+a DESC} + 26 10 41 {+b,+a DESC} + 27 27 24 {+b,+a DESC} + 28 27 49 {+b,+a DESC} + 31 10 24 {+b DESC,+a DESC} + 32 10 25 {+b DESC,+a DESC} + 33 10 26 {+b DESC,+a DESC} + 34 10 39 {+b DESC,+a DESC} + 35 10 40 {+b DESC,+a DESC} + 36 10 41 {+b DESC,+a DESC} + 37 27 24 {+b DESC,+a DESC} + 38 27 49 {+b DESC,+a DESC} + } { + set sql1 "SELECT a FROM t1 ORDER BY $orderby LIMIT $limit OFFSET $offset;" + set sql2 [string map {+ {}} $sql1] + # puts $sql2\n$sql1\n[db eval $sql2] + do_test $tn.21.$tx {db eval $::sql2} [db eval $sql1] + } + + ######################################################################## + # A second test table, t2, has many columns open to sorting. + do_test $tn.31 { + db eval "CREATE TABLE t2(a,b,c,d,e,f,PRIMARY KEY(b,c,d,e,f)) $rowidclause;" + db eval { + WITH RECURSIVE + cnt(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM cnt WHERE x<242) + INSERT INTO t2 SELECT x, x%3, (x/3)%3, (x/9)%3, (x/27)%3, (x/81)%3 + FROM cnt; + } + } {} + + do_execsql_test $tn.32 { + SELECT a FROM t2 ORDER BY b,c,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.33 { + SELECT a FROM t2 ORDER BY b,c,d,e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.34 { + SELECT a FROM t2 ORDER BY b,c,d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.35 { + SELECT a FROM t2 ORDER BY b,c,+d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.36 { + SELECT a FROM t2 ORDER BY b,+c,+d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + + do_execsql_test $tn.37 { + SELECT a FROM t2 ORDER BY b,c,d,e,f DESC; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f DESC;}] + do_execsql_test $tn.38 { + SELECT a FROM t2 ORDER BY b,c,d,e DESC,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e DESC,+f;}] + do_execsql_test $tn.39 { + SELECT a FROM t2 ORDER BY b,c,d DESC,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d DESC,+e,+f;}] + do_execsql_test $tn.40 { + SELECT a FROM t2 ORDER BY b,c DESC,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c DESC,+d,+e,+f;}] + do_execsql_test $tn.41 { + SELECT a FROM t2 ORDER BY b DESC,c,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b DESC,+c,+d,+e,+f;}] + + do_execsql_test $tn.42 { + SELECT a FROM t2 ORDER BY b DESC,c DESC,d,e,f LIMIT 31; + } [db eval {SELECT a FROM t2 ORDER BY +b DESC,+c DESC,+d,+e,+f LIMIT 31}] + do_execsql_test $tn.43 { + SELECT a FROM t2 ORDER BY b,c,d,e,f DESC LIMIT 8 OFFSET 7; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f DESC LIMIT 8 OFFSET 7}] + + +} + + + +finish_test From a22a75e5ca62599b22744fd089b3a4f504bc83d5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 18:16:23 +0000 Subject: [PATCH 104/177] Fix the OFFSET clause so that it works correctly on queries that lack a FROM clause. Ticket [07d6a0453d4ed8]. FossilOrigin-Name: 179ef81648b0ad557df78b7712f216b876b6fb65 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 3 ++- src/where.c | 3 ++- test/limit.test | 20 ++++++++++++++++++++ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9c280ab5f2..68306cb2c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\stestcase()\sthat\sis\snow\salways\strue\sdue\sto\sthe\s"x\sIN\s(?)"\soptimization.\nAdd\san\sALWAYS()\saround\sa\sconditional\sin\sthe\sparser\sthat\scannot\sbe\sfalse. -D 2014-03-20T19:04:56.106 +C Fix\sthe\sOFFSET\sclause\sso\sthat\sit\sworks\scorrectly\son\squeries\sthat\slack\na\sFROM\sclause.\s\sTicket\s[07d6a0453d4ed8]. +D 2014-03-21T18:16:23.398 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 +F src/select.c bea5181682a75940f4f555bfd1020169cc8f1657 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c a2f20bdc6565a3add38ac5e4d9e71523ed5b2f2b +F src/where.c eed0b83bd7ffef4ac2f7a2fcc9dcfa66b18b8b9e F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -648,7 +648,7 @@ F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test e191e536d0fcd722a6b965e7cd1ee0bfd12a5991 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da -F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 +F test/limit.test 3d7df19c35ac672a11f7de406cd3205d592babbb F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 01944c53f5c129f48ce8842faaedcf5607dd381e -R 608adcc59408221c1aef74de280cb552 +P d5a1530bdc7ace053d05d1a037551110021d3758 +R e96408bb9b93596f5e755db9dbf77630 U drh -Z 43de690b7d9666e906803652c8c9527b +Z f1228f80df0ab7ad7b6a23acd634c66b diff --git a/manifest.uuid b/manifest.uuid index 416a2c393f..0c9a994b80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5a1530bdc7ace053d05d1a037551110021d3758 \ No newline at end of file +179ef81648b0ad557df78b7712f216b876b6fb65 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 850bc6a902..65e4ae7d67 100644 --- a/src/select.c +++ b/src/select.c @@ -473,7 +473,7 @@ static void codeOffset( int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( iOffset>0 && iContinue!=0 ){ + if( iOffset>0 ){ int addr; sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); VdbeCoverage(v); @@ -579,6 +579,7 @@ static void selectInnerLoop( assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ + assert( iContinue!=0 ); codeOffset(v, p->iOffset, iContinue); } diff --git a/src/where.c b/src/where.c index e3ce8b8697..bc5dc04d14 100644 --- a/src/where.c +++ b/src/where.c @@ -47,6 +47,7 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ ** immediately with the next row of a WHERE clause. */ int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); return pWInfo->iContinue; } @@ -5453,7 +5454,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; pWInfo->pResultSet = pResultSet; - pWInfo->iBreak = sqlite3VdbeMakeLabel(v); + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = &pWInfo->sMaskSet; diff --git a/test/limit.test b/test/limit.test index c5b75c2b69..5d6ef336b2 100644 --- a/test/limit.test +++ b/test/limit.test @@ -616,4 +616,24 @@ do_test limit-13.81 { db eval {SELECT z FROM v13c LIMIT 1 OFFSET 8} } {} +do_execsql_test limit-14.1 { + SELECT 123 LIMIT 1 OFFSET 0 +} {123} +do_execsql_test limit-14.2 { + SELECT 123 LIMIT 1 OFFSET 1 +} {} +do_execsql_test limit-14.3 { + SELECT 123 LIMIT 0 OFFSET 0 +} {} +do_execsql_test limit-14.4 { + SELECT 123 LIMIT 0 OFFSET 1 +} {} +do_execsql_test limit-14.6 { + SELECT 123 LIMIT -1 OFFSET 0 +} {123} +do_execsql_test limit-14.7 { + SELECT 123 LIMIT -1 OFFSET 1 +} {} + + finish_test From 9afccba269eb1910404fc003fc8e96ca6fbd742e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Mar 2014 19:27:54 +0000 Subject: [PATCH 105/177] Avoid leaking memory in an obscure case where the flattener adds an ORDER BY clause to the recursive part of a recursive query. FossilOrigin-Name: 1f413aca00015100224273480e1ce39a76bf93ab --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 8 +++++++- src/sqliteInt.h | 2 +- test/with2.test | 27 +++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 68306cb2c1..9a15563c3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sOFFSET\sclause\sso\sthat\sit\sworks\scorrectly\son\squeries\sthat\slack\na\sFROM\sclause.\s\sTicket\s[07d6a0453d4ed8]. -D 2014-03-21T18:16:23.398 +C Avoid\sleaking\smemory\sin\san\sobscure\scase\swhere\sthe\sflattener\sadds\san\sORDER\sBY\sclause\sto\sthe\srecursive\spart\sof\sa\srecursive\squery. +D 2014-03-21T19:27:54.828 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bea5181682a75940f4f555bfd1020169cc8f1657 +F src/select.c 0f7542b85f92d8b0c59a21d7ddcf5c0228e60fdc F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 42acfa3d3b793822915ceb7e83c0cbc774d37d66 +F src/sqliteInt.h 828f61d3c58608ad2e871369950f3cf70428e917 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1096,7 +1096,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 268081a6b14817a262ced4d0ee34d4d2a1dd2068 -F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 +F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test e00a0a9dc9f0be651f011d61e8a32b7add5afb30 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d5a1530bdc7ace053d05d1a037551110021d3758 -R e96408bb9b93596f5e755db9dbf77630 -U drh -Z f1228f80df0ab7ad7b6a23acd634c66b +P 179ef81648b0ad557df78b7712f216b876b6fb65 +R d6ea5c2b32ee3608a1c16a03630c7806 +U dan +Z 37d6162a1c996286e576ee6c912ca01b diff --git a/manifest.uuid b/manifest.uuid index 0c9a994b80..e3acb3d652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -179ef81648b0ad557df78b7712f216b876b6fb65 \ No newline at end of file +1f413aca00015100224273480e1ce39a76bf93ab \ No newline at end of file diff --git a/src/select.c b/src/select.c index 65e4ae7d67..93cf15f79b 100644 --- a/src/select.c +++ b/src/select.c @@ -1939,6 +1939,7 @@ static void generateWithRecursiveQuery( sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); p->pOrderBy = pOrderBy; p->pLimit = pLimit; p->pOffset = pOffset; @@ -4487,9 +4488,14 @@ int sqlite3Select( if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistTable ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistTable || + pDest->eDest==SRT_DistQueue); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 55b7e080ae..b31686c3ff 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2283,7 +2283,7 @@ struct Select { #define SRT_Discard 4 /* Do not save the results anywhere */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard || (X->eDest)>SRT_Table) #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ diff --git a/test/with2.test b/test/with2.test index d702f8c962..eb0614729b 100644 --- a/test/with2.test +++ b/test/with2.test @@ -385,6 +385,33 @@ do_execsql_test 7.5 { ) } {14 28 42} +#------------------------------------------------------------------------- +# At one point the following was causing an assertion failure and a +# memory leak. +# +do_execsql_test 8.1 { + CREATE TABLE t7(y); + INSERT INTO t7 VALUES(NULL); + CREATE VIEW v AS SELECT * FROM t7 ORDER BY y; +} + +do_execsql_test 8.2 { + WITH q(a) AS ( + SELECT 1 + UNION + SELECT a+1 FROM q, v WHERE a<5 + ) + SELECT * FROM q; +} {1 2 3 4 5} + +do_execsql_test 8.3 { + WITH q(a) AS ( + SELECT 1 + UNION ALL + SELECT a+1 FROM q, v WHERE a<5 + ) + SELECT * FROM q; +} {1 2 3 4 5} finish_test From 8e1ee88c8811f9653882c4c1a894951d65d36bbf Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 19:56:09 +0000 Subject: [PATCH 106/177] Change the names of SRT_DistTable and SRT_Table used by CTE to more meaningful SRT_DistFifo and SRT_Fifo, respectively. Simplify the IgnorableOrderby() macro in the process. FossilOrigin-Name: 45d8cc678d128f1dda6469864215a8ed9de4366a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 22 ++++++++++++---------- src/sqliteInt.h | 31 +++++++++++++++++-------------- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 9a15563c3f..88f61607f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sleaking\smemory\sin\san\sobscure\scase\swhere\sthe\sflattener\sadds\san\sORDER\sBY\sclause\sto\sthe\srecursive\spart\sof\sa\srecursive\squery. -D 2014-03-21T19:27:54.828 +C Change\sthe\snames\sof\sSRT_DistTable\sand\sSRT_Table\sused\sby\sCTE\sto\smore\nmeaningful\sSRT_DistFifo\sand\sSRT_Fifo,\srespectively.\s\sSimplify\sthe\nIgnorableOrderby()\smacro\sin\sthe\sprocess. +D 2014-03-21T19:56:09.443 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0f7542b85f92d8b0c59a21d7ddcf5c0228e60fdc +F src/select.c cbee644f795a5fa14fa79bd760c3c9c883a6b4d8 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 828f61d3c58608ad2e871369950f3cf70428e917 +F src/sqliteInt.h 76b1cf0836f0d022aec5045474f5c6c90f7b9730 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 179ef81648b0ad557df78b7712f216b876b6fb65 -R d6ea5c2b32ee3608a1c16a03630c7806 -U dan -Z 37d6162a1c996286e576ee6c912ca01b +P 1f413aca00015100224273480e1ce39a76bf93ab +R 26118dcbaf585402aab527afe8f97345 +U drh +Z d7e9ffdc5a42e976a5961e8287a0561b diff --git a/manifest.uuid b/manifest.uuid index e3acb3d652..253fab48e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f413aca00015100224273480e1ce39a76bf93ab \ No newline at end of file +45d8cc678d128f1dda6469864215a8ed9de4366a \ No newline at end of file diff --git a/src/select.c b/src/select.c index 93cf15f79b..973b444407 100644 --- a/src/select.c +++ b/src/select.c @@ -700,7 +700,8 @@ static void selectInnerLoop( /* Store the result as data using a unique key. */ - case SRT_DistTable: + case SRT_Fifo: + case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); @@ -708,8 +709,8 @@ static void selectInnerLoop( testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE - if( eDest==SRT_DistTable ){ - /* If the destination is DistTable, then cursor (iParm+1) is open + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open ** on an ephemeral index. If the current row is already present ** in the index, do not write it to the output. If not, add the ** current row to the index and proceed with writing it to the @@ -1835,7 +1836,7 @@ static void generateWithRecursiveQuery( int regCurrent; /* Register holding Current table */ int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; /* How to write to Queue */ + int eDest = SRT_Fifo; /* How to write to Queue */ SelectDest destQueue; /* SelectDest targetting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ @@ -1867,13 +1868,13 @@ static void generateWithRecursiveQuery( /* Allocate cursors numbers for Queue and Distinct. The cursor number for ** the Distinct table must be exactly one greater than Queue in order - ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ iQueue = pParse->nTab++; if( p->op==TK_UNION ){ - eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; iDistinct = pParse->nTab++; }else{ - eDest = pOrderBy ? SRT_Queue : SRT_Table; + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; } sqlite3SelectDestInit(&destQueue, eDest, iQueue); @@ -4488,14 +4489,15 @@ int sqlite3Select( if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); - assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistTable ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || - pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistTable || - pDest->eDest==SRT_DistQueue); + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b31686c3ff..724b22fec1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2261,13 +2261,15 @@ struct Select { ** starting with pDest->iSdst. ** ** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. +** SRT_Fifo This is like SRT_EphemTab except that the table +** is assumed to already be open. SRT_Fifo has +** the additional property of being able to ignore +** the ORDER BY clause. ** -** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** SRT_DistFifo Store results in a temporary table pDest->iSDParm. ** But also use temporary table pDest->iSDParm+1 as ** a record of all prior results and ignore any duplicate -** rows. Name means: "Distinct Table". +** rows. Name means: "Distinct Fifo". ** ** SRT_Queue Store results in priority queue pDest->iSDParm (really ** an index). Append a sequence number so that all entries @@ -2281,19 +2283,20 @@ struct Select { #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_Fifo 5 /* Store result as data with an automatic rowid */ +#define SRT_DistFifo 6 /* Like SRT_Fifo, but unique results only */ +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_DistQueue 8 /* Like SRT_Queue, but unique results only */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard || (X->eDest)>SRT_Table) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Output 5 /* Output each row of result */ -#define SRT_Mem 6 /* Store result in a memory cell */ -#define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 9 /* Generate a single row of result */ -#define SRT_Table 10 /* Store result as data with an automatic rowid */ -#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ -#define SRT_Queue 12 /* Store result in an queue */ -#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ /* ** An instance of this object describes where to put of the results of From 59b8f2e108b3de087b31a2bd75f38a9114fb71d7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Mar 2014 00:27:14 +0000 Subject: [PATCH 107/177] Fix the ORDER BY optimization logic so that it will do a block-sort on a partial DESC ORDER BY. This enhancement uncovered a memory leak in pushUntoSorter() which is also fixed. FossilOrigin-Name: c36f74611cf17ad2ff198a2ac2054d7ab451a72c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + src/where.c | 22 +++++++++++----------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5abb89ba80..eca837d211 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\suse\sindices\sfor\sthe\sfirst\sfew\sterms\sof\san\sORDER\sBY\sclause,\nthen\ssort\sin\sbatches\sto\shandle\sthe\slater\sterms. -D 2014-03-21T20:58:42.176 +C Fix\sthe\sORDER\sBY\soptimization\slogic\sso\sthat\sit\swill\sdo\sa\sblock-sort\son\na\spartial\sDESC\sORDER\sBY.\s\sThis\senhancement\suncovered\sa\smemory\sleak\sin\npushUntoSorter()\swhich\sis\salso\sfixed. +D 2014-03-22T00:27:14.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 633c56adeb382173d01016bea9e04e4dac54dfbc +F src/select.c 89bfc2228c6dda5200040d3c61a67a7fc815a24e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c f394453e4960462e62671158199f060ac956f8bc +F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 45d8cc678d128f1dda6469864215a8ed9de4366a 71e9ae72c272dc86720b2bfe719f57de437c400b -R 3371108f475a5908c384cc490e83cfb1 +P fa06a6fed9f48322d9b89721799ba12c46efa898 +R c7b944946040f3192afd04e740c2e57c U drh -Z 76333cbbd5592a854905d3a7e7bd7626 +Z 5cc077b206326620cdb0818e6e6de660 diff --git a/manifest.uuid b/manifest.uuid index 4099c22d66..a151346c9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa06a6fed9f48322d9b89721799ba12c46efa898 \ No newline at end of file +c36f74611cf17ad2ff198a2ac2054d7ab451a72c \ No newline at end of file diff --git a/src/select.c b/src/select.c index fe65e2909a..77ca0de8cc 100644 --- a/src/select.c +++ b/src/select.c @@ -488,6 +488,7 @@ static void pushOntoSorter( addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + 1; sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); diff --git a/src/where.c b/src/where.c index 03066043d4..15084f099a 100644 --- a/src/where.c +++ b/src/where.c @@ -4872,23 +4872,23 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 1; break; } + if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } if( isMatch ){ if( iColumn<0 ){ testcase( distinctColumns==0 ); distinctColumns = 1; } obSat |= MASKBIT(i); - if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ - /* Make sure the sort order is compatible in an ORDER BY clause. - ** Sort order is irrelevant for a GROUP BY clause. */ - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; - }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; - if( rev ) *pRevMask |= MASKBIT(iLoop); - revSet = 1; - } - } }else{ /* No match found */ if( j==0 || j Date: Sat, 22 Mar 2014 01:52:17 +0000 Subject: [PATCH 108/177] Increase the version number to 3.8.5. FossilOrigin-Name: 9760879d086b61476890927f01806a93798d4eee --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index a93056bbe2..0cbfaed0d9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.4.1 +3.8.5 diff --git a/configure b/configure index 56d58c7b73..8be06033f0 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.4.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.5. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.4.1' -PACKAGE_STRING='sqlite 3.8.4.1' +PACKAGE_VERSION='3.8.5' +PACKAGE_STRING='sqlite 3.8.5' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.4.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.5 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.4.1:";; + short | recursive ) echo "Configuration of sqlite 3.8.5:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.4.1 +sqlite configure 3.8.5 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.4.1, which was +It was created by sqlite $as_me 3.8.5, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.4.1, which was +This file was extended by sqlite $as_me 3.8.5, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.4.1 +sqlite config.status 3.8.5 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index eca837d211..4300f5a3d4 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sthe\sORDER\sBY\soptimization\slogic\sso\sthat\sit\swill\sdo\sa\sblock-sort\son\na\spartial\sDESC\sORDER\sBY.\s\sThis\senhancement\suncovered\sa\smemory\sleak\sin\npushUntoSorter()\swhich\sis\salso\sfixed. -D 2014-03-22T00:27:14.500 +C Increase\sthe\sversion\snumber\sto\s3.8.5. +D 2014-03-22T01:52:17.580 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc fdacba6fb574868c71fde6db6b77789a383a4c19 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION c3b0d47c3c5cf25c5bd4ff9e6f3af2f9d7934ea6 +F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 6a7a482649dd20b7ee31424bf6553c6f20ca7936 x +F configure 5de98fac90452f876511fee8a35565a55dfb191f x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fa06a6fed9f48322d9b89721799ba12c46efa898 -R c7b944946040f3192afd04e740c2e57c +P c36f74611cf17ad2ff198a2ac2054d7ab451a72c +R c87966e812e5349386ca47468f11243f U drh -Z 5cc077b206326620cdb0818e6e6de660 +Z 4135b8e90212e093f926ac90192216e8 diff --git a/manifest.uuid b/manifest.uuid index a151346c9a..4af93967f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c36f74611cf17ad2ff198a2ac2054d7ab451a72c \ No newline at end of file +9760879d086b61476890927f01806a93798d4eee \ No newline at end of file From dbfca2b7f89d761d330e7c8da551e1ca11b4f185 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Mar 2014 02:19:53 +0000 Subject: [PATCH 109/177] Clear the sort-order flags on the KeyInfo object associated with the OP_Compare/OP_Jump of the block-sort logic, in order to make the OP_Jump fully testable. FossilOrigin-Name: 9b2838f23e5f52369eeb078f60bd231cc54362f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4300f5a3d4..8802b6c3e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.5. -D 2014-03-22T01:52:17.580 +C Clear\sthe\ssort-order\sflags\son\sthe\sKeyInfo\sobject\sassociated\swith\sthe\nOP_Compare/OP_Jump\sof\sthe\sblock-sort\slogic,\sin\sorder\sto\smake\sthe\sOP_Jump\nfully\stestable. +D 2014-03-22T02:19:53.523 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 89bfc2228c6dda5200040d3c61a67a7fc815a24e +F src/select.c 269c3e31a450fce642a10569221a49180348c88e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c36f74611cf17ad2ff198a2ac2054d7ab451a72c -R c87966e812e5349386ca47468f11243f +P 9760879d086b61476890927f01806a93798d4eee +R 9c4612b02334319090d26af6339ff92f U drh -Z 4135b8e90212e093f926ac90192216e8 +Z c09bf15737bf88995e1679f9f666a1a6 diff --git a/manifest.uuid b/manifest.uuid index 4af93967f6..f209cb2ae6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9760879d086b61476890927f01806a93798d4eee \ No newline at end of file +9b2838f23e5f52369eeb078f60bd231cc54362f8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 77ca0de8cc..c7f0b24a4a 100644 --- a/src/select.c +++ b/src/select.c @@ -481,6 +481,7 @@ static void pushOntoSorter( int addrJmp; /* Address of the OP_Jump opcode */ VdbeOp *pOp; /* Opcode that opens the sorter */ int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; @@ -490,7 +491,9 @@ static void pushOntoSorter( pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + 1; - sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); From 323913c88080d5013c404cd90ef689467e29406a Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Mar 2014 16:29:23 +0000 Subject: [PATCH 110/177] Avoid a possible use of an uninitialized variable following an I/O or OOM error. FossilOrigin-Name: 641408a1395bfc911ca619ef9e5f073b913d856b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8802b6c3e7..b22c63dd88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sthe\ssort-order\sflags\son\sthe\sKeyInfo\sobject\sassociated\swith\sthe\nOP_Compare/OP_Jump\sof\sthe\sblock-sort\slogic,\sin\sorder\sto\smake\sthe\sOP_Jump\nfully\stestable. -D 2014-03-22T02:19:53.523 +C Avoid\sa\spossible\suse\sof\san\suninitialized\svariable\sfollowing\san\sI/O\sor\sOOM\nerror. +D 2014-03-23T16:29:23.966 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 37cfae03a0c40515304fb0a8a1a97ac3aa965e87 +F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9760879d086b61476890927f01806a93798d4eee -R 9c4612b02334319090d26af6339ff92f +P 9b2838f23e5f52369eeb078f60bd231cc54362f8 +R 5e47a0912e77eb3fe11c659549f1088a U drh -Z c09bf15737bf88995e1679f9f666a1a6 +Z 7a01e94679f2d88e0c9f56d7135921af diff --git a/manifest.uuid b/manifest.uuid index f209cb2ae6..7e88ad746f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b2838f23e5f52369eeb078f60bd231cc54362f8 \ No newline at end of file +641408a1395bfc911ca619ef9e5f073b913d856b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7104c6d017..b70c5e1c9d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4514,6 +4514,7 @@ case OP_SorterNext: { /* jump */ pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); + res = 0; rc = sqlite3VdbeSorterNext(db, pC, &res); goto next_tail; case OP_PrevIfOpen: /* jump */ From 3d1d90a13da2d8a617f86756b79d9289b03ed566 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 15:00:15 +0000 Subject: [PATCH 111/177] Fix arithmetic operators so that they do not change the affinity of their input operands. Ticket [a8a0d2996a]. FossilOrigin-Name: 221f8f944703108e47d789fa8ce6c00fe2abcbb6 --- manifest | 15 ++++--- manifest.uuid | 2 +- src/vdbe.c | 35 ++++++++++++--- test/tkt-a8a0d2996a.test | 93 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 test/tkt-a8a0d2996a.test diff --git a/manifest b/manifest index 3c6309f95b..9e30ebbe2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. -D 2014-03-24T11:23:17.736 +C Fix\sarithmetic\soperators\sso\sthat\sthey\sdo\snot\schange\sthe\saffinity\sof\stheir\ninput\soperands.\s\sTicket\s[a8a0d2996a]. +D 2014-03-24T15:00:15.271 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d +F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -892,6 +892,7 @@ F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 +F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 @@ -1158,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 641408a1395bfc911ca619ef9e5f073b913d856b e3d2be3ba47cdaafd26347620ae3bc2813203f16 -R aadeef09c98319ee1e1acbdf39605080 -U dan -Z a7c50b19ab534dd04ee960894cca4791 +P db7d62c8d58eb1e8654a762c9b199ae4e2759038 +R c2ec5fb627231252d937ea015ddf100a +U drh +Z 7360db1b973f463e9c0480db05633068 diff --git a/manifest.uuid b/manifest.uuid index 3a41778163..9344d79651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db7d62c8d58eb1e8654a762c9b199ae4e2759038 \ No newline at end of file +221f8f944703108e47d789fa8ce6c00fe2abcbb6 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b70c5e1c9d..84f720b526 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -310,6 +310,29 @@ void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or +** none. +** +** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. +** But it does set pMem->r and pMem->u.i appropriately. +*/ +static u16 numericType(Mem *pMem){ + if( pMem->flags & (MEM_Int|MEM_Real) ){ + return pMem->flags & (MEM_Int|MEM_Real); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ + if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; + } + return 0; +} + #ifdef SQLITE_DEBUG /* ** Write a nice string representation of the contents of cell pMem @@ -1351,20 +1374,22 @@ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ char bIntint; /* Started out as two integer operands */ - int flags; /* Combined MEM_* flags from both inputs */ + u16 flags; /* Combined MEM_* flags from both inputs */ + u16 type1; /* Numeric type of left operand */ + u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ i64 iB; /* Integer value of right operand */ double rA; /* Real value of left operand */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - applyNumericAffinity(pIn1); + type1 = numericType(pIn1); pIn2 = &aMem[pOp->p2]; - applyNumericAffinity(pIn2); + type2 = numericType(pIn2); pOut = &aMem[pOp->p3]; flags = pIn1->flags | pIn2->flags; if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; - if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ + if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; bIntint = 1; @@ -1420,7 +1445,7 @@ fp_math: } pOut->r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( (flags & MEM_Real)==0 && !bIntint ){ + if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); } #endif diff --git a/test/tkt-a8a0d2996a.test b/test/tkt-a8a0d2996a.test new file mode 100644 index 0000000000..6b15e410e7 --- /dev/null +++ b/test/tkt-a8a0d2996a.test @@ -0,0 +1,93 @@ +# 2014-03-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests to verify that arithmetic operators do not change the type of +# input operands. Ticket [a8a0d2996a] +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-a8a0d2996a + +do_execsql_test 1.0 { + CREATE TABLE t(x,y); + INSERT INTO t VALUES('1','1'); + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 1.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 1.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 1.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 1.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + +do_execsql_test 2.0 { + UPDATE t SET x='1xyzzy'; + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 2.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 2.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 2.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 2.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + + +do_execsql_test 3.0 { + UPDATE t SET x='1.0'; + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 3.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 3.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 3.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 3.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + +do_execsql_test 4.0 { + SELECT 1+1.; +} {2.0} +do_execsql_test 4.1 { + SELECT '1.23e64'/'1.0000e+62'; +} {123.0} +do_execsql_test 4.2 { + SELECT '100x'+'-2y'; +} {98} +do_execsql_test 4.3 { + SELECT '100x'+'4.5y'; +} {104.5} +do_execsql_test 4.4 { + SELECT '-9223372036854775807x'-'1x'; +} {-9.22337203685478e+18} +do_execsql_test 4.5 { + SELECT '9223372036854775806x'+'1x'; +} {9.22337203685478e+18} +do_execsql_test 4.6 { + SELECT '1234x'/'10y'; +} {123.4} From b22f38a7791fed647b172dff6b2b1cf308b6e66c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 16:30:06 +0000 Subject: [PATCH 112/177] Remove unused variables Parse.nColCache and Parse.iColCache. FossilOrigin-Name: 4d7551ce464c8038147e81667368924f2a7485a6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9e30ebbe2c..58b47e5f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sarithmetic\soperators\sso\sthat\sthey\sdo\snot\schange\sthe\saffinity\sof\stheir\ninput\soperands.\s\sTicket\s[a8a0d2996a]. -D 2014-03-24T15:00:15.271 +C Remove\sunused\svariables\sParse.nColCache\sand\sParse.iColCache. +D 2014-03-24T16:30:06.093 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 1e16bac177ce6396e59af867afb90c92949895de +F src/sqliteInt.h afbf39e96736ceb85e1d896b281ba2406dd70aa0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P db7d62c8d58eb1e8654a762c9b199ae4e2759038 -R c2ec5fb627231252d937ea015ddf100a -U drh -Z 7360db1b973f463e9c0480db05633068 +P 221f8f944703108e47d789fa8ce6c00fe2abcbb6 +R dc20bd716d0015e30edc93a05e72ad4b +U dan +Z 0a61715f43b39f80ef65df2d7add7843 diff --git a/manifest.uuid b/manifest.uuid index 9344d79651..de90fd9eab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -221f8f944703108e47d789fa8ce6c00fe2abcbb6 \ No newline at end of file +4d7551ce464c8038147e81667368924f2a7485a6 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 27dfc08e0f..5e0dd91ec9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2393,8 +2393,6 @@ struct Parse { u8 checkSchema; /* Causes schema cookie check after an error */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ - u8 nColCache; /* Number of entries in aColCache[] */ - u8 iColCache; /* Next entry in aColCache[] to replace */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ From 86dd3716b998bbc2ca2f1aad2a5c01eeb0d7ece8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 11:00:21 +0000 Subject: [PATCH 113/177] Detect when a VdbeCursor is still pointing at a valid row but that row has moved, and invalidated the return from prior sqlite3BtreeDataFetch() or sqlite3BtreeKeyFetch() calls. FossilOrigin-Name: e6798871ce94961135762669af418cd78540c121 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 23 +++++++++++++++++++---- src/vdbeaux.c | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 58b47e5f6f..e6372d6699 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\svariables\sParse.nColCache\sand\sParse.iColCache. -D 2014-03-24T16:30:06.093 +C Detect\swhen\sa\sVdbeCursor\sis\sstill\spointing\sat\sa\svalid\srow\sbut\sthat\srow\shas\nmoved,\sand\sinvalidated\sthe\sreturn\sfrom\sprior\ssqlite3BtreeDataFetch()\sor\nsqlite3BtreeKeyFetch()\scalls. +D 2014-03-25T11:00:21.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 029cec7b98fe0a985922c03f101630391044c4ad +F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c +F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 221f8f944703108e47d789fa8ce6c00fe2abcbb6 -R dc20bd716d0015e30edc93a05e72ad4b -U dan -Z 0a61715f43b39f80ef65df2d7add7843 +P 4d7551ce464c8038147e81667368924f2a7485a6 +R 8613a90c52863cb0b93f254b2d121d4b +U drh +Z f61d5d3cf5132dbd8d9ed764bc0739f9 diff --git a/manifest.uuid b/manifest.uuid index de90fd9eab..b106017e83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d7551ce464c8038147e81667368924f2a7485a6 \ No newline at end of file +e6798871ce94961135762669af418cd78540c121 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 486cf8e6d5..43d41d67e9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -746,20 +746,32 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ ** at is deleted out from under them. ** ** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set to one if the cursor has moved and 0 if not. +** integer *pHasMoved is set as follows: +** +** 0: The cursor is unchanged +** 1: The cursor is still pointing at the same row, but the pointers +** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() +** might now be invalid because of a balance() or other change to the +** b-tree. +** 2: The cursor is no longer pointing to the row. The row might have +** been deleted out from under the cursor. */ int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ int rc; + if( pCur->eState==CURSOR_VALID ){ + *pHasMoved = 0; + return SQLITE_OK; + } rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 1; + *pHasMoved = 2; return rc; } if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 1; + *pHasMoved = 2; }else{ - *pHasMoved = 0; + *pHasMoved = 1; } return SQLITE_OK; } @@ -4188,10 +4200,13 @@ static const void *fetchPayload( assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorHoldsMutex(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->info.nSize>0 ); +#if 0 if( pCur->info.nSize==0 ){ btreeParseCell(pCur->apPage[pCur->iPage], pCur->aiIdx[pCur->iPage], &pCur->info); } +#endif *pAmt = pCur->info.nLocal; return (void*)(pCur->info.pCell + pCur->info.nHeader); } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6f1fa2670b..0d2299bab1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2735,7 +2735,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( rc ) return rc; if( hasMoved ){ p->cacheStatus = CACHE_STALE; - p->nullRow = 1; + if( hasMoved==2 ) p->nullRow = 1; } } return SQLITE_OK; From 0c60c1fe6674d50ea394ffd9ec08201414c5251b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 14:54:36 +0000 Subject: [PATCH 114/177] Add an ORDER BY test case to speedtest1.c FossilOrigin-Name: 588122641e57e957813d329ea071e13ccbde5acd --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e6372d6699..5a5497d41f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Detect\swhen\sa\sVdbeCursor\sis\sstill\spointing\sat\sa\svalid\srow\sbut\sthat\srow\shas\nmoved,\sand\sinvalidated\sthe\sreturn\sfrom\sprior\ssqlite3BtreeDataFetch()\sor\nsqlite3BtreeKeyFetch()\scalls. -D 2014-03-25T11:00:21.320 +C Add\san\sORDER\sBY\stest\scase\sto\sspeedtest1.c +D 2014-03-25T14:54:36.731 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 1603da7b4897716f9df15bd71b0310f56ec3181e +F test/speedtest1.c bbf90952e0464e464467c72785602cca82a4d842 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4d7551ce464c8038147e81667368924f2a7485a6 -R 8613a90c52863cb0b93f254b2d121d4b +P e6798871ce94961135762669af418cd78540c121 +R 63e64c2147fb938edf250ea589ed8a9b U drh -Z f61d5d3cf5132dbd8d9ed764bc0739f9 +Z 815d7c70c40cd78a37bdf42f1d8f6b2e diff --git a/manifest.uuid b/manifest.uuid index b106017e83..f5a99b9470 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6798871ce94961135762669af418cd78540c121 \ No newline at end of file +588122641e57e957813d329ea071e13ccbde5acd \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 05160e0eeb..e525f9cb18 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -512,6 +512,26 @@ void testset_main(void){ speedtest1_end_test(); + n = g.szTest/5; + speedtest1_begin_test(145, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_exec("BEGIN"); + speedtest1_prepare( + "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" + " ORDER BY a LIMIT 10; -- %d times", n + ); + for(i=1; i<=n; i++){ + x1 = speedtest1_random()%maxb; + zNum[0] = '%'; + len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2); + zNum[len] = '%'; + zNum[len+1] = 0; + sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + speedtest1_begin_test(150, "CREATE INDEX five times"); speedtest1_exec("BEGIN;"); speedtest1_exec("CREATE UNIQUE INDEX t1b ON t1(b);"); From c872966ef483aee010e327af97e7e66b065236bb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 17:45:49 +0000 Subject: [PATCH 115/177] Add another performance test case to speedtest1. This case is another ORDER BY test but this time without LIMIT. FossilOrigin-Name: 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 21 ++++++++++++++++++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5a5497d41f..d9f3760dc0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sORDER\sBY\stest\scase\sto\sspeedtest1.c -D 2014-03-25T14:54:36.731 +C Add\sanother\sperformance\stest\scase\sto\sspeedtest1.\s\sThis\scase\sis\sanother\nORDER\sBY\stest\sbut\sthis\stime\swithout\sLIMIT. +D 2014-03-25T17:45:49.623 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c bbf90952e0464e464467c72785602cca82a4d842 +F test/speedtest1.c 2bec93920c6d26adfa781b509b1c9ca0a4d8f39d F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6798871ce94961135762669af418cd78540c121 -R 63e64c2147fb938edf250ea589ed8a9b +P 588122641e57e957813d329ea071e13ccbde5acd +R 6b6fe86503bb240eaed31036f5ef1e04 U drh -Z 815d7c70c40cd78a37bdf42f1d8f6b2e +Z 513f3088d16b4a60fe04e39489233db6 diff --git a/manifest.uuid b/manifest.uuid index f5a99b9470..0289c1d2a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -588122641e57e957813d329ea071e13ccbde5acd \ No newline at end of file +9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index e525f9cb18..b39493354d 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -513,7 +513,26 @@ void testset_main(void){ n = g.szTest/5; - speedtest1_begin_test(145, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_begin_test(142, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_exec("BEGIN"); + speedtest1_prepare( + "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" + " ORDER BY a; -- %d times", n + ); + for(i=1; i<=n; i++){ + x1 = speedtest1_random()%maxb; + zNum[0] = '%'; + len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2); + zNum[len] = '%'; + zNum[len+1] = 0; + sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + n = g.szTest/5; + speedtest1_begin_test(145, "%d SELECTS w/ORDER BY and LIMIT, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" From 5e8980ddb1394cf09d39ced95e8ce69680289daa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 20:28:38 +0000 Subject: [PATCH 116/177] Adjust the repeat counts on several tests in speedtest1 so that the relative time spent on the test is roughly the same regardless of --size. FossilOrigin-Name: 7922809ee0d3978ce9221fbb7df63aa0684d0c4d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d9f3760dc0..c0390453c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sanother\sperformance\stest\scase\sto\sspeedtest1.\s\sThis\scase\sis\sanother\nORDER\sBY\stest\sbut\sthis\stime\swithout\sLIMIT. -D 2014-03-25T17:45:49.623 +C Adjust\sthe\srepeat\scounts\son\sseveral\stests\sin\sspeedtest1\sso\sthat\sthe\srelative\ntime\sspent\son\sthe\stest\sis\sroughly\sthe\ssame\sregardless\sof\s--size. +D 2014-03-25T20:28:38.512 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 2bec93920c6d26adfa781b509b1c9ca0a4d8f39d +F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 588122641e57e957813d329ea071e13ccbde5acd -R 6b6fe86503bb240eaed31036f5ef1e04 +P 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d +R 8ec1b9b9d2439aad92a9a0c48f5fc5b8 U drh -Z 513f3088d16b4a60fe04e39489233db6 +Z 44fa9dae5e4277ca9cc29c89c29e96c1 diff --git a/manifest.uuid b/manifest.uuid index 0289c1d2a0..52ad9d7d36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d \ No newline at end of file +7922809ee0d3978ce9221fbb7df63aa0684d0c4d \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index b39493354d..28c24e0885 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -474,7 +474,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/2; + n = 25; speedtest1_begin_test(130, "%d SELECTS, numeric BETWEEN, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -492,7 +492,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/5; + n = 10; speedtest1_begin_test(140, "%d SELECTS, LIKE, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -512,7 +512,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/5; + n = 10; speedtest1_begin_test(142, "%d SELECTS w/ORDER BY, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -531,7 +531,7 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); - n = g.szTest/5; + n = 10; //g.szTest/5; speedtest1_begin_test(145, "%d SELECTS w/ORDER BY and LIMIT, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( From e2ac506704e76a26352626788910d62f00dd1f63 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 12:02:38 +0000 Subject: [PATCH 117/177] Add an assert() and a comment to clarify the operation of the vdbeRecordCompareInt() routine. FossilOrigin-Name: 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c0390453c4..ce3e7af4be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\srepeat\scounts\son\sseveral\stests\sin\sspeedtest1\sso\sthat\sthe\srelative\ntime\sspent\son\sthe\stest\sis\sroughly\sthe\ssame\sregardless\sof\s--size. -D 2014-03-25T20:28:38.512 +C Add\san\sassert()\sand\sa\scomment\sto\sclarify\sthe\soperation\sof\sthe\nvdbeRecordCompareInt()\sroutine. +D 2014-03-26T12:02:38.907 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a +F src/vdbeaux.c 9898720db88a8e733acce7d72fbd56f4565d6d82 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d -R 8ec1b9b9d2439aad92a9a0c48f5fc5b8 +P 7922809ee0d3978ce9221fbb7df63aa0684d0c4d +R 0dc0a56d9b4e58b2ad9ea1b209a9c778 U drh -Z 44fa9dae5e4277ca9cc29c89c29e96c1 +Z d2e0055a9ca933f7d0981ccf44961bc7 diff --git a/manifest.uuid b/manifest.uuid index 52ad9d7d36..7ab08ac131 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7922809ee0d3978ce9221fbb7df63aa0684d0c4d \ No newline at end of file +851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0d2299bab1..18862028e4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3589,6 +3589,9 @@ int sqlite3VdbeRecordCompare( ** that (a) the first field of pPKey2 is an integer, and (b) the ** size-of-header varint at the start of (pKey1/nKey1) fits in a single ** byte (i.e. is less than 128). +** +** To avoid concerns about buffer overreads, this routine is only used +** on schemas where the maximum valid header size is 63 bytes or less. */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ @@ -3605,6 +3608,7 @@ static int vdbeRecordCompareInt( UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); + assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ lhs = ONE_BYTE_INT(aKey); From 182ea4e3246fce6a468625812f1bf603b9690a30 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 14:51:07 +0000 Subject: [PATCH 118/177] Earlier detection of corruption in sqlite3VdbeRecordCompare() in order to avoid a potential buffer overread. FossilOrigin-Name: 28ddecff044dbc2dff50a7d8406ab67dfe06587f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 1 + test/corruptI.test | 6 +----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ce3e7af4be..b9c9cba60c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sand\sa\scomment\sto\sclarify\sthe\soperation\sof\sthe\nvdbeRecordCompareInt()\sroutine. -D 2014-03-26T12:02:38.907 +C Earlier\sdetection\sof\scorruption\sin\ssqlite3VdbeRecordCompare()\sin\sorder\sto\navoid\sa\spotential\sbuffer\soverread. +D 2014-03-26T14:51:07.017 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 9898720db88a8e733acce7d72fbd56f4565d6d82 +F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 88886ec9cd1bdba835263566bbf60ee009c6ea09 +F test/corruptI.test 645794bfc0bbcb962df34302a52118b0ed6a21fb F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7922809ee0d3978ce9221fbb7df63aa0684d0c4d -R 0dc0a56d9b4e58b2ad9ea1b209a9c778 +P 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 +R 08734d76a115f4a142a4825f21f3cdab U drh -Z d2e0055a9ca933f7d0981ccf44961bc7 +Z 2973842cba6cb939f8ad2063fb07de64 diff --git a/manifest.uuid b/manifest.uuid index 7ab08ac131..8e8ee672b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 \ No newline at end of file +28ddecff044dbc2dff50a7d8406ab67dfe06587f \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 18862028e4..f5e4b0a9fc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3434,6 +3434,7 @@ int sqlite3VdbeRecordCompare( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; + if( d1>(unsigned)nKey1 ) return 1; /* Corruption */ i = 0; } diff --git a/test/corruptI.test b/test/corruptI.test index 51cf64c9be..c2eec8ba70 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -40,13 +40,9 @@ do_test 1.2 { set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] set off [expr 2*1024 + $offset + 1] hexio_write test.db $off FF06 - - breakpoint - sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } -} {1 {database disk image is malformed}} +} {0 {}} finish_test - From d9c131c3664b224dc7be2f30149bc73b6c2b47c5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 15:05:56 +0000 Subject: [PATCH 119/177] Add a test case to verify that the previous change avoids excess buffer overread in sqlite3VdbeRecordCompare(). FossilOrigin-Name: 2b28e8d582cf10936fa1faca04a16ca2eeead66f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/corruptI.test | 13 +++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b9c9cba60c..be675fdb5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Earlier\sdetection\sof\scorruption\sin\ssqlite3VdbeRecordCompare()\sin\sorder\sto\navoid\sa\spotential\sbuffer\soverread. -D 2014-03-26T14:51:07.017 +C Add\sa\stest\scase\sto\sverify\sthat\sthe\sprevious\schange\savoids\sexcess\sbuffer\noverread\sin\ssqlite3VdbeRecordCompare(). +D 2014-03-26T15:05:56.169 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 645794bfc0bbcb962df34302a52118b0ed6a21fb +F test/corruptI.test 487e9e5eac227fca974eed0a0bd5791cccc9eb86 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 -R 08734d76a115f4a142a4825f21f3cdab +P 28ddecff044dbc2dff50a7d8406ab67dfe06587f +R 6b54752b8296f7dffb679821462bd1b5 U drh -Z 2973842cba6cb939f8ad2063fb07de64 +Z 7c53e6dad71c7413b3950e5c5e55a2eb diff --git a/manifest.uuid b/manifest.uuid index 8e8ee672b5..bdd003989c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28ddecff044dbc2dff50a7d8406ab67dfe06587f \ No newline at end of file +2b28e8d582cf10936fa1faca04a16ca2eeead66f \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index c2eec8ba70..bb8d984938 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -32,14 +32,23 @@ do_execsql_test 1.1 { PRAGMA auto_vacuum=0; CREATE TABLE t1(a); CREATE INDEX i1 ON t1(a); - INSERT INTO t1 VALUES('a'); + INSERT INTO t1 VALUES('abcdefghijklmnop'); } {} db close do_test 1.2 { set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] set off [expr 2*1024 + $offset + 1] - hexio_write test.db $off FF06 + hexio_write test.db $off 7f06 + sqlite3 db test.db + catchsql { SELECT * FROM t1 WHERE a = 10 } +} {0 {}} + +do_test 1.3 { + db close + set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] + set off [expr 2*1024 + $offset + 1] + hexio_write test.db $off FFFF7f02 sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } } {0 {}} From 68916c9fd3d9829a462a993ded1e58fca82637d2 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Mar 2014 15:14:59 +0000 Subject: [PATCH 120/177] Add an extra test case for the potential buffer overread patched by [28ddecff04]. FossilOrigin-Name: f585f5d7a0f9bf8c590388654a3638231eba8892 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/corruptI.test | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index be675fdb5a..fed419c7cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sto\sverify\sthat\sthe\sprevious\schange\savoids\sexcess\sbuffer\noverread\sin\ssqlite3VdbeRecordCompare(). -D 2014-03-26T15:05:56.169 +C Add\san\sextra\stest\scase\sfor\sthe\spotential\sbuffer\soverread\spatched\sby\s[28ddecff04]. +D 2014-03-26T15:14:59.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 487e9e5eac227fca974eed0a0bd5791cccc9eb86 +F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 28ddecff044dbc2dff50a7d8406ab67dfe06587f -R 6b54752b8296f7dffb679821462bd1b5 -U drh -Z 7c53e6dad71c7413b3950e5c5e55a2eb +P 2b28e8d582cf10936fa1faca04a16ca2eeead66f +R 76c12b0a4284d7654cb2771b2fc9c8eb +U dan +Z 22c3e1f8cdae5a20dae420dc313d8edd diff --git a/manifest.uuid b/manifest.uuid index bdd003989c..bdc4496d4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b28e8d582cf10936fa1faca04a16ca2eeead66f \ No newline at end of file +f585f5d7a0f9bf8c590388654a3638231eba8892 \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index bb8d984938..087a0f3b01 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -53,5 +53,27 @@ do_test 1.3 { catchsql { SELECT * FROM t1 WHERE a = 10 } } {0 {}} +do_test 2.0 { + execsql { + CREATE TABLE r(x); + INSERT INTO r VALUES('ABCDEFGHIJK'); + CREATE INDEX r1 ON r(x); + } + set pg [db one {SELECT rootpage FROM sqlite_master WHERE name = 'r1'}] +} {5} + +do_test 2.1 { + db close + set offset [hexio_get_int [hexio_read test.db [expr (5-1)*1024 + 8] 2]] + set off [expr (5-1)*1024 + $offset + 1] + hexio_write test.db $off FFFF0004 + sqlite3 db test.db + catchsql { SELECT * FROM r WHERE x >= 10.0 } +} {1 {database disk image is malformed}} + +do_test 2.2 { + catchsql { SELECT * FROM r WHERE x >= 10 } +} {1 {database disk image is malformed}} + finish_test From 382bdeabefc740398f8fd6e55d12a8d952965d85 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 14:05:38 +0000 Subject: [PATCH 121/177] Enhance the logest.c utility with new operators: "dup", "inv", "log", and "nlogn". Provide help on an invalid input. FossilOrigin-Name: b4bd2a062c4baf5f622d61b7411f00de5904ef56 --- manifest | 14 ++++++------ manifest.uuid | 2 +- tool/logest.c | 62 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index fed419c7cf..fc133ce7b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\stest\scase\sfor\sthe\spotential\sbuffer\soverread\spatched\sby\s[28ddecff04]. -D 2014-03-26T15:14:59.762 +C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. +D 2014-03-27T14:05:38.733 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1121,7 +1121,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 +F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b28e8d582cf10936fa1faca04a16ca2eeead66f -R 76c12b0a4284d7654cb2771b2fc9c8eb -U dan -Z 22c3e1f8cdae5a20dae420dc313d8edd +P f585f5d7a0f9bf8c590388654a3638231eba8892 +R fc449dde2f7cf3aac00c9e1a5da52aee +U drh +Z ebedee01c152a936b8dfbac0e8a85bc4 diff --git a/manifest.uuid b/manifest.uuid index bdc4496d4e..bf7192882e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f585f5d7a0f9bf8c590388654a3638231eba8892 \ No newline at end of file +b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 8dad6cc9b6..1ac337d36c 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -17,13 +17,7 @@ ** ** ./LogEst ARGS ** -** Arguments: -** -** 'x' Multiple the top two elements of the stack -** '+' Add the top two elements of the stack -** NUM Convert NUM from integer to LogEst and push onto the stack -** ^NUM Interpret NUM as a LogEst and push onto stack. -** +** See the showHelp() routine for a description of valid arguments. ** Examples: ** ** To convert 123 from LogEst to integer: @@ -97,12 +91,31 @@ static LogEst logEstFromDouble(double x){ return e*10; } +int isInteger(const char *z){ + while( z[0]>='0' && z[0]<='9' ) z++; + return z[0]==0; +} + int isFloat(const char *z){ - while( z[0] ){ - if( z[0]=='.' || z[0]=='E' || z[0]=='e' ) return 1; - z++; - } - return 0; + char c; + while( ((c=z[0])>='0' && c<='9') || c=='.' || c=='E' || c=='e' + || c=='+' || c=='-' ) z++; + return z[0]==0; +} + +static void showHelp(const char *zArgv0){ + printf("Usage: %s ARGS...\n", zArgv0); + printf("Arguments:\n" + " NUM Convert NUM from integer to LogEst and push onto the stack\n" + " ^NUM Interpret NUM as a LogEst and push onto stack\n" + " x Multiple the top two elements of the stack\n" + " + Add the top two elements of the stack\n" + " dup Dupliate the top element on the stack\n" + " inv Take the reciprocal of the top of stack. N = 1/N.\n" + " log Find the LogEst of the number on top of stack\n" + " nlogn Compute NlogN where N is the top of stack\n" + ); + exit(1); } int main(int argc, char **argv){ @@ -111,30 +124,43 @@ int main(int argc, char **argv){ LogEst a[100]; for(i=1; i=2 ){ a[n-2] = logEstAdd(a[n-2],a[n-1]); n--; } - }else if( z[0]=='x' ){ + }else if( strcmp(z,"x")==0 ){ if( n>=2 ){ a[n-2] = logEstMultiply(a[n-2],a[n-1]); n--; } + }else if( strcmp(z,"dup")==0 ){ + if( n>0 ){ + a[n] = a[n-1]; + n++; + } + }else if( strcmp(z,"log")==0 ){ + if( n>0 ) a[n-1] = logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"nlogn")==0 ){ + if( n>0 ) a[n-1] += logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"inv")==0 ){ + if( n>0 ) a[n-1] = -a[n-1]; }else if( z[0]=='^' ){ a[n++] = atoi(z+1); - }else if( isFloat(z) ){ + }else if( isInteger(z) ){ + a[n++] = logEstFromInteger(atoi(z)); + }else if( isFloat(z) && z[0]!='-' ){ a[n++] = logEstFromDouble(atof(z)); }else{ - a[n++] = logEstFromInteger(atoi(z)); + showHelp(argv[0]); } } for(i=n-1; i>=0; i--){ if( a[i]<0 ){ - printf("%d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; - printf("%d (%lld.%02lld)\n", a[i], x/100, x%100); + printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); } } return 0; From 27de5c5cb623fcdbfe4aa128a30a08a5efa268f4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 18:36:34 +0000 Subject: [PATCH 122/177] Minor cleanup of the code in the query planner that computes the costs estimates for the various plans. There are no changes to the costs at this time. But the code is slightly more readable now and that might facilitate future enhancements. FossilOrigin-Name: 9b4d7226bcee38be5ac68a54bee03b4179cb69fc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 44 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index fc133ce7b9..c8233bfa12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. -D 2014-03-27T14:05:38.733 +C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. +D 2014-03-27T18:36:34.321 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca +F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f585f5d7a0f9bf8c590388654a3638231eba8892 -R fc449dde2f7cf3aac00c9e1a5da52aee +P b4bd2a062c4baf5f622d61b7411f00de5904ef56 +R 96966a646967b8a2388d32a924ac0ae2 U drh -Z ebedee01c152a936b8dfbac0e8a85bc4 +Z 308d20d60c0618b1b3ae79dc397c2638 diff --git a/manifest.uuid b/manifest.uuid index bf7192882e..65bd2a013d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file +9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file diff --git a/src/where.c b/src/where.c index 15084f099a..16e7e40bc3 100644 --- a/src/where.c +++ b/src/where.c @@ -4328,18 +4328,34 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: The base cost of an index scan is N + log2(N). + ** The log2(N) is for the initial seek to the beginning and the N + ** is for the scan itself. */ + pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); if( m==0 ){ /* TUNING: Cost of a covering index scan is K*(N + log2(N)). ** + The extra factor K of between 1.1 and 3.0 that depends ** on the relative sizes of the table and the index. K ** is smaller for smaller indices, thus favoring them. + ** The upper bound on K (3.0) matches the penalty factor + ** on a full table scan that tries to encourage the use of + ** indexed lookups over full scans. */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 1 + - (15*pProbe->szIdxRow)/pTab->szTabRow; + pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; }else{ - /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) - ** which we will simplify to just N*log2(N) */ - pNew->rRun = rSize + rLogSize; + /* TUNING: The cost of scanning a non-covering index is multiplied + ** by log2(N) to account for the binary search of the main table + ** that must happen for each row of the index. + ** TODO: Should there be a multiplier here, analogous to the 3x + ** multiplier for a fulltable scan or covering index scan, to + ** further discourage the use of an index scan? Or is the log2(N) + ** term sufficient discouragement? + ** TODO: What if some or all of the WHERE clause terms can be + ** computed without reference to the original table. Then the + ** penality should reduce to logK where K is the number of output + ** rows. + */ + pNew->rRun += rLogSize; } whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); @@ -5041,11 +5057,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 ); + rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; + rSortCost = nRowEst + estLog(nRowEst) + rScale; /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ From a1f7c0a21c65431a3fbfeba25ab7f1e9ad827273 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 03:12:48 +0000 Subject: [PATCH 123/177] Enhance the sqlite3VdbeRecordCompare() routines so that if they encounter database corruption, they will set the UnpackedRecord.isCorrupt field and return 0. The sqlite3BtreeMovetoUnpacked() routine detects this and returns SQLITE_CORRUPT, causing the corruption to be reported back to the top-level. FossilOrigin-Name: 7fa85eaaaf6d211378620d728a759fdfe30a15b0 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 3 +++ src/sqliteInt.h | 1 + src/vdbe.h | 4 ++-- src/vdbeInt.h | 2 +- src/vdbeaux.c | 27 +++++++++++++++++++-------- test/corruptG.test | 11 +++-------- test/corruptI.test | 2 +- 9 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index c8233bfa12..48ca1421b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. -D 2014-03-27T18:36:34.321 +C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. +D 2014-03-28T03:12:48.763 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1 +F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -222,7 +222,7 @@ F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h afbf39e96736ceb85e1d896b281ba2406dd70aa0 +F src/sqliteInt.h cc9582a91b2910404ccda7b7e198815ea0f75948 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -279,10 +279,10 @@ F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 -F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 +F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 +F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd +F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -403,9 +403,9 @@ F test/corruptC.test 02405cf7ed0c1e989060e1aab6d02ffbc3906fbb F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 -F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d +F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff +F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b4bd2a062c4baf5f622d61b7411f00de5904ef56 -R 96966a646967b8a2388d32a924ac0ae2 +P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc +R e6ff5922142fb694f528da1c1b2a1baf U drh -Z 308d20d60c0618b1b3ae79dc397c2638 +Z 2ba581858b0e7a94365bc5b4fc44e179 diff --git a/manifest.uuid b/manifest.uuid index 65bd2a013d..ef58fd2f6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file +7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 43d41d67e9..c3055836c9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4588,6 +4588,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->isCorrupt = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4711,6 +4712,7 @@ int sqlite3BtreeMovetoUnpacked( c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } + assert( pIdxKey->isCorrupt==0 || c==0 ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4720,6 +4722,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; + if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5e0dd91ec9..0f67748fc8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1627,6 +1627,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ + u8 isCorrupt; /* Corruption detected by xRecordCompare() */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ diff --git a/src/vdbe.h b/src/vdbe.h index 8e300b88a6..10a4140ee1 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -211,10 +211,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,const UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,const UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b752478035..043b56da59 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -392,7 +392,7 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,const UnpackedRecord*,int*); +int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3VdbeExec(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f5e4b0a9fc..0ce21378d5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3405,10 +3405,13 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** Key1 and Key2 do not have to contain the same number of fields. If all ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. +** +** If database corruption is discovered, set pPKey2->isCorrupt to non-zero +** and return 0. */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ @@ -3434,7 +3437,10 @@ int sqlite3VdbeRecordCompare( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( d1>(unsigned)nKey1 ) return 1; /* Corruption */ + if( d1>(unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } i = 0; } @@ -3511,7 +3517,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; @@ -3537,7 +3544,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3596,7 +3604,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; @@ -3694,7 +3702,7 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip ){ const u8 *aKey1 = (const u8*)pKey1; @@ -3715,7 +3723,10 @@ static int vdbeRecordCompareString( int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; - if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ + if( (szHdr + nStr) > nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); @@ -3880,7 +3891,7 @@ idx_rowid_corruption: */ int sqlite3VdbeIdxKeyCompare( VdbeCursor *pC, /* The cursor to compare against */ - const UnpackedRecord *pUnpacked, /* Unpacked version of key */ + UnpackedRecord *pUnpacked, /* Unpacked version of key */ int *res /* Write the comparison result here */ ){ i64 nCellKey = 0; diff --git a/test/corruptG.test b/test/corruptG.test index 1ec5d6f6a9..af920edf41 100644 --- a/test/corruptG.test +++ b/test/corruptG.test @@ -47,12 +47,12 @@ do_test 1.2 { catchsql { SELECT c FROM t1 WHERE a>'abc'; } -} {0 {}} +} {1 {database disk image is malformed}} do_test 1.3 { catchsql { PRAGMA integrity_check } -} {0 ok} +} {1 {database disk image is malformed}} do_test 1.4 { catchsql { SELECT c FROM t1 ORDER BY a; @@ -71,11 +71,6 @@ do_test 2.1 { catchsql { SELECT rowid FROM t1 WHERE a='abc' and b='xyz123456789XYZ'; } - # The following test result is brittle. The point above is to try to - # force a buffer overread by a corrupt database file. If we get an - # incorrect answer from a corrupt database file, that is OK. If the - # result below changes, that just means that "undefined behavior" has - # changed. -} {/0 .*/} +} {1 {database disk image is malformed}} finish_test diff --git a/test/corruptI.test b/test/corruptI.test index 087a0f3b01..ed34c0f8c3 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -51,7 +51,7 @@ do_test 1.3 { hexio_write test.db $off FFFF7f02 sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } -} {0 {}} +} {1 {database disk image is malformed}} do_test 2.0 { execsql { From 36ed03428d501b8de6b86aeca09f27c9ec5869ad Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 12:56:57 +0000 Subject: [PATCH 124/177] Fix a harmless compiler warning. FossilOrigin-Name: a4e47150f32b3a4120b1f89ccc66d633d829e3bb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 48ca1421b2..3f74055a6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. -D 2014-03-28T03:12:48.763 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-03-28T12:56:57.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d +F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc -R e6ff5922142fb694f528da1c1b2a1baf +P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 +R 84c408a5c4c5e7a44048ef6100ea9043 U drh -Z 2ba581858b0e7a94365bc5b4fc44e179 +Z 645b10a99877639c66e2a5f7ed2d5a66 diff --git a/manifest.uuid b/manifest.uuid index ef58fd2f6f..b73eec43f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file +a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file diff --git a/src/where.c b/src/where.c index 16e7e40bc3..93ee8c59c3 100644 --- a/src/where.c +++ b/src/where.c @@ -4936,7 +4936,7 @@ static i8 wherePathSatisfiesOrderBy( } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return nOrderBy; + if( obSat==obDone ) return (i8)nOrderBy; if( !isOrderDistinct ){ for(i=nOrderBy-1; i>0; i--){ Bitmask m = MASKBIT(i) - 1; From 70c93204ea3e1b37750dd7f362dfe379101cba68 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 14:41:35 +0000 Subject: [PATCH 125/177] Disable the wal64k.test script for non-unix systems since it depends on unix-only features. FossilOrigin-Name: 27deb6e49bcc76714dbdc61b34748603155ac770 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal64k.test | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f74055a6f..52d212de1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2014-03-28T12:56:57.685 +C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. +D 2014-03-28T14:41:35.536 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1056,7 +1056,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e -F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698 +F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 -R 84c408a5c4c5e7a44048ef6100ea9043 +P a4e47150f32b3a4120b1f89ccc66d633d829e3bb +R 578568415288cd31fe0adba6128329da U drh -Z 645b10a99877639c66e2a5f7ed2d5a66 +Z 045e2748905f8bd05ecf1197b97d7f20 diff --git a/manifest.uuid b/manifest.uuid index b73eec43f2..d9c988d4d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file +27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file diff --git a/test/wal64k.test b/test/wal64k.test index afd820778e..e962da128e 100644 --- a/test/wal64k.test +++ b/test/wal64k.test @@ -19,6 +19,11 @@ set testprefix wal64k ifcapable !wal {finish_test ; return } +if {$tcl_platform(platform) != "unix"} { + finish_test + return +} + db close test_syscall pagesize 65536 sqlite3 db test.db @@ -44,4 +49,3 @@ integrity_check 1.3 db close test_syscall pagesize -1 finish_test - From 7a4b1642a732f6ef183931bb51dc5b5a0dcdb502 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 29 Mar 2014 21:16:07 +0000 Subject: [PATCH 126/177] Experiments in picking better query plans, especially when the usage of one index is a subset of another. FossilOrigin-Name: 8f869ca7a6eaa9ca7a08102290e6c606735f9090 --- manifest | 15 +++-- manifest.uuid | 2 +- src/where.c | 176 +++++++++++++++++++++++++++++++------------------- 3 files changed, 119 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index 52d212de1d..9ef65fa6b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. -D 2014-03-28T14:41:35.536 +C Experiments\sin\spicking\sbetter\squery\splans,\sespecially\swhen\sthe\susage\sof\sone\nindex\sis\sa\ssubset\sof\sanother. +D 2014-03-29T21:16:07.103 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb +F src/where.c 26c6f6260b0cc2b0038dcc68ace0d7db20c7dcee F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a4e47150f32b3a4120b1f89ccc66d633d829e3bb -R 578568415288cd31fe0adba6128329da +P 27deb6e49bcc76714dbdc61b34748603155ac770 +R 1df7985395668b26c614cc4e641f29c8 +T *branch * query-plan-experiments +T *sym-query-plan-experiments * +T -sym-trunk * U drh -Z 045e2748905f8bd05ecf1197b97d7f20 +Z 1c5ff3f986da6bbe350c60cf72c963dd diff --git a/manifest.uuid b/manifest.uuid index d9c988d4d5..05916c40aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file +8f869ca7a6eaa9ca7a08102290e6c606735f9090 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 93ee8c59c3..c2b2cf2111 100644 --- a/src/where.c +++ b/src/where.c @@ -3712,59 +3712,24 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Insert or replace a WhereLoop entry using the template supplied. +** Search the list of WhereLoops in *ppPrev looking for one that can be +** supplanted by pTemplate. ** -** An existing WhereLoop entry might be overwritten if the new template -** is better and has fewer dependencies. Or the template will be ignored -** and no insert will occur if an existing WhereLoop is faster and has -** fewer dependencies than the template. Otherwise a new WhereLoop is -** added based on the template. +** Return NULL if the WhereLoop list contains an entry that can supplant +** pTemplate, in other words if pTemplate does not belong on the list. ** -** If pBuilder->pOrSet is not NULL then we only care about only the -** prerequisites and rRun and nOut costs of the N best loops. That -** information is gathered in the pBuilder->pOrSet object. This special -** processing mode is used only for OR clause processing. +** If pX is a WhereLoop that pTemplate can supplant, then return the +** link that points to pX. ** -** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we -** still might overwrite similar loops with the new template if the -** template is better. Loops may be overwritten if the following -** conditions are met: -** -** (1) They have the same iTab. -** (2) They have the same iSortIdx. -** (3) The template has same or fewer dependencies than the current loop -** (4) The template has the same or lower cost than the current loop -** (5) The template uses more terms of the same index but has no additional -** dependencies +** If pTemplate cannot supplant any existing element of the list but needs +** to be added to the list, then return a pointer to the tail of the list. */ -static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ - WhereLoop **ppPrev, *p, *pNext = 0; - WhereInfo *pWInfo = pBuilder->pWInfo; - sqlite3 *db = pWInfo->pParse->db; - - /* If pBuilder->pOrSet is defined, then only keep track of the costs - ** and prereqs. - */ - if( pBuilder->pOrSet!=0 ){ -#if WHERETRACE_ENABLED - u16 n = pBuilder->pOrSet->n; - int x = -#endif - whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, - pTemplate->nOut); -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; - } - - /* Search for an existing WhereLoop to overwrite, or which takes - ** priority over pTemplate. - */ - for(ppPrev=&pWInfo->pLoops, p=*ppPrev; p; ppPrev=&p->pNextLoop, p=*ppPrev){ +static WhereLoop **whereLoopFindLesser( + WhereLoop **ppPrev, + const WhereLoop *pTemplate +){ + WhereLoop *p; + for(p=(*ppPrev); p; ppPrev=&p->pNextLoop, p=*ppPrev){ if( p->iTab!=pTemplate->iTab || p->iSortIdx!=pTemplate->iSortIdx ){ /* If either the iTab or iSortIdx values for two WhereLoop are different ** then those WhereLoops need to be considered separately. Neither is @@ -3799,12 +3764,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ){ /* Overwrite an existing WhereLoop with an similar one that uses ** more terms of the index */ - pNext = p->pNextLoop; break; }else{ - /* pTemplate is not helpful. - ** Return without changing or adding anything */ - goto whereLoopInsert_noop; + /* There is an existing WhereLoop that is better than pTemplate */ + return 0; } } if( (p->prereq & pTemplate->prereq)==pTemplate->prereq @@ -3816,10 +3779,79 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** or (4) number of output rows, and is no worse in any of those ** categories. */ assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - pNext = p->pNextLoop; break; } } + return ppPrev; +} + +/* +** Insert or replace a WhereLoop entry using the template supplied. +** +** An existing WhereLoop entry might be overwritten if the new template +** is better and has fewer dependencies. Or the template will be ignored +** and no insert will occur if an existing WhereLoop is faster and has +** fewer dependencies than the template. Otherwise a new WhereLoop is +** added based on the template. +** +** If pBuilder->pOrSet is not NULL then we care about only the +** prerequisites and rRun and nOut costs of the N best loops. That +** information is gathered in the pBuilder->pOrSet object. This special +** processing mode is used only for OR clause processing. +** +** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we +** still might overwrite similar loops with the new template if the +** template is better. Loops may be overwritten if the following +** conditions are met: +** +** (1) They have the same iTab. +** (2) They have the same iSortIdx. +** (3) The template has same or fewer dependencies than the current loop +** (4) The template has the same or lower cost than the current loop +** (5) The template uses more terms of the same index but has no additional +** dependencies +*/ +static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ + WhereLoop **ppPrev, *p; + WhereInfo *pWInfo = pBuilder->pWInfo; + sqlite3 *db = pWInfo->pParse->db; + + /* If pBuilder->pOrSet is defined, then only keep track of the costs + ** and prereqs. + */ + if( pBuilder->pOrSet!=0 ){ +#if WHERETRACE_ENABLED + u16 n = pBuilder->pOrSet->n; + int x = +#endif + whereOrInsert(pBuilder->pOrSet, pTemplate->prereq, pTemplate->rRun, + pTemplate->nOut); +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + } + + /* Look for an existing WhereLoop to replace with pTemplate + */ + ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); + + if( ppPrev==0 ){ + /* There already exists a WhereLoop on the list that is better + ** than pTemplate, so just ignore pTemplate */ +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-noop: "); + whereLoopPrint(pTemplate, pBuilder->pWC); + } +#endif + return SQLITE_OK; + }else{ + p = *ppPrev; + } /* If we reach this point it means that either p[] should be overwritten ** with pTemplate[] if p[] exists, or if p==NULL then allocate a new @@ -3836,13 +3868,33 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } #endif if( p==0 ){ - p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); + /* Allocate a new WhereLoop to add to the end of the list */ + *ppPrev = p = sqlite3DbMallocRaw(db, sizeof(WhereLoop)); if( p==0 ) return SQLITE_NOMEM; whereLoopInit(p); + p->pNextLoop = 0; + }else{ + /* We will be overwriting WhereLoop p[]. But before we do, first + ** go through the rest of the list and delete any other entries besides + ** p[] that are also supplated by pTemplate */ + WhereLoop **ppTail = &p->pNextLoop; + WhereLoop *pToDel; + while( *ppTail ){ + ppTail = whereLoopFindLesser(ppTail, pTemplate); + if( NEVER(ppTail==0) ) break; + pToDel = *ppTail; + if( pToDel==0 ) break; + *ppTail = pToDel->pNextLoop; +#if WHERETRACE_ENABLED /* 0x8 */ + if( sqlite3WhereTrace & 0x8 ){ + sqlite3DebugPrintf("ins-del: "); + whereLoopPrint(pToDel, pBuilder->pWC); + } +#endif + whereLoopDelete(db, pToDel); + } } whereLoopXfer(db, p, pTemplate); - p->pNextLoop = pNext; - *ppPrev = p; if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; if( pIndex && pIndex->tnum==0 ){ @@ -3850,16 +3902,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ } } return SQLITE_OK; - - /* Jump here if the insert is a no-op */ -whereLoopInsert_noop: -#if WHERETRACE_ENABLED /* 0x8 */ - if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); - whereLoopPrint(pTemplate, pBuilder->pWC); - } -#endif - return SQLITE_OK; } /* From b663587890f557c9926964710f6f1e8121040c88 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 13:42:42 +0000 Subject: [PATCH 127/177] Avoid a (harmless) buffer overread that is possible on an OOM when MEMSYS5 is engaged. FossilOrigin-Name: b3296267fb67b9f59719a37093253062edde3746 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem5.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52d212de1d..887bb92f5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. -D 2014-03-28T14:41:35.536 +C Avoid\sa\s(harmless)\sbuffer\soverread\sthat\sis\spossible\son\san\sOOM\swhen\nMEMSYS5\sis\sengaged. +D 2014-03-31T13:42:42.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 -F src/mem5.c aeb019f271ea53de83d651ec526877e6ba863450 +F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a4e47150f32b3a4120b1f89ccc66d633d829e3bb -R 578568415288cd31fe0adba6128329da +P 27deb6e49bcc76714dbdc61b34748603155ac770 +R 74e3f11f049d7792447568e00e413b7f U drh -Z 045e2748905f8bd05ecf1197b97d7f20 +Z 4989508149fff1b415b46911a02fce9e diff --git a/manifest.uuid b/manifest.uuid index d9c988d4d5..70f95ac337 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file +b3296267fb67b9f59719a37093253062edde3746 \ No newline at end of file diff --git a/src/mem5.c b/src/mem5.c index 5d75611a32..67615bb964 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -248,7 +248,7 @@ static void *memsys5MallocUnsafe(int nByte){ ** block. If not, then split a block of the next larger power of ** two in order to create a new free block of size iLogsize. */ - for(iBin=iLogsize; mem5.aiFreelist[iBin]<0 && iBin<=LOGMAX; iBin++){} + for(iBin=iLogsize; iBin<=LOGMAX && mem5.aiFreelist[iBin]<0; iBin++){} if( iBin>LOGMAX ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte); From 53cd10afcd77c4ff2296999a9fe5116c51333688 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 18:24:18 +0000 Subject: [PATCH 128/177] Make sure that an index that covers a proper superset of the WHERE clause terms of some other index has a lower cost than the other index. FossilOrigin-Name: ea8b0910040198751551b0b960e6b783913607df --- manifest | 15 ++++---- manifest.uuid | 2 +- src/where.c | 99 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 74 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 9ef65fa6b9..ac1faf182f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\sin\spicking\sbetter\squery\splans,\sespecially\swhen\sthe\susage\sof\sone\nindex\sis\sa\ssubset\sof\sanother. -D 2014-03-29T21:16:07.103 +C Make\ssure\sthat\san\sindex\sthat\scovers\sa\sproper\ssuperset\sof\sthe\sWHERE\sclause\nterms\sof\ssome\sother\sindex\shas\sa\slower\scost\sthan\sthe\sother\sindex. +D 2014-03-31T18:24:18.867 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 26c6f6260b0cc2b0038dcc68ace0d7db20c7dcee +F src/where.c 50ac3154473b5c8df15c7b8dbd19da385fa859e1 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,10 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 27deb6e49bcc76714dbdc61b34748603155ac770 -R 1df7985395668b26c614cc4e641f29c8 -T *branch * query-plan-experiments -T *sym-query-plan-experiments * -T -sym-trunk * +P 8f869ca7a6eaa9ca7a08102290e6c606735f9090 +R 76b8d8088ba83ec014dc8bf476fa6749 U drh -Z 1c5ff3f986da6bbe350c60cf72c963dd +Z ecebe13d1c9f7ea1da970c5f6118dc52 diff --git a/manifest.uuid b/manifest.uuid index 05916c40aa..649fc9530b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f869ca7a6eaa9ca7a08102290e6c606735f9090 \ No newline at end of file +ea8b0910040198751551b0b960e6b783913607df \ No newline at end of file diff --git a/src/where.c b/src/where.c index c2b2cf2111..38c3712977 100644 --- a/src/where.c +++ b/src/where.c @@ -3711,6 +3711,51 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } +/* +** Compare every WhereLoop X on the list p against pTemplate. If: +** +** (1) both X and pTemplate refer to the same table, and +** (2) both X and pTemplate use a single index, and +** (3) pTemplate uses all the same WHERE clause terms as X plus +** at least one more term, +** +** then make sure the cost pTemplate is less than the cost of X, adjusting +** the cost of pTemplate downward if necessary. +** +** Example: When computing the query plan for the SELECT below: +** +** CREATE TABLE t1(a,b,c,d); +** CREATE INDEX t1abc ON t1(a,b,c); +** CREATE INDEX t1bc ON t1(b,c); +** SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +** +** Make sure the cost of using three three columns of index t1abc is less +** than the cost of using both columns of t1bc. +*/ +static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ + if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( pTemplate->nLTerm==0 ) return; + for(; p; p=p->pNextLoop){ + if( p->iTab==pTemplate->iTab + && (p->wsFlags & WHERE_INDEXED)!=0 + && p->nLTermnLTerm + && (p->rRunrRun || (p->rRun==pTemplate->rRun && + p->nOut<=pTemplate->nOut)) + ){ + int i, j; + for(j=0, i=p->nLTerm-1; i>=0 && j>=0; i--){ + for(j=pTemplate->nLTerm-1; j>=0; j--){ + if( pTemplate->aLTerm[j]==p->aLTerm[i] ) break; + } + } + if( j>=0 ){ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + } + } + } +} + /* ** Search the list of WhereLoops in *ppPrev looking for one that can be ** supplanted by pTemplate. @@ -3747,39 +3792,30 @@ static WhereLoop **whereLoopFindLesser( ** rSetup. Call this SETUP-INVARIANT */ assert( p->rSetup>=pTemplate->rSetup ); - if( (p->prereq & pTemplate->prereq)==p->prereq - && p->rSetup<=pTemplate->rSetup - && p->rRun<=pTemplate->rRun - && p->nOut<=pTemplate->nOut + /* If existing WhereLoop p is better than pTemplate, pTemplate can be + ** discarded. WhereLoop p is better if: + ** (1) p has no more dependencies than pTemplate, and + ** (2) p has an equal or lower cost than pTemplate + */ + if( (p->prereq & pTemplate->prereq)==p->prereq /* (1) */ + && p->rSetup<=pTemplate->rSetup /* (2a) */ + && p->rRun<=pTemplate->rRun /* (2b) */ + && p->nOut<=pTemplate->nOut /* (2c) */ ){ - /* This branch taken when p is equal or better than pTemplate in - ** all of (1) dependencies (2) setup-cost, (3) run-cost, and - ** (4) number of output rows. */ - assert( p->rSetup==pTemplate->rSetup ); - if( p->prereq==pTemplate->prereq - && p->nLTermnLTerm - && (p->wsFlags & pTemplate->wsFlags & WHERE_INDEXED)!=0 - && (p->u.btree.pIndex==pTemplate->u.btree.pIndex - || pTemplate->rRun+p->nLTerm<=p->rRun+pTemplate->nLTerm) - ){ - /* Overwrite an existing WhereLoop with an similar one that uses - ** more terms of the index */ - break; - }else{ - /* There is an existing WhereLoop that is better than pTemplate */ - return 0; - } + return 0; /* Discard pTemplate */ } - if( (p->prereq & pTemplate->prereq)==pTemplate->prereq - && p->rRun>=pTemplate->rRun - && p->nOut>=pTemplate->nOut + + /* If pTemplate is always better than p, then cause p to be overwritten + ** with pTemplate. pTemplate is better than p if: + ** (1) pTemplate has no more dependences than p, and + ** (2) pTemplate has an equal or lower cost than p. + */ + if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ + && p->rRun>=pTemplate->rRun /* (2a) */ + && p->nOut>=pTemplate->nOut /* (2b) */ ){ - /* Overwrite an existing WhereLoop with a better one: one that is - ** better at one of (1) dependencies, (2) setup-cost, (3) run-cost - ** or (4) number of output rows, and is no worse in any of those - ** categories. */ assert( p->rSetup>=pTemplate->rSetup ); /* SETUP-INVARIANT above */ - break; + break; /* Cause p to be overwritten by pTemplate */ } } return ppPrev; @@ -3801,15 +3837,13 @@ static WhereLoop **whereLoopFindLesser( ** ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we ** still might overwrite similar loops with the new template if the -** template is better. Loops may be overwritten if the following +** new template is better. Loops may be overwritten if the following ** conditions are met: ** ** (1) They have the same iTab. ** (2) They have the same iSortIdx. ** (3) The template has same or fewer dependencies than the current loop ** (4) The template has the same or lower cost than the current loop -** (5) The template uses more terms of the same index but has no additional -** dependencies */ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ WhereLoop **ppPrev, *p; @@ -3837,6 +3871,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ /* Look for an existing WhereLoop to replace with pTemplate */ + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); if( ppPrev==0 ){ From 3fb183d2f7f83cdc9cb406dccbe97d731f1e0644 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 19:49:00 +0000 Subject: [PATCH 129/177] Also make sure an index that is a proper subset of some other index has a higher cost than that other index. Add test cases. FossilOrigin-Name: b7830d232b073a197aa1092e78cb24e88cb10fd3 --- manifest | 13 +++-- manifest.uuid | 2 +- src/where.c | 68 +++++++++++++---------- test/whereH.test | 139 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 185 insertions(+), 37 deletions(-) create mode 100644 test/whereH.test diff --git a/manifest b/manifest index ac1faf182f..7824fe6caa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\san\sindex\sthat\scovers\sa\sproper\ssuperset\sof\sthe\sWHERE\sclause\nterms\sof\ssome\sother\sindex\shas\sa\slower\scost\sthan\sthe\sother\sindex. -D 2014-03-31T18:24:18.867 +C Also\smake\ssure\san\sindex\sthat\sis\sa\sproper\ssubset\sof\ssome\sother\sindex\shas\sa\nhigher\scost\sthan\sthat\sother\sindex.\s\sAdd\stest\scases. +D 2014-03-31T19:49:00.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 50ac3154473b5c8df15c7b8dbd19da385fa859e1 +F src/where.c 182f16d91060418dfcc7401d24e43d8ec24e026c F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1093,6 +1093,7 @@ F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test eb3a46b3eaf38e25e3013433b2db8a25a866c215 +F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1159,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8f869ca7a6eaa9ca7a08102290e6c606735f9090 -R 76b8d8088ba83ec014dc8bf476fa6749 +P ea8b0910040198751551b0b960e6b783913607df +R 176e1ff66c14893be5de17b3890f1972 U drh -Z ecebe13d1c9f7ea1da970c5f6118dc52 +Z 3bf4657410567331f2f5534ba4eae60e diff --git a/manifest.uuid b/manifest.uuid index 649fc9530b..a0ffd4b3db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea8b0910040198751551b0b960e6b783913607df \ No newline at end of file +b7830d232b073a197aa1092e78cb24e88cb10fd3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 38c3712977..9882b4873f 100644 --- a/src/where.c +++ b/src/where.c @@ -3712,46 +3712,54 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Compare every WhereLoop X on the list p against pTemplate. If: +** Return TRUE if the set of WHERE clause terms used by pA is a proper +** subset of the WHERE clause terms used by pB. +*/ +static int whereLoopProperSubset(const WhereLoop *pA, const WhereLoop *pB){ + int i, j; + if( pA->nLTerm>=pB->nLTerm ) return 0; + for(j=0, i=pA->nLTerm-1; i>=0 && j>=0; i--){ + for(j=pB->nLTerm-1; j>=0; j--){ + if( pB->aLTerm[j]==pA->aLTerm[i] ) break; + } + } + return j>=0; +} + +/* +** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so +** that: ** -** (1) both X and pTemplate refer to the same table, and -** (2) both X and pTemplate use a single index, and -** (3) pTemplate uses all the same WHERE clause terms as X plus -** at least one more term, +** (1) pTemplate costs less than any other WhereLoops that are a proper +** subset of pTemplate ** -** then make sure the cost pTemplate is less than the cost of X, adjusting -** the cost of pTemplate downward if necessary. +** (2) pTemplate costs more than any other WhereLoops for which pTemplate +** is a proper subset. ** -** Example: When computing the query plan for the SELECT below: -** -** CREATE TABLE t1(a,b,c,d); -** CREATE INDEX t1abc ON t1(a,b,c); -** CREATE INDEX t1bc ON t1(b,c); -** SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; -** -** Make sure the cost of using three three columns of index t1abc is less -** than the cost of using both columns of t1bc. +** To say "WhereLoop X is a proper subset of Y" means that X uses fewer +** WHERE clause terms than Y and that every WHERE clause term used by X is +** also used by Y. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( pTemplate->nLTerm==0 ) return; for(; p; p=p->pNextLoop){ - if( p->iTab==pTemplate->iTab - && (p->wsFlags & WHERE_INDEXED)!=0 - && p->nLTermnLTerm + if( p->iTab!=pTemplate->iTab ) continue; + if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( p->nLTermnLTerm && (p->rRunrRun || (p->rRun==pTemplate->rRun && p->nOut<=pTemplate->nOut)) + && whereLoopProperSubset(p, pTemplate) ){ - int i, j; - for(j=0, i=p->nLTerm-1; i>=0 && j>=0; i--){ - for(j=pTemplate->nLTerm-1; j>=0; j--){ - if( pTemplate->aLTerm[j]==p->aLTerm[i] ) break; - } - } - if( j>=0 ){ - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut - 1; - } + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut - 1; + }else + if( p->nLTerm>pTemplate->nLTerm + && (p->rRun>pTemplate->rRun || (p->rRun==pTemplate->rRun && + p->nOut>=pTemplate->nOut)) + && whereLoopProperSubset(pTemplate, p) + ){ + pTemplate->rRun = p->rRun; + pTemplate->nOut = p->nOut + 1; } } } diff --git a/test/whereH.test b/test/whereH.test new file mode 100644 index 0000000000..c252fe1ef7 --- /dev/null +++ b/test/whereH.test @@ -0,0 +1,139 @@ +# 2014-03-31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for query planning decisions where one candidate index +# covers a proper superset of the WHERE clause terms of another +# candidate index. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test whereH-1.1 { + CREATE TABLE t1(a,b,c,d); + CREATE INDEX t1abc ON t1(a,b,c); + CREATE INDEX t1bc ON t1(b,c); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {/INDEX t1abc /} +do_execsql_test whereH-1.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-2.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d); + CREATE INDEX t1bc ON t1(b,c); + CREATE INDEX t1abc ON t1(a,b,c); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {/INDEX t1abc /} +do_execsql_test whereH-2.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c>=? ORDER BY c; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-3.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-3.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-4.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-4.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-5.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-5.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-6.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1cd ON t1(c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-6.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-7.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + CREATE INDEX t1cd ON t1(c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-7.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + +do_execsql_test whereH-8.1 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c,d,e); + CREATE INDEX t1abcd ON t1(a,b,c,d); + CREATE INDEX t1cd ON t1(c,d); + CREATE INDEX t1bcd ON t1(b,c,d); + + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {/INDEX t1abcd /} +do_execsql_test whereH-8.2 { + EXPLAIN QUERY PLAN + SELECT d FROM t1 WHERE a=? AND b=? AND c=? AND d>=? ORDER BY d; +} {~/TEMP B-TREE FOR ORDER BY/} + + + +finish_test From d3c156638a227b72ecee81b8d87e539ca044102d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 20:05:20 +0000 Subject: [PATCH 130/177] Remove an unnecessary conditional. FossilOrigin-Name: 7473c4dfc10a47594affa6f4e071a08dc8838c0c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7824fe6caa..35a72e84b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Also\smake\ssure\san\sindex\sthat\sis\sa\sproper\ssubset\sof\ssome\sother\sindex\shas\sa\nhigher\scost\sthan\sthat\sother\sindex.\s\sAdd\stest\scases. -D 2014-03-31T19:49:00.374 +C Remove\san\sunnecessary\sconditional. +D 2014-03-31T20:05:20.135 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 182f16d91060418dfcc7401d24e43d8ec24e026c +F src/where.c 7b5010f5c4d6cf81a8d6fcc9e09b05faad71b395 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ea8b0910040198751551b0b960e6b783913607df -R 176e1ff66c14893be5de17b3890f1972 +P b7830d232b073a197aa1092e78cb24e88cb10fd3 +R ddf1c9d2d849c5f13c78d81687d216d7 U drh -Z 3bf4657410567331f2f5534ba4eae60e +Z 7ba197d4f98bb2ab782217a2f7675fd9 diff --git a/manifest.uuid b/manifest.uuid index a0ffd4b3db..2b145830ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7830d232b073a197aa1092e78cb24e88cb10fd3 \ No newline at end of file +7473c4dfc10a47594affa6f4e071a08dc8838c0c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9882b4873f..299acbaf0f 100644 --- a/src/where.c +++ b/src/where.c @@ -3717,7 +3717,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ */ static int whereLoopProperSubset(const WhereLoop *pA, const WhereLoop *pB){ int i, j; - if( pA->nLTerm>=pB->nLTerm ) return 0; + assert( pA->nLTermnLTerm ); /* Checked by calling function */ for(j=0, i=pA->nLTerm-1; i>=0 && j>=0; i--){ for(j=pB->nLTerm-1; j>=0; j--){ if( pB->aLTerm[j]==pA->aLTerm[i] ) break; From 4c417181c218c1e469f86bb754045b4134189ee7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 23:57:41 +0000 Subject: [PATCH 131/177] Fix a compiler warning when SQLITE_DIRECT_OVERFLOW_READ is defined. Minor performance enhancement and size reduction. FossilOrigin-Name: 96385dc460545807a5c8fcf6280a971700f84866 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 9 +++++++-- src/btreeInt.h | 8 ++++---- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 1266012e4a..a1eded678c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sin\sthe\slatest\schanges\sand\sfixes\sfrom\strunk. -D 2014-03-31T22:03:32.269 +C Fix\sa\scompiler\swarning\swhen\sSQLITE_DIRECT_OVERFLOW_READ\sis\sdefined.\nMinor\sperformance\senhancement\sand\ssize\sreduction. +D 2014-03-31T23:57:41.627 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,9 +164,9 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 33338d61b912b5219fc5b466db8e0d7385c0b18b +F src/btree.c b1d6309d6623f4ea8d9466aebd1ec84adc40cb9b F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 -F src/btreeInt.h d1784d1e17d08d29e890190dbb9836fa64573381 +F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d8e1f75ddf10f3c0b21acd5455f90fdcea54a948 b3296267fb67b9f59719a37093253062edde3746 -R 6014daa58aba2f75bf5f17a4d16cbf23 +P f6211540c9d66a08dc580dd733e4f4a98968ae30 +R a9646e8cad6830bc030ac01ef7720050 U drh -Z 6a8ee270f5925545ed8c5f82edf09850 +Z be4271c513a3efe06acb17f300ec9933 diff --git a/manifest.uuid b/manifest.uuid index 36c381c508..e03a98f749 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6211540c9d66a08dc580dd733e4f4a98968ae30 \ No newline at end of file +96385dc460545807a5c8fcf6280a971700f84866 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9b50ebae4f..cbb968ad8b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3633,7 +3633,8 @@ static int btreeCursor( pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; + assert( wrFlag==0 || wrFlag==BTCF_WriteFlag ); + pCur->curFlags = wrFlag; pCur->pNext = pBt->pCursor; if( pCur->pNext ){ pCur->pNext->pPrev = pCur; @@ -3965,7 +3966,9 @@ static int accessPayload( int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ - int bEnd; /* True if reading to end of data */ +#ifdef SQLITE_DIRECT_OVERFLOW_READ + int bEnd; /* True if reading to end of data */ +#endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); @@ -3975,7 +3978,9 @@ static int accessPayload( getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); +#ifdef SQLITE_DIRECT_OVERFLOW_READ bEnd = (offset+amt==nKey+pCur->info.nData); +#endif if( NEVER(offset+amt > nKey+pCur->info.nData) || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] diff --git a/src/btreeInt.h b/src/btreeInt.h index dff07f145e..d1cdd46983 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -514,10 +514,10 @@ struct BtCursor { /* ** Legal values for BtCursor.curFlags */ -#define BTCF_ValidNKey 0x01 /* True if info.nKey is valid */ -#define BTCF_ValidOvfl 0x02 /* True if aOverflow is valid */ -#define BTCF_AtLast 0x04 /* Cursor is pointing ot the last entry */ -#define BTCF_WriteFlag 0x08 /* True if a write cursor */ +#define BTCF_WriteFlag 0x01 /* True if a write cursor */ +#define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ +#define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ +#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ /* From a38c9516e847f90d914dce229b7c601698e94102 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 1 Apr 2014 01:24:34 +0000 Subject: [PATCH 132/177] Remove an unnecessary conditional, replacing it with an assert(). Improvements to comments. FossilOrigin-Name: 834bf1c367e1ccd498c7f9f843be2d2aa11ffb3c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 24 ++++++++++++++++-------- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index a1eded678c..dbe10602d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scompiler\swarning\swhen\sSQLITE_DIRECT_OVERFLOW_READ\sis\sdefined.\nMinor\sperformance\senhancement\sand\ssize\sreduction. -D 2014-03-31T23:57:41.627 +C Remove\san\sunnecessary\sconditional,\sreplacing\sit\swith\san\sassert().\nImprovements\sto\scomments. +D 2014-04-01T01:24:34.512 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b1d6309d6623f4ea8d9466aebd1ec84adc40cb9b +F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f6211540c9d66a08dc580dd733e4f4a98968ae30 -R a9646e8cad6830bc030ac01ef7720050 +P 96385dc460545807a5c8fcf6280a971700f84866 +R 193f537069a669cf49d79f97b46daa7e U drh -Z be4271c513a3efe06acb17f300ec9933 +Z 9306db9f51cd1c998fccb67bf7fedc94 diff --git a/manifest.uuid b/manifest.uuid index e03a98f749..44d48365ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96385dc460545807a5c8fcf6280a971700f84866 \ No newline at end of file +834bf1c367e1ccd498c7f9f843be2d2aa11ffb3c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cbb968ad8b..2bf026b840 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3974,6 +3974,7 @@ static int accessPayload( assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pCell + pCur->info.nHeader; @@ -4009,12 +4010,13 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); - /* If the BTCF_Incrblob flag is set and the BtCursor.aOverflow[] - ** has not been allocated, allocate it now. The array is sized at - ** one entry for each overflow page in the overflow chain. The - ** page number of the first overflow page is stored in aOverflow[0], - ** etc. A value of 0 in the aOverflow[] array means "not yet known" - ** (the cache is lazily populated). + /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. + ** Except, do not allocate aOverflow[] for eOp==2. + ** + ** The aOverflow[] array is sized at one entry for each overflow page + ** in the overflow chain. The page number of the first overflow page is + ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array + ** means "not yet known" (the cache is lazily populated). */ if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; @@ -4059,11 +4061,17 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. + ** + ** Note that the aOverflow[] array must be allocated because eOp!=2 + ** here. If eOp==2, then offset==0 and this branch is never taken. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[iIdx+1] ){ + assert( eOp!=2 ); + assert( pCur->curFlags & BTCF_ValidOvfl ); + if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; - } else + }else{ rc = getOverflowPage(pBt, nextPage, 0, &nextPage); + } offset -= ovflSize; }else{ /* Need to read this page properly. It contains some of the From 54d0d2dd9a5898ac33363877b0aba360a81d1d5e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Apr 2014 00:32:13 +0000 Subject: [PATCH 133/177] In the command-line shell, run set writable_schema before running the ".clone" command. FossilOrigin-Name: 9d2ae6342c8afa904bec591ebe134ff7f536b71c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 887bb92f5b..8f224e8e55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\s(harmless)\sbuffer\soverread\sthat\sis\spossible\son\san\sOOM\swhen\nMEMSYS5\sis\sengaged. -D 2014-03-31T13:42:42.977 +C In\sthe\scommand-line\sshell,\srun\sset\swritable_schema\sbefore\srunning\sthe\n".clone"\scommand. +D 2014-04-03T00:32:13.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 269c3e31a450fce642a10569221a49180348c88e -F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf +F src/shell.c 5260f2ada8dd06e9f5ae0a448c8c01e7a75dd881 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 27deb6e49bcc76714dbdc61b34748603155ac770 -R 74e3f11f049d7792447568e00e413b7f +P b3296267fb67b9f59719a37093253062edde3746 +R 3857796e3bb74e57525267a0574c950a U drh -Z 4989508149fff1b415b46911a02fce9e +Z e5eef1fbe323934e85f2b63b760c4a7b diff --git a/manifest.uuid b/manifest.uuid index 70f95ac337..e11bac748d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3296267fb67b9f59719a37093253062edde3746 \ No newline at end of file +9d2ae6342c8afa904bec591ebe134ff7f536b71c \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1313112709..f380962a86 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2130,10 +2130,12 @@ static void tryToClone(struct callback_data *p, const char *zNewDb){ fprintf(stderr, "Cannot create output database: %s\n", sqlite3_errmsg(newDb)); }else{ + sqlite3_exec(p->db, "PRAGMA writable_schema=ON;", 0, 0, 0); sqlite3_exec(newDb, "BEGIN EXCLUSIVE;", 0, 0, 0); tryToCloneSchema(p, newDb, "type='table'", tryToCloneData); tryToCloneSchema(p, newDb, "type!='table'", 0); sqlite3_exec(newDb, "COMMIT;", 0, 0, 0); + sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); } sqlite3_close(newDb); } From 831116d1646b01498ca8d7d2b2280eaaea61052b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Apr 2014 14:31:00 +0000 Subject: [PATCH 134/177] Fix a typo in the "Synopsis:" comment for the OP_VFilter opcode. FossilOrigin-Name: 48ecdd4aff03741f96c070dced69c3c273b652cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8f224e8e55..a63a365841 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\srun\sset\swritable_schema\sbefore\srunning\sthe\n".clone"\scommand. -D 2014-04-03T00:32:13.777 +C Fix\sa\stypo\sin\sthe\s"Synopsis:"\scomment\sfor\sthe\sOP_VFilter\sopcode. +D 2014-04-03T14:31:00.074 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 +F src/vdbe.c e811a0081149fb90db367026d154cd7efb3c7098 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b3296267fb67b9f59719a37093253062edde3746 -R 3857796e3bb74e57525267a0574c950a +P 9d2ae6342c8afa904bec591ebe134ff7f536b71c +R 06523ad1e0d97e48aa9a162217a28d3b U drh -Z e5eef1fbe323934e85f2b63b760c4a7b +Z 64c7c8526eed6d4962d61576f8ab00f6 diff --git a/manifest.uuid b/manifest.uuid index e11bac748d..5984b7794c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d2ae6342c8afa904bec591ebe134ff7f536b71c \ No newline at end of file +48ecdd4aff03741f96c070dced69c3c273b652cb \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 84f720b526..2d1d23e079 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5929,7 +5929,7 @@ case OP_VOpen: { #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VFilter P1 P2 P3 P4 * -** Synopsis: iPlan=r[P3] zPlan='P4' +** Synopsis: iplan=r[P3] zplan='P4' ** ** P1 is a cursor opened using VOpen. P2 is an address to jump to if ** the filtered result set is empty. From c438df1be06f636ef68265f083aff4af95d326e6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Apr 2014 16:29:31 +0000 Subject: [PATCH 135/177] Use OP_Copy instead of OP_SCopy when moving results out of a subquery, to prevent the subquery results from changing out from under the outer query. Fix for ticket [1e64dd782a126f48d78]. FossilOrigin-Name: d5513dfa23baa0b0a095aaf17d19aacd30dcef61 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/whereG.test | 13 +++++++++++++ 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a63a365841..4bcee0a832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\s"Synopsis:"\scomment\sfor\sthe\sOP_VFilter\sopcode. -D 2014-04-03T14:31:00.074 +C Use\sOP_Copy\sinstead\sof\sOP_SCopy\swhen\smoving\sresults\sout\sof\sa\ssubquery,\nto\sprevent\sthe\ssubquery\sresults\sfrom\schanging\sout\sfrom\sunder\sthe\souter\nquery.\s\sFix\sfor\sticket\s[1e64dd782a126f48d78]. +D 2014-04-03T16:29:31.330 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb +F src/where.c ebad891b7494d0c5f925cf7ab135380bd958cba3 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1092,7 +1092,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test eb3a46b3eaf38e25e3013433b2db8a25a866c215 +F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1159,7 +1159,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9d2ae6342c8afa904bec591ebe134ff7f536b71c -R 06523ad1e0d97e48aa9a162217a28d3b +P 48ecdd4aff03741f96c070dced69c3c273b652cb +Q +ec6a06246e04eee5f25f1c28507df73b697099c0 +R b6a2fefa5b61ea0e64da4db2aaa10286 U drh -Z 64c7c8526eed6d4962d61576f8ab00f6 +Z f1f1ceb5f31f4e26d736a13a2a5aca0f diff --git a/manifest.uuid b/manifest.uuid index 5984b7794c..601b0b4f2e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48ecdd4aff03741f96c070dced69c3c273b652cb \ No newline at end of file +d5513dfa23baa0b0a095aaf17d19aacd30dcef61 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 93ee8c59c3..dd6893f69f 100644 --- a/src/where.c +++ b/src/where.c @@ -5924,7 +5924,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){ for(; kp1!=pLevel->iTabCur ) continue; if( pOp->opcode==OP_Column ){ - pOp->opcode = OP_SCopy; + pOp->opcode = OP_Copy; pOp->p1 = pOp->p2 + pTabItem->regResult; pOp->p2 = pOp->p3; pOp->p3 = 0; diff --git a/test/whereG.test b/test/whereG.test index 490fffe64e..17d5653223 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -166,5 +166,18 @@ do_eqp_test whereG-3.4 { SELECT * FROM a, b WHERE a2=5 AND a1=b1; } {/.*SCAN TABLE a.*SEARCH TABLE b USING INDEX .*b_1 .b1=..*/} +# Ticket [1e64dd782a126f48d78c43a664844a41d0e6334e]: +# Incorrect result in a nested GROUP BY/DISTINCT due to the use of an OP_SCopy +# where an OP_Copy was needed. +# +do_execsql_test whereG-4.0 { + CREATE TABLE t4(x); + INSERT INTO t4 VALUES('right'),('wrong'); + SELECT DISTINCT x + FROM (SELECT x FROM t4 GROUP BY x) + WHERE x='right' + ORDER BY x; +} {right} + finish_test From 1fb6a11072900e7f347980ee07612d2e37ae2c54 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 14:12:52 +0000 Subject: [PATCH 136/177] Ensure the "PRAGMA journal_mode=WAL" works coming from any other journal_mode with ATTACH-ed databases. FossilOrigin-Name: e54330b43127e46fc6494748cbb353a6fc91cfd7 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/pager.c | 3 +-- test/wal.test | 13 +++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4bcee0a832..26b58d03cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sOP_Copy\sinstead\sof\sOP_SCopy\swhen\smoving\sresults\sout\sof\sa\ssubquery,\nto\sprevent\sthe\ssubquery\sresults\sfrom\schanging\sout\sfrom\sunder\sthe\souter\nquery.\s\sFix\sfor\sticket\s[1e64dd782a126f48d78]. -D 2014-04-03T16:29:31.330 +C Ensure\sthe\s"PRAGMA\sjournal_mode=WAL"\sworks\scoming\sfrom\sany\sother\sjournal_mode\nwith\sATTACH-ed\sdatabases. +D 2014-04-04T14:12:52.073 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e -F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e +F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 @@ -1050,7 +1050,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 -F test/wal.test 40073e54359d43354925b2b700b7eebd5e207285 +F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1159,8 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 48ecdd4aff03741f96c070dced69c3c273b652cb -Q +ec6a06246e04eee5f25f1c28507df73b697099c0 -R b6a2fefa5b61ea0e64da4db2aaa10286 +P d5513dfa23baa0b0a095aaf17d19aacd30dcef61 +R dda37fe73fefd9a3a9bbbb33bf2f4700 U drh -Z f1f1ceb5f31f4e26d736a13a2a5aca0f +Z ec7ec754ade71c9dc3d6dc5092f63d72 diff --git a/manifest.uuid b/manifest.uuid index 601b0b4f2e..f9214073e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5513dfa23baa0b0a095aaf17d19aacd30dcef61 \ No newline at end of file +e54330b43127e46fc6494748cbb353a6fc91cfd7 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index c6485a4d45..b09d6cb930 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1624,12 +1624,11 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ if( !zMaster || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_OFF + || !isOpen(pPager->jfd) ){ return SQLITE_OK; } pPager->setMaster = 1; - assert( isOpen(pPager->jfd) ); assert( pPager->journalHdr <= pPager->journalOff ); /* Calculate the length in bytes and the checksum of zMaster */ diff --git a/test/wal.test b/test/wal.test index 3a69de54f5..675be73791 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1574,4 +1574,17 @@ sqlite3_shutdown test_sqlite3_log sqlite3_initialize +# Make sure PRAGMA journal_mode=WAL works with ATTACHED databases in +# all journal modes. +# +foreach mode {OFF MEMORY PERSIST DELETE TRUNCATE WAL} { + delete_file test.db test2.db + sqlite3 db test.db + do_test wal-25.$mode { + db eval "PRAGMA journal_mode=$mode" + db eval {ATTACH 'test2.db' AS t2; PRAGMA journal_mode=WAL;} + } {wal} + db close +} + finish_test From d83cad235a5360fe670057d5a70da65b43aea793 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 10 Apr 2014 02:24:48 +0000 Subject: [PATCH 137/177] Modify the OR-clause handling so that it can safely deal with OR-clause with 17 or more terms. Fix for ticket [10fb063b1179be53ea0b53bb]. FossilOrigin-Name: a67b5312f63909737c052fe58ab5772d45530d49 --- manifest | 19 ++++++++--------- manifest.uuid | 2 +- src/rowset.c | 6 +++--- src/sqliteInt.h | 2 +- src/vdbe.c | 4 +--- test/whereD.test | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 51582695c0..0b43c2f6e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovements\swhen\sreading\slarge\sblobs,\sespecially\sif\nSQLITE_DIRECT_OVERFLOW_READ\sis\sdefined. -D 2014-04-04T18:49:19.417 +C Modify\sthe\sOR-clause\shandling\sso\sthat\sit\scan\ssafely\sdeal\swith\sOR-clause\s\nwith\s17\sor\smore\sterms.\s\sFix\sfor\sticket\s[10fb063b1179be53ea0b53bb]. +D 2014-04-10T02:24:48.549 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,13 +216,13 @@ F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 -F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 +F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 269c3e31a450fce642a10569221a49180348c88e F src/shell.c 5260f2ada8dd06e9f5ae0a448c8c01e7a75dd881 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h cc9582a91b2910404ccda7b7e198815ea0f75948 +F src/sqliteInt.h 9b1361478e821b1a68f7993047a922e3155ca87d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c e811a0081149fb90db367026d154cd7efb3c7098 +F src/vdbe.c bf284edea1cee0508dc93c79a73498e0f317edb7 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1089,7 +1089,7 @@ F test/where9.test 4f3eab951353a3ae164befc521c777dfa903e46c F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a -F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 +F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 @@ -1160,8 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 683dd379a293b2f330e1e4cd746f190527fe48ee 834bf1c367e1ccd498c7f9f843be2d2aa11ffb3c -R 3de83004edd963c4959934c7f5a61b03 -T +closed 834bf1c367e1ccd498c7f9f843be2d2aa11ffb3c +P 2312eb6a9eb31051db4e0baf19e033ba39adc7b1 +R 2f4de3ec11a98155d746ed7623586007 U drh -Z f4df6ef034f1a63bb656a3ddc69e0011 +Z d44fb7b759451f899c9b9bdcaf2f6d96 diff --git a/manifest.uuid b/manifest.uuid index 0466f25ed2..41b50df795 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2312eb6a9eb31051db4e0baf19e033ba39adc7b1 \ No newline at end of file +a67b5312f63909737c052fe58ab5772d45530d49 \ No newline at end of file diff --git a/src/rowset.c b/src/rowset.c index 5761f98550..ba2e056bd4 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -112,8 +112,8 @@ struct RowSet { struct RowSetEntry *pFresh; /* Source of new entry objects */ struct RowSetEntry *pForest; /* List of binary trees of entries */ u16 nFresh; /* Number of objects on pFresh */ - u8 rsFlags; /* Various flags */ - u8 iBatch; /* Current insert batch */ + u16 rsFlags; /* Various flags */ + int iBatch; /* Current insert batch */ }; /* @@ -447,7 +447,7 @@ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ ** on pRowSet->pEntry, then sort those entires into the forest at ** pRowSet->pForest so that they can be tested. */ -int sqlite3RowSetTest(RowSet *pRowSet, u8 iBatch, sqlite3_int64 iRowid){ +int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ struct RowSetEntry *p, *pTree; /* This routine is never called after sqlite3RowSetNext() */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0f67748fc8..57481b1084 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3019,7 +3019,7 @@ int sqlite3BitvecBuiltinTest(int,int*); RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); void sqlite3RowSetClear(RowSet*); void sqlite3RowSetInsert(RowSet*, i64); -int sqlite3RowSetTest(RowSet*, u8 iBatch, i64); +int sqlite3RowSetTest(RowSet*, int iBatch, i64); int sqlite3RowSetNext(RowSet*, i64*); void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); diff --git a/src/vdbe.c b/src/vdbe.c index 2d1d23e079..6102cf3c7e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5227,9 +5227,7 @@ case OP_RowSetTest: { /* jump, in1, in3 */ assert( pOp->p4type==P4_INT32 ); assert( iSet==-1 || iSet>=0 ); if( iSet ){ - exists = sqlite3RowSetTest(pIn1->u.pRowSet, - (u8)(iSet>=0 ? iSet & 0xf : 0xff), - pIn3->u.i); + exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); if( exists ){ pc = pOp->p2 - 1; diff --git a/test/whereD.test b/test/whereD.test index eb0f864025..db993040b0 100644 --- a/test/whereD.test +++ b/test/whereD.test @@ -217,6 +217,59 @@ do_execsql_test 4.7 { ORDER BY a; } {3 4 3 4} +# Verify fix of a bug reported on the mailing list by Peter Reid +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t; + CREATE TABLE t(c0,c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17); + CREATE INDEX tc0 ON t(c0); + CREATE INDEX tc1 ON t(c1); + CREATE INDEX tc2 ON t(c2); + CREATE INDEX tc3 ON t(c3); + CREATE INDEX tc4 ON t(c4); + CREATE INDEX tc5 ON t(c5); + CREATE INDEX tc6 ON t(c6); + CREATE INDEX tc7 ON t(c7); + CREATE INDEX tc8 ON t(c8); + CREATE INDEX tc9 ON t(c9); + CREATE INDEX tc10 ON t(c10); + CREATE INDEX tc11 ON t(c11); + CREATE INDEX tc12 ON t(c12); + CREATE INDEX tc13 ON t(c13); + CREATE INDEX tc14 ON t(c14); + CREATE INDEX tc15 ON t(c15); + CREATE INDEX tc16 ON t(c16); + CREATE INDEX tc17 ON t(c17); + + INSERT INTO t(c0, c16) VALUES (1,1); + + SELECT * FROM t WHERE + c0=1 or c1=1 or c2=1 or c3=1 or + c4=1 or c5=1 or c6=1 or c7=1 or + c8=1 or c9=1 or c10=1 or c11=1 or + c12=1 or c13=1 or c14=1 or c15=1 or + c16=1 or c17=1; +} {1 {} {} {} {} {} {} {} {} {} {} {} {} {} {} {} 1 {}} +do_execsql_test 5.2 { + DELETE FROM t; + INSERT INTO t(c0,c17) VALUES(1,1); + SELECT * FROM t WHERE + c0=1 or c1=1 or c2=1 or c3=1 or + c4=1 or c5=1 or c6=1 or c7=1 or + c8=1 or c9=1 or c10=1 or c11=1 or + c12=1 or c13=1 or c14=1 or c15=1 or + c16=1 or c17=1; +} {1 {} {} {} {} {} {} {} {} {} {} {} {} {} {} {} {} 1} +do_execsql_test 5.3 { + DELETE FROM t; + INSERT INTO t(c0,c15) VALUES(1,1); + SELECT * FROM t WHERE + c0=1 or c1=1 or c2=1 or c3=1 or + c4=1 or c5=1 or c6=1 or c7=1 or + c8=1 or c9=1 or c10=1 or c11=1 or + c12=1 or c13=1 or c14=1 or c15=1 or + c16=1 or c17=1; +} {1 {} {} {} {} {} {} {} {} {} {} {} {} {} {} 1 {} {}} finish_test From 87744513e7374cae5f2df94d4ac1b014483683bc Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 13 Apr 2014 19:15:49 +0000 Subject: [PATCH 138/177] Make sure column cache elements are cleared correctly when jumping over code for key generation in a partial index. Fix for ticket [2ea3e9fe6379fc3f6]. FossilOrigin-Name: 3122b8364082be783821da01d4af2af6a9586327 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 2 +- src/delete.c | 18 ++++++++++++++++-- src/pragma.c | 2 +- src/sqliteInt.h | 1 + test/index6.test | 19 +++++++++++++++++++ 7 files changed, 49 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 0b43c2f6e5..80d510a4f9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sOR-clause\shandling\sso\sthat\sit\scan\ssafely\sdeal\swith\sOR-clause\s\nwith\s17\sor\smore\sterms.\s\sFix\sfor\sticket\s[10fb063b1179be53ea0b53bb]. -D 2014-04-10T02:24:48.549 +C Make\ssure\scolumn\scache\selements\sare\scleared\scorrectly\swhen\sjumping\sover\ncode\sfor\skey\sgeneration\sin\sa\spartial\sindex.\s\nFix\sfor\sticket\s[2ea3e9fe6379fc3f6]. +D 2014-04-13T19:15:49.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,12 +167,12 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 +F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 +F src/delete.c d5e3a958519677049a752ed37657240b55ce5806 F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf @@ -211,7 +211,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 10f169b9650f0930a7a6df67e1387a4c2c449f38 +F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -222,7 +222,7 @@ F src/shell.c 5260f2ada8dd06e9f5ae0a448c8c01e7a75dd881 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 9b1361478e821b1a68f7993047a922e3155ca87d +F src/sqliteInt.h 97df117e13dea9d2fe8a8e32376222e03fe4a4c9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -610,7 +610,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test 936979c3a1e87b81feaed2d00505665bf142d764 +F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2312eb6a9eb31051db4e0baf19e033ba39adc7b1 -R 2f4de3ec11a98155d746ed7623586007 +P a67b5312f63909737c052fe58ab5772d45530d49 +R 47ced825f4356f6492f482f71c0fa4e8 U drh -Z d44fb7b759451f899c9b9bdcaf2f6d96 +Z d6e06c047caf6d5aab62e0b2ec59c712 diff --git a/manifest.uuid b/manifest.uuid index 41b50df795..6f61a4975d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a67b5312f63909737c052fe58ab5772d45530d49 \ No newline at end of file +3122b8364082be783821da01d4af2af6a9586327 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 10077e5018..4d4155ba90 100644 --- a/src/build.c +++ b/src/build.c @@ -2680,7 +2680,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); - sqlite3VdbeResolveLabel(v, iPartIdxLabel); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); diff --git a/src/delete.c b/src/delete.c index 79e83cae52..c6e1762215 100644 --- a/src/delete.c +++ b/src/delete.c @@ -739,7 +739,7 @@ void sqlite3GenerateRowIndexDelete( &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); - sqlite3VdbeResolveLabel(v, iPartIdxLabel); + sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } } @@ -758,10 +758,11 @@ void sqlite3GenerateRowIndexDelete( ** ** If *piPartIdxLabel is not NULL, fill it in with a label and jump ** to that label if pIdx is a partial index that should be skipped. +** The label should be resolved using sqlite3ResolvePartIdxLabel(). ** A partial index should be skipped if its WHERE clause evaluates ** to false or null. If pIdx is not a partial index, *piPartIdxLabel ** will be set to zero which is an empty label that is ignored by -** sqlite3VdbeResolveLabel(). +** sqlite3ResolvePartIdxLabel(). ** ** The pPrior and regPrior parameters are used to implement a cache to ** avoid unnecessary register loads. If pPrior is not NULL, then it is @@ -794,6 +795,7 @@ int sqlite3GenerateIndexKey( if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); pParse->iPartIdxTab = iDataCur; + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); }else{ @@ -821,3 +823,15 @@ int sqlite3GenerateIndexKey( sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; } + +/* +** If a prior call to sqlite3GenerateIndexKey() generated a jump-over label +** because it was a partial index, then this routine should be called to +** resolve that label. +*/ +void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ + if( iLabel ){ + sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); + sqlite3ExprCachePop(pParse, 1); + } +} diff --git a/src/pragma.c b/src/pragma.c index 20da9a6892..66d0c3328e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1928,7 +1928,7 @@ void sqlite3Pragma( sqlite3VdbeAddOp0(v, OP_Halt); sqlite3VdbeJumpHere(v, jmp4); sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeResolveLabel(v, jmp3); + sqlite3ResolvePartIdxLabel(pParse, jmp3); } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 57481b1084..7d3302eeb7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3135,6 +3135,7 @@ int sqlite3IsRowid(const char*); void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); +void sqlite3ResolvePartIdxLabel(Parse*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*); void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); diff --git a/test/index6.test b/test/index6.test index 48706be3cc..3451e5c1df 100644 --- a/test/index6.test +++ b/test/index6.test @@ -248,4 +248,23 @@ do_execsql_test index6-5.0 { SELECT stat+0 FROM sqlite_stat1 WHERE idx='t3b'; } {6 6} +# Test case for ticket [2ea3e9fe6379fc3f6ce7e090ce483c1a3a80d6c9] from +# 2014-04-13: Partial index causes assertion fault on UPDATE OR REPLACE. +# +do_execsql_test index6-6.0 { + CREATE TABLE t6(a,b); + CREATE UNIQUE INDEX t6ab ON t1(a,b); + CREATE INDEX t6b ON t6(b) WHERE b=1; + INSERT INTO t6(a,b) VALUES(123,456); + SELECT * FROM t6; +} {123 456} +do_execsql_test index6-6.1 { + UPDATE OR REPLACE t6 SET b=789; + SELECT * FROM t6; +} {123 789} +do_execsql_test index6-6.2 { + PRAGMA integrity_check; +} {ok} + + finish_test From d2490904507b0f9abeb077a40c2d91ad3a222f7f Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 13 Apr 2014 19:28:15 +0000 Subject: [PATCH 139/177] Remove the unused second argument from sqlite3ExprCachePop(). Add an ALWAYS() on an always-true conditional in sqlite3VdbeResolveLabel(). FossilOrigin-Name: ab23abf392175d3e256ff619b13abbbe732a49ed --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/delete.c | 2 +- src/expr.c | 31 +++++++++++++++---------------- src/sqliteInt.h | 2 +- src/vdbeaux.c | 2 +- src/where.c | 2 +- 7 files changed, 30 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 80d510a4f9..3301b08ca6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\scolumn\scache\selements\sare\scleared\scorrectly\swhen\sjumping\sover\ncode\sfor\skey\sgeneration\sin\sa\spartial\sindex.\s\nFix\sfor\sticket\s[2ea3e9fe6379fc3f6]. -D 2014-04-13T19:15:49.713 +C Remove\sthe\sunused\ssecond\sargument\sfrom\ssqlite3ExprCachePop().\s\sAdd\san\nALWAYS()\son\san\salways-true\sconditional\sin\ssqlite3VdbeResolveLabel(). +D 2014-04-13T19:28:15.499 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,8 +172,8 @@ F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c d5e3a958519677049a752ed37657240b55ce5806 -F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 +F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf +F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -222,7 +222,7 @@ F src/shell.c 5260f2ada8dd06e9f5ae0a448c8c01e7a75dd881 F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 97df117e13dea9d2fe8a8e32376222e03fe4a4c9 +F src/sqliteInt.h dadd17dbd05fd17cad157a7d96339bf4a0b41172 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -282,7 +282,7 @@ F src/vdbe.c bf284edea1cee0508dc93c79a73498e0f317edb7 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d +F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 70eb10da08cebbd6a576ced18edb7c8f79d41597 +F src/where.c 7614c4383d8b6143558dc349da286d0325704d35 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a67b5312f63909737c052fe58ab5772d45530d49 -R 47ced825f4356f6492f482f71c0fa4e8 +P 3122b8364082be783821da01d4af2af6a9586327 +R e1df7805a07beb887bab893b56a4a7f9 U drh -Z d6e06c047caf6d5aab62e0b2ec59c712 +Z 77065e6cb595843b2b0b90941a9d16f6 diff --git a/manifest.uuid b/manifest.uuid index 6f61a4975d..39f1fc5832 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3122b8364082be783821da01d4af2af6a9586327 \ No newline at end of file +ab23abf392175d3e256ff619b13abbbe732a49ed \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index c6e1762215..c74d8eab7d 100644 --- a/src/delete.c +++ b/src/delete.c @@ -832,6 +832,6 @@ int sqlite3GenerateIndexKey( void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ if( iLabel ){ sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); } } diff --git a/src/expr.c b/src/expr.c index b070621ebd..08c121e386 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1883,7 +1883,7 @@ int sqlite3CodeSubselect( if( testAddr>=0 ){ sqlite3VdbeJumpHere(v, testAddr); } - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); return rReg; } @@ -2018,7 +2018,7 @@ static void sqlite3ExprCodeIN( } } sqlite3ReleaseTempReg(pParse, r1); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); VdbeComment((v, "end IN expr")); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -2201,15 +2201,14 @@ void sqlite3ExprCachePush(Parse *pParse){ /* ** Remove from the column cache any entries that were added since the -** the previous N Push operations. In other words, restore the cache -** to the state it was in N Pushes ago. +** the previous sqlite3ExprCachePush operation. In other words, restore +** the cache to the state it was in prior the most recent Push. */ -void sqlite3ExprCachePop(Parse *pParse, int N){ +void sqlite3ExprCachePop(Parse *pParse){ int i; struct yColCache *p; - assert( N>0 ); - assert( pParse->iCacheLevel>=N ); - pParse->iCacheLevel -= N; + assert( pParse->iCacheLevel>=1 ); + pParse->iCacheLevel--; #ifdef SQLITE_DEBUG if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ printf("POP to %d\n", pParse->iCacheLevel); @@ -2687,7 +2686,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ExprCacheRemove(pParse, target, 1); sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pFarg->a[i].pExpr, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); } sqlite3VdbeResolveLabel(v, endCoalesce); break; @@ -2741,7 +2740,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); - sqlite3ExprCachePop(pParse, 1); /* Ticket 2ea2425d34be */ + sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ r1 = 0; } @@ -2961,13 +2960,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeAddOp2(v, OP_Goto, 0, endLabel); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } if( (nExpr&1)!=0 ){ sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } @@ -3546,7 +3545,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { @@ -3554,7 +3553,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -3700,7 +3699,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_OR: { @@ -3710,7 +3709,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); break; } case TK_NOT: { diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7d3302eeb7..da46676adf 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3083,7 +3083,7 @@ void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); void sqlite3ExprCodeMove(Parse*, int, int, int); void sqlite3ExprCacheStore(Parse*, int, int, int); void sqlite3ExprCachePush(Parse*); -void sqlite3ExprCachePop(Parse*, int); +void sqlite3ExprCachePop(Parse*); void sqlite3ExprCacheRemove(Parse*, int, int); void sqlite3ExprCacheClear(Parse*); void sqlite3ExprCacheAffinityChange(Parse*, int, int); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0ce21378d5..0a6b536720 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -276,7 +276,7 @@ void sqlite3VdbeResolveLabel(Vdbe *v, int x){ int j = -1-x; assert( v->magic==VDBE_MAGIC_INIT ); assert( jnLabel ); - if( j>=0 && p->aLabel ){ + if( ALWAYS(j>=0) && p->aLabel ){ p->aLabel[j] = v->nOp; } p->iFixedOp = v->nOp - 1; diff --git a/src/where.c b/src/where.c index 16db3e7f29..e51eee5350 100644 --- a/src/where.c +++ b/src/where.c @@ -2841,7 +2841,7 @@ static Bitmask codeOneLoopStart( pLevel->p1 = iCur; pLevel->p2 = sqlite3VdbeCurrentAddr(v); sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); - sqlite3ExprCachePop(pParse, 1); + sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ From 65b9ac522430cdc55855b24de406c54d12199346 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 14 Apr 2014 19:48:25 +0000 Subject: [PATCH 140/177] Make sure the undocumented sqlite_rename_parent() function does not crash even if it is invoked with NULL arguments. Ticket [264b970c4379fd4b94] FossilOrigin-Name: c6fa8d0d82805be230f672eabccdfa5680d4ddfd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/alter.c | 1 + test/alter.test | 38 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3301b08ca6..5ba411e0a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\ssecond\sargument\sfrom\ssqlite3ExprCachePop().\s\sAdd\san\nALWAYS()\son\san\salways-true\sconditional\sin\ssqlite3VdbeResolveLabel(). -D 2014-04-13T19:28:15.499 +C Make\ssure\sthe\sundocumented\ssqlite_rename_parent()\sfunction\sdoes\snot\ncrash\seven\sif\sit\sis\sinvoked\swith\sNULL\sarguments.\nTicket\s[264b970c4379fd4b94] +D 2014-04-14T19:48:25.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c 5d99edbac5bc416032772b723ee30182ee6e5de0 +F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 @@ -298,7 +298,7 @@ F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 -F test/alter.test e88dfa77e020c2b48e52a8020c70171ab828e079 +F test/alter.test 547dc2d292644301ac9a7dda22b319b74f9c08d2 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3122b8364082be783821da01d4af2af6a9586327 -R e1df7805a07beb887bab893b56a4a7f9 +P ab23abf392175d3e256ff619b13abbbe732a49ed +R 410ec05c8f2a9918dc68eaaf465de80d U drh -Z 77065e6cb595843b2b0b90941a9d16f6 +Z 8590c3c7b25dad5b568a22b07ca7e219 diff --git a/manifest.uuid b/manifest.uuid index 39f1fc5832..1f38f99a6e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab23abf392175d3e256ff619b13abbbe732a49ed \ No newline at end of file +c6fa8d0d82805be230f672eabccdfa5680d4ddfd \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 1a83e570db..64204b7b24 100644 --- a/src/alter.c +++ b/src/alter.c @@ -116,6 +116,7 @@ static void renameParentFunc( int token; /* Type of token */ UNUSED_PARAMETER(NotUsed); + if( zInput==0 || zOld==0 ) return; for(z=zInput; *z; z=z+n){ n = sqlite3GetToken(z, &token); if( token==TK_REFERENCES ){ diff --git a/test/alter.test b/test/alter.test index 557162f837..ddf1698824 100644 --- a/test/alter.test +++ b/test/alter.test @@ -875,4 +875,42 @@ do_execsql_test alter-16.2 { SELECT * FROM t16a_rn ORDER BY a; } {abc 1.25 99 xyzzy cba 5.5 98 fizzle} +#------------------------------------------------------------------------- +# Verify that NULL values into the internal-use-only sqlite_rename_*() +# functions do not cause problems. +# +do_execsql_test alter-17.1 { + SELECT sqlite_rename_table('CREATE TABLE xyz(a,b,c)','abc'); +} {{CREATE TABLE "abc"(a,b,c)}} +do_execsql_test alter-17.2 { + SELECT sqlite_rename_table('CREATE TABLE xyz(a,b,c)',NULL); +} {{CREATE TABLE "(NULL)"(a,b,c)}} +do_execsql_test alter-17.3 { + SELECT sqlite_rename_table(NULL,'abc'); +} {{}} +do_execsql_test alter-17.4 { + SELECT sqlite_rename_trigger('CREATE TRIGGER r1 ON xyz WHEN','abc'); +} {{CREATE TRIGGER r1 ON "abc" WHEN}} +do_execsql_test alter-17.5 { + SELECT sqlite_rename_trigger('CREATE TRIGGER r1 ON xyz WHEN',NULL); +} {{CREATE TRIGGER r1 ON "(NULL)" WHEN}} +do_execsql_test alter-17.6 { + SELECT sqlite_rename_trigger(NULL,'abc'); +} {{}} +do_execsql_test alter-17.7 { + SELECT sqlite_rename_parent('CREATE TABLE t1(a REFERENCES "xyzzy")', + 'xyzzy','lmnop'); +} {{CREATE TABLE t1(a REFERENCES "lmnop")}} +do_execsql_test alter-17.8 { + SELECT sqlite_rename_parent('CREATE TABLE t1(a REFERENCES "xyzzy")', + 'xyzzy',NULL); +} {{CREATE TABLE t1(a REFERENCES "(NULL)")}} +do_execsql_test alter-17.9 { + SELECT sqlite_rename_parent('CREATE TABLE t1(a REFERENCES "xyzzy")', + NULL, 'lmnop'); +} {{}} +do_execsql_test alter-17.10 { + SELECT sqlite_rename_parent(NULL,'abc','xyz'); +} {{}} + finish_test From 2cf4acbd9f774e1c7bbaa232edbbdf93cff553ee Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 00:06:02 +0000 Subject: [PATCH 141/177] Add the SQLITE_TESTCTRL_BYTEORDER test control to interrogate SQLite's notion of the processor byte order and whether the byte order is known at compile-time or determined at run-time. FossilOrigin-Name: 9c6961967ae00e563ebe2859eaf2639a79f2cb01 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/main.c | 16 ++++++++++++++++ src/shell.c | 6 ++++-- src/sqlite.h.in | 3 ++- src/sqliteInt.h | 9 ++++++--- 6 files changed, 38 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 5ba411e0a6..a1452995b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sundocumented\ssqlite_rename_parent()\sfunction\sdoes\snot\ncrash\seven\sif\sit\sis\sinvoked\swith\sNULL\sarguments.\nTicket\s[264b970c4379fd4b94] -D 2014-04-14T19:48:25.984 +C Add\sthe\sSQLITE_TESTCTRL_BYTEORDER\stest\scontrol\sto\sinterrogate\sSQLite's\snotion\nof\sthe\sprocessor\sbyte\sorder\sand\swhether\sthe\sbyte\sorder\sis\sknown\sat\scompile-time\nor\sdetermined\sat\srun-time. +D 2014-04-18T00:06:02.110 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db +F src/main.c 0a8cfb6b2899649880e5874ac9bfd1c6f7c6a4a3 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -218,11 +218,11 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 269c3e31a450fce642a10569221a49180348c88e -F src/shell.c 5260f2ada8dd06e9f5ae0a448c8c01e7a75dd881 -F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 +F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be +F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h dadd17dbd05fd17cad157a7d96339bf4a0b41172 +F src/sqliteInt.h 35a265ef6e137e9af289d6e3b55745323148ad66 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ab23abf392175d3e256ff619b13abbbe732a49ed -R 410ec05c8f2a9918dc68eaaf465de80d +P c6fa8d0d82805be230f672eabccdfa5680d4ddfd +R f2d9932522491a86ed313bf5e3a1693d U drh -Z 8590c3c7b25dad5b568a22b07ca7e219 +Z 4f6d25a564bc084e4d5fbc71ec75b6b1 diff --git a/manifest.uuid b/manifest.uuid index 1f38f99a6e..c03388d6de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c6fa8d0d82805be230f672eabccdfa5680d4ddfd \ No newline at end of file +9c6961967ae00e563ebe2859eaf2639a79f2cb01 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 9e83d4963a..65521f4466 100644 --- a/src/main.c +++ b/src/main.c @@ -3205,6 +3205,22 @@ int sqlite3_test_control(int op, ...){ break; } + /* + ** sqlite3_test_control(SQLITE_TESTCTRL_BYTEORDER); + ** + ** The integer returned reveals the byte-order of the computer on which + ** SQLite is running: + ** + ** 1 big-endian, determined at run-time + ** 10 little-endian, determined at run-time + ** 432101 big-endian, determined at compile-time + ** 123410 little-endian, determined at compile-time + */ + case SQLITE_TESTCTRL_BYTEORDER: { + rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; + break; + } + /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) ** ** Set the nReserve size to N for the main database on the database diff --git a/src/shell.c b/src/shell.c index f380962a86..61965d2d39 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3027,6 +3027,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ { "optimizations", SQLITE_TESTCTRL_OPTIMIZATIONS }, { "iskeyword", SQLITE_TESTCTRL_ISKEYWORD }, { "scratchmalloc", SQLITE_TESTCTRL_SCRATCHMALLOC }, + { "byteorder", SQLITE_TESTCTRL_BYTEORDER }, }; int testctrl = -1; int rc = 0; @@ -3067,9 +3068,10 @@ static int do_meta_command(char *zLine, struct callback_data *p){ break; /* sqlite3_test_control(int) */ - case SQLITE_TESTCTRL_PRNG_SAVE: - case SQLITE_TESTCTRL_PRNG_RESTORE: + case SQLITE_TESTCTRL_PRNG_SAVE: + case SQLITE_TESTCTRL_PRNG_RESTORE: case SQLITE_TESTCTRL_PRNG_RESET: + case SQLITE_TESTCTRL_BYTEORDER: if( nArg==2 ){ rc = sqlite3_test_control(testctrl); fprintf(p->out, "%d (0x%08x)\n", rc, rc); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5d2c87552c..e8dddc468c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6118,7 +6118,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 -#define SQLITE_TESTCTRL_LAST 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status diff --git a/src/sqliteInt.h b/src/sqliteInt.h index da46676adf..2a12466a4e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -553,15 +553,18 @@ const int sqlite3one = 1; #else extern const int sqlite3one; #endif -#if defined(i386) || defined(__i386__) || defined(_M_IX86)\ - || defined(__x86_64) || defined(__x86_64__) +#if defined(i386) || defined(__i386__) || defined(_M_IX86) \ + || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) \ + || defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) +# define SQLITE_BYTEORDER 1234 # define SQLITE_BIGENDIAN 0 # define SQLITE_LITTLEENDIAN 1 # define SQLITE_UTF16NATIVE SQLITE_UTF16LE #else +# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) -# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) +# define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) #endif /* From 71794dbaeb73f594ef704fa265715b207f98edc7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 00:49:29 +0000 Subject: [PATCH 142/177] Add the SQLITE_RUNTIME_BYTEORDER compile-time option to force SQLite to check the processor byte-order at run-time. Add additional compile-time byte order checks for ARM, PPC, and SPARC. FossilOrigin-Name: 2c5363873a6f990a0abaacac6303acd46b48befc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 24 +++++++++++++++++++----- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a1452995b3..97b4f2b46f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_TESTCTRL_BYTEORDER\stest\scontrol\sto\sinterrogate\sSQLite's\snotion\nof\sthe\sprocessor\sbyte\sorder\sand\swhether\sthe\sbyte\sorder\sis\sknown\sat\scompile-time\nor\sdetermined\sat\srun-time. -D 2014-04-18T00:06:02.110 +C Add\sthe\sSQLITE_RUNTIME_BYTEORDER\scompile-time\soption\sto\sforce\sSQLite\sto\scheck\nthe\sprocessor\sbyte-order\sat\srun-time.\s\sAdd\sadditional\scompile-time\sbyte\sorder\nchecks\sfor\sARM,\sPPC,\sand\sSPARC. +D 2014-04-18T00:49:29.419 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 35a265ef6e137e9af289d6e3b55745323148ad66 +F src/sqliteInt.h d3ca0bdd6743c9af2f333cb372b4bf95b19cd1fd F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c6fa8d0d82805be230f672eabccdfa5680d4ddfd -R f2d9932522491a86ed313bf5e3a1693d +P 9c6961967ae00e563ebe2859eaf2639a79f2cb01 +R 1b4cbcab83595d3a8147d0a70e1ae961 U drh -Z 4f6d25a564bc084e4d5fbc71ec75b6b1 +Z a694f54c65c4a003c92caa29b70309b3 diff --git a/manifest.uuid b/manifest.uuid index c03388d6de..05a8b60e7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c6961967ae00e563ebe2859eaf2639a79f2cb01 \ No newline at end of file +2c5363873a6f990a0abaacac6303acd46b48befc \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2a12466a4e..8197a61eb5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -546,21 +546,35 @@ typedef INT16_TYPE LogEst; /* ** Macros to determine whether the machine is big or little endian, -** evaluated at runtime. +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. */ #ifdef SQLITE_AMALGAMATION const int sqlite3one = 1; #else extern const int sqlite3one; #endif -#if defined(i386) || defined(__i386__) || defined(_M_IX86) \ - || defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) \ - || defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) +#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) # define SQLITE_BYTEORDER 1234 # define SQLITE_BIGENDIAN 0 # define SQLITE_LITTLEENDIAN 1 # define SQLITE_UTF16NATIVE SQLITE_UTF16LE -#else +#endif +#if (defined(sparc) || defined(__ppc__)) \ + && !defined(SQLITE_RUNTIME_BYTEORDER) +# define SQLITE_BYTEORDER 4321 +# define SQLITE_BIGENDIAN 1 +# define SQLITE_LITTLEENDIAN 0 +# define SQLITE_UTF16NATIVE SQLITE_UTF16BE +#endif +#if !defined(SQLITE_BYTEORDER) # define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) From b355c2cc12636ffa182433f5868668d001d0ddb3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 22:20:31 +0000 Subject: [PATCH 143/177] Clean up the proper-subset cost adjustment logic to make it more compact and easier to read and so that full branch test coverage is more easily obtained. FossilOrigin-Name: 9a5d38c79d2482a23bcfbc3ff35ca4fa269c768d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 53 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 40 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 97b4f2b46f..ad0f59f21e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_RUNTIME_BYTEORDER\scompile-time\soption\sto\sforce\sSQLite\sto\scheck\nthe\sprocessor\sbyte-order\sat\srun-time.\s\sAdd\sadditional\scompile-time\sbyte\sorder\nchecks\sfor\sARM,\sPPC,\sand\sSPARC. -D 2014-04-18T00:49:29.419 +C Clean\sup\sthe\sproper-subset\scost\sadjustment\slogic\sto\smake\sit\smore\scompact\nand\seasier\sto\sread\sand\sso\sthat\sfull\sbranch\stest\scoverage\sis\smore\seasily\nobtained. +D 2014-04-18T22:20:31.054 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7614c4383d8b6143558dc349da286d0325704d35 +F src/where.c 114b480d3c2eefd9fbc395ee96e03fbd883f418d F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9c6961967ae00e563ebe2859eaf2639a79f2cb01 -R 1b4cbcab83595d3a8147d0a70e1ae961 +P 2c5363873a6f990a0abaacac6303acd46b48befc +R b712e75910761b4771536f725cdc6409 U drh -Z a694f54c65c4a003c92caa29b70309b3 +Z e77c4b2dd81e4b608cc5f9bf1e51b6ae diff --git a/manifest.uuid b/manifest.uuid index 05a8b60e7e..3f9af7e6d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c5363873a6f990a0abaacac6303acd46b48befc \ No newline at end of file +9a5d38c79d2482a23bcfbc3ff35ca4fa269c768d \ No newline at end of file diff --git a/src/where.c b/src/where.c index e51eee5350..31a0dc5452 100644 --- a/src/where.c +++ b/src/where.c @@ -3712,18 +3712,36 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if the set of WHERE clause terms used by pA is a proper -** subset of the WHERE clause terms used by pB. +** Return TRUE if both of the following are true: +** +** (1) X has the same or lower cost that Y +** (2) X is a proper subset of Y +** +** By "proper subset" we mean that X uses fewer WHERE clause terms +** than Y and that every WHERE clause term used by X is also used +** by Y. +** +** If X is a proper subset of Y then Y is a better choice and ought +** to have a lower cost. This routine returns TRUE when that cost +** relationship is inverted and needs to be adjusted. */ -static int whereLoopProperSubset(const WhereLoop *pA, const WhereLoop *pB){ +static int whereLoopCheaperProperSubset( + const WhereLoop *pX, /* First WhereLoop to compare */ + const WhereLoop *pY /* Compare against this WhereLoop */ +){ int i, j; - assert( pA->nLTermnLTerm ); /* Checked by calling function */ - for(j=0, i=pA->nLTerm-1; i>=0 && j>=0; i--){ - for(j=pB->nLTerm-1; j>=0; j--){ - if( pB->aLTerm[j]==pA->aLTerm[i] ) break; - } + if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->rRun >= pY->rRun ){ + if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ + if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } - return j>=0; + for(j=0, i=pX->nLTerm-1; i>=0; i--){ + for(j=pY->nLTerm-1; j>=0; j--){ + if( pY->aLTerm[j]==pX->aLTerm[i] ) break; + } + if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + } + return 1; /* All conditions meet */ } /* @@ -3745,19 +3763,14 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( p->nLTermnLTerm - && (p->rRunrRun || (p->rRun==pTemplate->rRun && - p->nOut<=pTemplate->nOut)) - && whereLoopProperSubset(p, pTemplate) - ){ + if( whereLoopCheaperProperSubset(p, pTemplate) ){ + /* Adjust pTemplate cost downward so that it is cheaper than its + ** subset p */ pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; - }else - if( p->nLTerm>pTemplate->nLTerm - && (p->rRun>pTemplate->rRun || (p->rRun==pTemplate->rRun && - p->nOut>=pTemplate->nOut)) - && whereLoopProperSubset(pTemplate, p) - ){ + }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ + /* Adjust pTemplate cost upward so that it is costlier than p since + ** pTemplate is a proper subset of p */ pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } From 374cd78c1db5b7b085cf2c289cfc3b04afaa2716 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 21 Apr 2014 13:21:56 +0000 Subject: [PATCH 144/177] Avoid discarding an ORDER BY clause in the case where an identical GROUP BY clauses uses an index to group, but not sort, the rows. Fix for [b75a9ca6b0]. FossilOrigin-Name: de9a490f594183f337a2ec9e0f87792eac83548b --- manifest | 21 +++++------ manifest.uuid | 2 +- src/select.c | 43 ++++++++++++++++------- src/sqliteInt.h | 2 ++ src/where.c | 47 +++++++++++++++++++++++-- src/whereInt.h | 1 + test/tkt-b75a9ca6b0.test | 75 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 164 insertions(+), 27 deletions(-) create mode 100644 test/tkt-b75a9ca6b0.test diff --git a/manifest b/manifest index ad0f59f21e..caaf603930 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\sproper-subset\scost\sadjustment\slogic\sto\smake\sit\smore\scompact\nand\seasier\sto\sread\sand\sso\sthat\sfull\sbranch\stest\scoverage\sis\smore\seasily\nobtained. -D 2014-04-18T22:20:31.054 +C Avoid\sdiscarding\san\sORDER\sBY\sclause\sin\sthe\scase\swhere\san\sidentical\sGROUP\sBY\sclauses\suses\san\sindex\sto\sgroup,\sbut\snot\ssort,\sthe\srows.\sFix\sfor\s[b75a9ca6b0]. +D 2014-04-21T13:21:56.964 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c 269c3e31a450fce642a10569221a49180348c88e +F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h d3ca0bdd6743c9af2f333cb372b4bf95b19cd1fd +F src/sqliteInt.h 03e2f60ccb0745fa2d3a072cb4f75fa29251d2ee F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 114b480d3c2eefd9fbc395ee96e03fbd883f418d -F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af +F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -896,6 +896,7 @@ F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 +F test/tkt-b75a9ca6b0.test 3f4f7ca6ab552b6efdb664f821bc798a08929b2c F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 @@ -1160,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2c5363873a6f990a0abaacac6303acd46b48befc -R b712e75910761b4771536f725cdc6409 -U drh -Z e77c4b2dd81e4b608cc5f9bf1e51b6ae +P 9a5d38c79d2482a23bcfbc3ff35ca4fa269c768d +R ee944819c64836615905c8a0073df726 +U dan +Z 6ac990d90beb1121a87e97a89a4f2118 diff --git a/manifest.uuid b/manifest.uuid index 3f9af7e6d6..6776a57944 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a5d38c79d2482a23bcfbc3ff35ca4fa269c768d \ No newline at end of file +de9a490f594183f337a2ec9e0f87792eac83548b \ No newline at end of file diff --git a/src/select.c b/src/select.c index c7f0b24a4a..6efdde4b72 100644 --- a/src/select.c +++ b/src/select.c @@ -4719,18 +4719,6 @@ int sqlite3Select( } #endif - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then disable the ORDER BY clause since the GROUP BY - ** will cause elements to come out in the correct order. This is - ** an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER - ** to disable this optimization for testing purposes. - */ - if( sqlite3ExprListCompare(p->pGroupBy, sSort.pOrderBy, -1)==0 - && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ - sSort.pOrderBy = 0; - } - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: @@ -4859,6 +4847,7 @@ int sqlite3Select( int addrEnd; /* End of processing for this SELECT */ int sortPTab = 0; /* Pseudotable used to decode sorting results */ int sortOut = 0; /* Output register from the sorter */ + int orderByGrp = 0; /* True if the GROUP BY and ORDER BY are the same */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -4878,6 +4867,18 @@ int sqlite3Select( p->nSelectRow = 1; } + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY may use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } /* Create a label to jump to when we want to abort the query */ addrEnd = sqlite3VdbeMakeLabel(v); @@ -4958,7 +4959,8 @@ int sqlite3Select( */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, - WHERE_GROUPBY, 0); + WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + ); if( pWInfo==0 ) goto select_end; if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so @@ -5023,6 +5025,21 @@ int sqlite3Select( VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); + + } + + /* If the index or temporary table used by the GROUP BY sort + ** will naturally deliver rows in the order required by the ORDER BY + ** clause, cancel the ephemeral table open coded earlier. + ** + ** This is an optimization - the correct answer should result regardless. + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** disable this optimization for testing purposes. */ + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + && (groupBySort || sqlite3WhereIsSorted(pWInfo)) + ){ + sSort.pOrderBy = 0; + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Evaluate the current GROUP BY terms and store in b0, b1, b2... diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8197a61eb5..8e81b7a970 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2125,6 +2125,7 @@ struct SrcList { #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -3092,6 +3093,7 @@ void sqlite3WhereEnd(WhereInfo*); u64 sqlite3WhereOutputRowCount(WhereInfo*); int sqlite3WhereIsDistinct(WhereInfo*); int sqlite3WhereIsOrdered(WhereInfo*); +int sqlite3WhereIsSorted(WhereInfo*); int sqlite3WhereContinueLabel(WhereInfo*); int sqlite3WhereBreakLabel(WhereInfo*); int sqlite3WhereOkOnePass(WhereInfo*, int*); diff --git a/src/where.c b/src/where.c index 31a0dc5452..9bde27e52f 100644 --- a/src/where.c +++ b/src/where.c @@ -4796,7 +4796,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY -** and DISTINT do not require rows to appear in any particular order as long +** and DISTINCT do not require rows to appear in any particular order as long ** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. @@ -4965,7 +4965,7 @@ static i8 wherePathSatisfiesOrderBy( } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and and mark that ORDER BY term off + ** of the index and mark that ORDER BY term off */ bOnce = 1; isMatch = 0; @@ -5045,6 +5045,36 @@ static i8 wherePathSatisfiesOrderBy( return -1; } + +/* +** If the WHERE_GROUPBY flag is set in the mask passed to sqlite3WhereBegin(), +** the planner assumes that the specified pOrderBy list is actually a GROUP +** BY clause - and so any order that groups rows as required satisfies the +** request. +** +** Normally, in this case it is not possible for the caller to determine +** whether or not the rows are really being delivered in sorted order, or +** just in some other order that provides the required grouping. However, +** if the WHERE_SORTBYGROUP flag is also passed to sqlite3WhereBegin(), then +** this function may be called on the returned WhereInfo object. It returns +** true if the rows really will be sorted in the specified order, or false +** otherwise. +** +** For example, assuming: +** +** CREATE INDEX i1 ON t1(x, Y); +** +** then +** +** SELECT * FROM t1 GROUP BY x,y ORDER BY x,y; -- IsSorted()==1 +** SELECT * FROM t1 GROUP BY y,x ORDER BY y,x; -- IsSorted()==0 +*/ +int sqlite3WhereIsSorted(WhereInfo *pWInfo){ + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); + return pWInfo->sorted; +} + #ifdef WHERETRACE_ENABLED /* For debugging use only: */ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ @@ -5057,7 +5087,6 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ } #endif - /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -5338,7 +5367,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; } + if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) + && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr + ){ + Bitmask notUsed = 0; + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used + ); + assert( pWInfo->sorted==0 ); + pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + } } + + pWInfo->nRowOut = pFrom->nRow; /* Free temporary memory and return success */ diff --git a/src/whereInt.h b/src/whereInt.h index 419bb81c0e..72e7530db9 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -398,6 +398,7 @@ struct WhereInfo { LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 sorted; /* True if really sorted (not just grouped) */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ diff --git a/test/tkt-b75a9ca6b0.test b/test/tkt-b75a9ca6b0.test new file mode 100644 index 0000000000..38918bedaa --- /dev/null +++ b/test/tkt-b75a9ca6b0.test @@ -0,0 +1,75 @@ +# 2014 April 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Test that ticket [b75a9ca6b0] has been fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-b75a9ca6b0 + +do_execsql_test 1 { + CREATE TABLE t1 (x, y); + INSERT INTO t1 VALUES (1, 3); + INSERT INTO t1 VALUES (2, 2); + INSERT INTO t1 VALUES (3, 1); +} + +do_execsql_test 1.1 { + CREATE INDEX i1 ON t1(x, y); +} + +set idxscan {0 0 0 {SCAN TABLE t1 USING COVERING INDEX i1}} +set tblscan {0 0 0 {SCAN TABLE t1}} +set grpsort {0 0 0 {USE TEMP B-TREE FOR GROUP BY}} +set sort {0 0 0 {USE TEMP B-TREE FOR ORDER BY}} + +foreach {tn q res eqp} [subst -nocommands { + 1 "SELECT * FROM t1 GROUP BY x, y ORDER BY x,y" + {1 3 2 2 3 1} {$idxscan} + + 2 "SELECT * FROM t1 GROUP BY x, y ORDER BY x" + {1 3 2 2 3 1} {$idxscan $sort} + + 3 "SELECT * FROM t1 GROUP BY y, x ORDER BY y, x" + {3 1 2 2 1 3} {$idxscan $sort} + + 4 "SELECT * FROM t1 GROUP BY x ORDER BY x" + {1 3 2 2 3 1} {$idxscan} + + 5 "SELECT * FROM t1 GROUP BY y ORDER BY y" + {3 1 2 2 1 3} {$tblscan $grpsort} + + 6 "SELECT * FROM t1 GROUP BY y ORDER BY x" + {1 3 2 2 3 1} {$tblscan $grpsort $sort} + + 7 "SELECT * FROM t1 GROUP BY x, y ORDER BY x, y DESC" + {1 3 2 2 3 1} {$idxscan $sort} + + 8 "SELECT * FROM t1 GROUP BY x, y ORDER BY x DESC, y DESC" + {3 1 2 2 1 3} {$idxscan $sort} + + 9 "SELECT * FROM t1 GROUP BY x, y ORDER BY x ASC, y ASC" + {1 3 2 2 3 1} {$idxscan} + + 10 "SELECT * FROM t1 GROUP BY x, y ORDER BY x COLLATE nocase, y" + {1 3 2 2 3 1} {$idxscan $sort} + +}] { + do_execsql_test 1.$tn.1 $q $res + do_eqp_test 1.$tn.2 $q $eqp +} + + +finish_test + + + From f0582f43dd5c2a26117bac75995869eb84fb0521 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 21 Apr 2014 13:36:54 +0000 Subject: [PATCH 145/177] Comment tweaks on the test case for the [b75a9ca6b0] bug fix. FossilOrigin-Name: 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/tkt-b75a9ca6b0.test | 10 ++++++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index caaf603930..7f4e9189fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sdiscarding\san\sORDER\sBY\sclause\sin\sthe\scase\swhere\san\sidentical\sGROUP\sBY\sclauses\suses\san\sindex\sto\sgroup,\sbut\snot\ssort,\sthe\srows.\sFix\sfor\s[b75a9ca6b0]. -D 2014-04-21T13:21:56.964 +C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. +D 2014-04-21T13:36:54.639 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -896,7 +896,7 @@ F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 -F test/tkt-b75a9ca6b0.test 3f4f7ca6ab552b6efdb664f821bc798a08929b2c +F test/tkt-b75a9ca6b0.test 97cc2d5eeaf82799eb42138c0a1ff64370238ce4 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9a5d38c79d2482a23bcfbc3ff35ca4fa269c768d -R ee944819c64836615905c8a0073df726 -U dan -Z 6ac990d90beb1121a87e97a89a4f2118 +P de9a490f594183f337a2ec9e0f87792eac83548b +R ce888b84132e0cad3bcca115a32951d3 +U drh +Z cf9f241149456ab1fa24984e95a412d2 diff --git a/manifest.uuid b/manifest.uuid index 6776a57944..a2761a597b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de9a490f594183f337a2ec9e0f87792eac83548b \ No newline at end of file +65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file diff --git a/test/tkt-b75a9ca6b0.test b/test/tkt-b75a9ca6b0.test index 38918bedaa..0c81a534da 100644 --- a/test/tkt-b75a9ca6b0.test +++ b/test/tkt-b75a9ca6b0.test @@ -1,4 +1,4 @@ -# 2014 April 21 +# 2014-04-21 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -11,6 +11,11 @@ # # Test that ticket [b75a9ca6b0] has been fixed. # +# Ticket [b75a9ca6b0] concerns queries that have both a GROUP BY +# and an ORDER BY. This code verifies that SQLite is able to +# optimize out the ORDER BY in some circumstances, but retains the +# ORDER BY when necessary. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -70,6 +75,3 @@ foreach {tn q res eqp} [subst -nocommands { finish_test - - - From aa9933c11529cabf659615d9c94277846dacbd15 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Apr 2014 20:04:49 +0000 Subject: [PATCH 146/177] Changes to the way the planner calculates the costs of various table and index scans. Some test cases still failing. FossilOrigin-Name: c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 --- manifest | 25 +++++----- manifest.uuid | 2 +- src/where.c | 109 ++++++++++++++++++++++--------------------- test/analyze9.test | 8 ++-- test/autoindex1.test | 2 + test/where3.test | 3 +- test/whereG.test | 32 +++++++++++++ 7 files changed, 111 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..2b621bd738 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. +D 2014-04-24T20:04:49.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 +F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -329,7 +329,7 @@ F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test d4dfe14001dfcb74cfbd7107f45a79fc1ab6183e +F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1079,7 +1079,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db -F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e +F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 +F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 -U drh -Z cf9f241149456ab1fa24984e95a412d2 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R 6cc54703275bf2ed6708c34ae52cf7ea +T *branch * experimental-costs +T *sym-experimental-costs * +T -sym-trunk * +U dan +Z 868aa60f36dda291ae018583501645e5 diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..4c2b675cff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6a75840525 100644 --- a/src/where.c +++ b/src/where.c @@ -227,7 +227,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -1975,6 +1975,31 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( pTerm->wtFlags & TERM_VNULL ){ + nRet -= 10; assert( 10==sqlite3LogEst(2) ); + }else{ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2127,17 +2152,9 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -3987,7 +4004,9 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } @@ -4081,6 +4100,7 @@ static int whereLoopAddBtreeIndex( pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + LogEst rCostIdx; int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; @@ -4154,7 +4174,8 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm-2] : 0; } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); } @@ -4181,13 +4202,16 @@ static int whereLoopAddBtreeIndex( } } #endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -4319,6 +4343,7 @@ static int whereLoopAddBtree( sPk.aiRowEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; aiRowEstPk[0] = pTab->nRowEst; aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; @@ -4396,10 +4421,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4426,35 +4449,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: The base cost of an index scan is N + log2(N). - ** The log2(N) is for the initial seek to the beginning and the N - ** is for the scan itself. */ - pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - ** The upper bound on K (3.0) matches the penalty factor - ** on a full table scan that tries to encourage the use of - ** indexed lookups over full scans. - */ - pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: The cost of scanning a non-covering index is multiplied - ** by log2(N) to account for the binary search of the main table - ** that must happen for each row of the index. - ** TODO: Should there be a multiplier here, analogous to the 3x - ** multiplier for a fulltable scan or covering index scan, to - ** further discourage the use of an index scan? Or is the log2(N) - ** term sufficient discouragement? - ** TODO: What if some or all of the WHERE clause terms can be - ** computed without reference to the original table. Then the - ** penality should reduce to logK where K is the number of output - ** rows. - */ - pNew->rRun += rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4732,8 +4736,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->iSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + pNew->rRun = sSum.a[i].rRun; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/analyze9.test b/test/analyze9.test index 820bcdb0e7..125cecf182 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -577,16 +577,16 @@ do_test 13.1 { execsql ANALYZE } {} do_eqp_test 13.2.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.2.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.3.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} do_eqp_test 13.3.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} #------------------------------------------------------------------------- diff --git a/test/autoindex1.test b/test/autoindex1.test index 0e5032b9b9..6cb0ab146a 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -97,6 +97,8 @@ do_test autoindex1-210 { PRAGMA automatic_index=ON; ANALYZE; UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + -- Table t2 actually contains 8 rows. + UPDATE sqlite_stat1 SET stat='16' WHERE tbl='t2'; ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } diff --git a/test/where3.test b/test/where3.test index 8fa9fa7840..c2804b5579 100644 --- a/test/where3.test +++ b/test/where3.test @@ -231,6 +231,7 @@ do_execsql_test where3-3.0 { CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); CREATE INDEX t301c ON t301(c); INSERT INTO t301 VALUES(1,2,3); + INSERT INTO t301 VALUES(2,2,3); CREATE TABLE t302(x, y); INSERT INTO t302 VALUES(4,5); ANALYZE; @@ -251,7 +252,7 @@ do_execsql_test where3-3.2 { } {} do_execsql_test where3-3.3 { SELECT * FROM t301 WHERE c=3 AND a IS NOT NULL; -} {1 2 3} +} {1 2 3 2 2 3} if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be diff --git a/test/whereG.test b/test/whereG.test index 17d5653223..6274213491 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix whereG do_execsql_test whereG-1.0 { CREATE TABLE composer( @@ -179,5 +180,36 @@ do_execsql_test whereG-4.0 { ORDER BY x; } {right} +#------------------------------------------------------------------------- +# + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); +} +do_eqp_test 5.1.2 { + SELECT * FROM t1 WHERE a>? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} +do_eqp_test 5.1.3 { + SELECT * FROM t1 WHERE likelihood(a>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_test 5.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 VALUES('abc', $i, $i); } + } + execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } + execsql { ANALYZE } +} {} + +do_eqp_test 5.2.2 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.01) +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} + +do_eqp_test 5.2.3 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} finish_test + From a01c7c70fd4d36fcbb6b71c783efb877b6cfbd5f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 12:35:31 +0000 Subject: [PATCH 147/177] When VDBE_PROFILE is enabled, ignore negative opcode times, which sometimes occur, perhaps due to context swaps. FossilOrigin-Name: 4e88042f9d3e678914da96c0eb276f3d8fca5a94 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..b0a9e4b587 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C When\sVDBE_PROFILE\sis\senabled,\signore\snegative\sopcode\stimes,\swhich\ssometimes\noccur,\sperhaps\sdue\sto\scontext\sswaps. +D 2014-04-25T12:35:31.621 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c bf284edea1cee0508dc93c79a73498e0f317edb7 +F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R c6b40d1b54d6305d3c2bd44472b1c76f U drh -Z cf9f241149456ab1fa24984e95a412d2 +Z 6f02403cb59a60204ab5357a363dc55d diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..1b88d91a9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +4e88042f9d3e678914da96c0eb276f3d8fca5a94 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6102cf3c7e..4ef6d0738c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6323,8 +6323,8 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { - u64 elapsed = sqlite3Hwtime() - start; - pOp->cycles += elapsed; + u64 endTime = sqlite3Hwtime(); + if( endTime>start ) pOp->cycles += endTime - start; pOp->cnt++; } #endif From 9dfaf621d1e88d5570cc6547a2ab495f66f1a12f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 14:42:17 +0000 Subject: [PATCH 148/177] Make sure ORDER BY clauses on joins that involve virtual tables and that have a DISTINCT clause work correctly. This is a candidate fix for ticket [388d01d4bb8f9]. Test cases for that ticket will be checked in separately. FossilOrigin-Name: 171138122690faafde0dcab0201b90bdf02d3637 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 ++++--------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b0a9e4b587..459765f3d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sVDBE_PROFILE\sis\senabled,\signore\snegative\sopcode\stimes,\swhich\ssometimes\noccur,\sperhaps\sdue\sto\scontext\sswaps. -D 2014-04-25T12:35:31.621 +C Make\ssure\sORDER\sBY\sclauses\son\sjoins\sthat\sinvolve\svirtual\stables\sand\sthat\nhave\sa\sDISTINCT\sclause\swork\scorrectly.\s\sThis\sis\sa\scandidate\sfix\sfor\nticket\s[388d01d4bb8f9].\s\sTest\scases\sfor\sthat\sticket\swill\sbe\schecked\sin\nseparately. +D 2014-04-25T14:42:17.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c 6ae02f1e8b1b29744d9e8cd9b95eac4c5232736d F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R c6b40d1b54d6305d3c2bd44472b1c76f +P 4e88042f9d3e678914da96c0eb276f3d8fca5a94 +R 1db9e5d4fb5773f03fcd55df7d35ffca U drh -Z 6f02403cb59a60204ab5357a363dc55d +Z 5fd090bfe2e284c7964226b5abb1772a diff --git a/manifest.uuid b/manifest.uuid index 1b88d91a9c..62175a4a6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e88042f9d3e678914da96c0eb276f3d8fca5a94 \ No newline at end of file +171138122690faafde0dcab0201b90bdf02d3637 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6342b7b50b 100644 --- a/src/where.c +++ b/src/where.c @@ -4857,14 +4857,6 @@ static i8 wherePathSatisfiesOrderBy( */ assert( pOrderBy!=0 ); - - /* Sortability of virtual tables is determined by the xBestIndex method - ** of the virtual table itself */ - if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ - testcase( nLoop>0 ); /* True when outer loops are one-row and match - ** no ORDER BY terms */ - return pLast->u.vtab.isOrdered; - } if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nOrderBy = pOrderBy->nExpr; @@ -4877,7 +4869,10 @@ static i8 wherePathSatisfiesOrderBy( for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; pLoop = iLoopaLoop[iLoop] : pLast; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; /* Mark off any ORDER BY term X that is a column in the table of From cfc9df76e11b05d275e77d067f359c12044ea943 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 15:01:01 +0000 Subject: [PATCH 149/177] Store values loaded from the stat1 table as logarithmic values in memory. FossilOrigin-Name: 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/analyze.c | 19 ++++++++++++------- src/build.c | 26 +++++++++++++++++++------- src/pragma.c | 6 ++++-- src/select.c | 10 +++++----- src/sqliteInt.h | 5 ++++- src/where.c | 30 ++++++++++++++++-------------- 8 files changed, 72 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 2b621bd738..0c855c53e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. -D 2014-04-24T20:04:49.939 +C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. +D 2014-04-25T15:01:01.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 +F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 +F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -211,18 +211,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 +F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 +F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 03e2f60ccb0745fa2d3a072cb4f75fa29251d2ee +F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee +F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,10 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 6cc54703275bf2ed6708c34ae52cf7ea -T *branch * experimental-costs -T *sym-experimental-costs * -T -sym-trunk * +P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 +R 63fad85eb66cf540e6aa11923b167cbf U dan -Z 868aa60f36dda291ae018583501645e5 +Z 04f4e3645ebd9e33526df3bd26c04a76 diff --git a/manifest.uuid b/manifest.uuid index 4c2b675cff..5436721392 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file +1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2a03e292c8..4fbaaa5173 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1371,6 +1371,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -1389,7 +1390,11 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; + if( aOut ){ + aOut[i] = v; + }else{ + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1445,12 +1450,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1642,9 +1647,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the diff --git a/src/build.c b/src/build.c index 4d4155ba90..fb025495dc 100644 --- a/src/build.c +++ b/src/build.c @@ -905,7 +905,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -2730,15 +2730,15 @@ Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -2968,7 +2968,7 @@ Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -3249,7 +3249,7 @@ exit_create_index: ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -3260,6 +3260,7 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ +#if 0 tRowcnt *a = pIdx->aiRowEst; int i; tRowcnt n; @@ -3274,6 +3275,17 @@ void sqlite3DefaultRowEst(Index *pIdx){ if( pIdx->onError!=OE_None ){ a[pIdx->nKeyCol] = 1; } +#endif + /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ + LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + + a[0] = pIdx->pTable->nRowLogEst; + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + if( pIdx->onError!=OE_None ){ + a[pIdx->nKeyCol] = 0; + } } /* diff --git a/src/pragma.c b/src/pragma.c index 66d0c3328e..4c69ceb4fd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1488,13 +1488,15 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } diff --git a/src/select.c b/src/select.c index 6efdde4b72..dfca6d3f83 100644 --- a/src/select.c +++ b/src/select.c @@ -1690,7 +1690,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -3829,7 +3829,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4005,7 +4005,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -4655,7 +4655,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); @@ -4686,7 +4686,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8e81b7a970..3ddcc8610f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1471,7 +1471,7 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ @@ -1680,7 +1680,10 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ +#if 0 tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ +#endif + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ diff --git a/src/where.c b/src/where.c index 6a75840525..909129f666 100644 --- a/src/where.c +++ b/src/where.c @@ -1956,7 +1956,8 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); @@ -2092,7 +2093,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = p->aiRowEst[0]; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -2251,6 +2252,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -2259,14 +2261,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -4059,8 +4061,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; + nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; nRowEst = 0; @@ -4074,7 +4075,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4082,10 +4083,11 @@ static int whereLoopAddBtreeIndex( ** number 18 was found by experimentation to be the payoff point where ** skip-scan become faster than a full-scan. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4093,7 +4095,7 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; pNew->rRun = rLogSize + nIter; pNew->nOut += nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); @@ -4305,7 +4307,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -4340,12 +4342,12 @@ static int whereLoopAddBtree( memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -4354,7 +4356,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX From b79c718f1ab8667bd14040c339a8e87c90bcff82 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 17:37:16 +0000 Subject: [PATCH 150/177] Add test cases to ensure correct operation of joins with a virtual table that include DISTINCT and ORDER BY clauses. Verification for ticket [388d01d4bb8f9]. FossilOrigin-Name: 5ada136f43ce744ae8c349eff39838eb44611b6e --- manifest | 11 ++--- manifest.uuid | 2 +- test/orderby7.test | 106 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 test/orderby7.test diff --git a/manifest b/manifest index 459765f3d0..4bb29e9787 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sORDER\sBY\sclauses\son\sjoins\sthat\sinvolve\svirtual\stables\sand\sthat\nhave\sa\sDISTINCT\sclause\swork\scorrectly.\s\sThis\sis\sa\scandidate\sfix\sfor\nticket\s[388d01d4bb8f9].\s\sTest\scases\sfor\sthat\sticket\swill\sbe\schecked\sin\nseparately. -D 2014-04-25T14:42:17.085 +C Add\stest\scases\sto\sensure\scorrect\soperation\sof\sjoins\swith\sa\svirtual\stable\nthat\sinclude\sDISTINCT\sand\sORDER\sBY\sclauses.\s\sVerification\sfor\sticket\n[388d01d4bb8f9]. +D 2014-04-25T17:37:16.863 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -724,6 +724,7 @@ F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 +F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e88042f9d3e678914da96c0eb276f3d8fca5a94 -R 1db9e5d4fb5773f03fcd55df7d35ffca +P 171138122690faafde0dcab0201b90bdf02d3637 +R a91060e3dc13463e0dda2e961246be75 U drh -Z 5fd090bfe2e284c7964226b5abb1772a +Z 5a55daa8fd800f752600937698ffd8a1 diff --git a/manifest.uuid b/manifest.uuid index 62175a4a6f..f56246ea09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -171138122690faafde0dcab0201b90bdf02d3637 \ No newline at end of file +5ada136f43ce744ae8c349eff39838eb44611b6e \ No newline at end of file diff --git a/test/orderby7.test b/test/orderby7.test new file mode 100644 index 0000000000..8c5fc9a85f --- /dev/null +++ b/test/orderby7.test @@ -0,0 +1,106 @@ +# 2014-04-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing ORDER BY optimizations on joins +# that involve virtual tables. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby7 + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE fts USING fts3(content TEXT); + INSERT INTO fts(rowid,content) + VALUES(1,'this is a test of the fts3 virtual'), + (2,'table used as part of a join together'), + (3,'with the DISTINCT keyword. There was'), + (4,'a bug at one time (2013-06 through 2014-04)'), + (5,'that prevented this from working correctly.'), + (11,'a row that occurs twice'), + (12,'a row that occurs twice'); + + CREATE TABLE t1(x TEXT PRIMARY KEY, y); + INSERT OR IGNORE INTO t1 SELECT content, rowid+100 FROM fts; +} {} +do_execsql_test 1.1 { + SELECT DISTINCT fts.rowid, t1.y + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY y; +} {11 111 12 111} +do_execsql_test 1.2 { + SELECT DISTINCT fts.rowid, t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {11 {a row that occurs twice} 12 {a row that occurs twice}} +do_execsql_test 1.3 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {{a row that occurs twice}} +do_execsql_test 1.4 { + SELECT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {{a row that occurs twice} {a row that occurs twice}} +do_execsql_test 1.5 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x; +} {{a row that occurs twice}} +do_execsql_test 1.6 { + SELECT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x; +} {{a row that occurs twice} {a row that occurs twice}} + +do_execsql_test 2.1 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY fts.rowid; +} {{a row that occurs twice}} +do_execsql_test 2.2 { + SELECT DISTINCT t1.* + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY fts.rowid; +} {{a row that occurs twice} 111} +do_execsql_test 2.3 { + SELECT DISTINCT t1.* + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY t1.y +} {{a row that occurs twice} 111} + + + + +finish_test From 8ad1d8ba38ef4bf965ab1a657892b0e57be00a97 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 20:22:45 +0000 Subject: [PATCH 151/177] If the user provides likelihood() data for a WHERE clause term used as part of an index key, have the planner use it when calculating the expected number of rows visited by the loop. FossilOrigin-Name: c51efaa5d29ee0a91b9e6a83a8dd82530670811a --- manifest | 18 +++--- manifest.uuid | 2 +- src/sqliteInt.h | 3 - src/where.c | 146 ++++++++++++++++++++++++++++------------------- src/whereInt.h | 1 + test/whereG.test | 18 ++++-- 6 files changed, 111 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 0c855c53e6..afdc8889fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. -D 2014-04-25T15:01:01.691 +C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. +D 2014-04-25T20:22:45.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e +F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 +F test/whereG.test 0ac23e5e8311b69d87245f4a85112de321031658 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 -R 63fad85eb66cf540e6aa11923b167cbf +P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 +R 804d3aff8175381c892eb4eebbb86307 U dan -Z 04f4e3645ebd9e33526df3bd26c04a76 +Z 6dd1c2a2c429274ab108441094bd0963 diff --git a/manifest.uuid b/manifest.uuid index 5436721392..71ddf4b2d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file +c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ddcc8610f..b584ff401d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1680,9 +1680,6 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ -#if 0 - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ -#endif LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ diff --git a/src/where.c b/src/where.c index 909129f666..73d3a22b7c 100644 --- a/src/where.c +++ b/src/where.c @@ -4040,7 +4040,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4061,10 +4060,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; - nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4095,35 +4092,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->rRun = rLogSize + nIter; - pNew->nOut += nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -4135,87 +4137,111 @@ static int whereLoopAddBtreeIndex( } assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->onError==OE_None ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + }else if( (eOp & WO_ISNULL)==0 ){ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 ** data, using some other estimate. */ - assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + pNew->wsFlags |= WHERE_LIKELIHOOD; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && (pNew->wsFlags & WHERE_LIKELIHOOD)==0 + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + assert( rc!=SQLITE_OK || nOut>0 ); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } } } -#endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = nOutUnadjusted; + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) ){ diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..010cd6e8ae 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -458,3 +458,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_LIKELIHOOD 0x00020000 /* A likelihood() is affecting nOut */ diff --git a/test/whereG.test b/test/whereG.test index 6274213491..66918a35fa 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -181,9 +181,14 @@ do_execsql_test whereG-4.0 { } {right} #------------------------------------------------------------------------- -# - +# Test that likelihood() specifications on indexed terms are taken into +# account by various forms of loops. +# +# 5.1.*: open ended range scans +# 5.2.*: skip-scans +# reset_db + do_execsql_test 5.1 { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a, b); @@ -202,14 +207,19 @@ do_test 5.2 { execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } execsql { ANALYZE } } {} - do_eqp_test 5.2.2 { SELECT * FROM t1 WHERE likelihood(b>?, 0.01) } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} - do_eqp_test 5.2.3 { SELECT * FROM t1 WHERE likelihood(b>?, 0.9) } {0 0 0 {SCAN TABLE t1}} +do_eqp_test 5.3.1 { + SELECT * FROM t1 WHERE a=? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} +do_eqp_test 5.3.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + finish_test From 9940e2aa45604e9e9bf4c23af16b00bb78a03384 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 14:07:57 +0000 Subject: [PATCH 152/177] Avoid transfering records between tables unless the default values for all columns are the same. Fix for [f67b41381a]. FossilOrigin-Name: f8c4c495e6de1f124d205383d4bafa46accbff5c --- manifest | 15 ++++++------ manifest.uuid | 2 +- src/insert.c | 13 ++++++++--- test/tkt-f67b41381a.test | 50 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 test/tkt-f67b41381a.test diff --git a/manifest b/manifest index 4bb29e9787..498632b350 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\sensure\scorrect\soperation\sof\sjoins\swith\sa\svirtual\stable\nthat\sinclude\sDISTINCT\sand\sORDER\sBY\sclauses.\s\sVerification\sfor\sticket\n[388d01d4bb8f9]. -D 2014-04-25T17:37:16.863 +C Avoid\stransfering\srecords\sbetween\stables\sunless\sthe\sdefault\svalues\sfor\sall\scolumns\sare\sthe\ssame.\sFix\sfor\s[f67b41381a]. +D 2014-04-26T14:07:57.099 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d8bb30535c8c0785876025a4a07f9074640a15d1 +F src/insert.c 08de23111439c650cac52861a5b80d10d9d40560 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -906,6 +906,7 @@ F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 +F test/tkt-f67b41381a.test 18d3477b6fe82e6c5ec5e233d79642d7e6d0de77 F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead @@ -1162,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 171138122690faafde0dcab0201b90bdf02d3637 -R a91060e3dc13463e0dda2e961246be75 -U drh -Z 5a55daa8fd800f752600937698ffd8a1 +P 5ada136f43ce744ae8c349eff39838eb44611b6e +R 72b6179d3d1a0fd8f077c8cde285a253 +U dan +Z 7eecc39cd60ba5323d907c2532e97144 diff --git a/manifest.uuid b/manifest.uuid index f56246ea09..6aa5d7fb27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ada136f43ce744ae8c349eff39838eb44611b6e \ No newline at end of file +f8c4c495e6de1f124d205383d4bafa46accbff5c \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index abdf1ada95..3c77761b32 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1865,15 +1865,22 @@ static int xferOptimization( return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ } for(i=0; inCol; i++){ - if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; + if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ return 0; /* Collating sequence must be the same on all columns */ } - if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ + if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } + if( (pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)) + ){ + return 0; /* Default values must be the same for all columns */ + } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ if( pDestIdx->onError!=OE_None ){ diff --git a/test/tkt-f67b41381a.test b/test/tkt-f67b41381a.test new file mode 100644 index 0000000000..d15d985342 --- /dev/null +++ b/test/tkt-f67b41381a.test @@ -0,0 +1,50 @@ +# 2014 April 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Test that ticket f67b41381a has been resolved. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-f67b41381a + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + ALTER TABLE t1 ADD COLUMN b DEFAULT 2; + CREATE TABLE t2(a, b); + INSERT INTO t2 SELECT * FROM t1; + SELECT * FROM t2; +} {1 2} + +db cache size 0 +foreach {tn tbls xfer} { + 1 { CREATE TABLE t1(a, b); CREATE TABLE t2(a, b) } 1 + 2 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b) } 0 + 3 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b DEFAULT 'x') } 1 + 4 { CREATE TABLE t1(a, b DEFAULT NULL); CREATE TABLE t2(a, b) } 0 + 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 0 + 6 { CREATE TABLE t1(a DEFAULT 1, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 +} { + + execsql { DROP TABLE t1; DROP TABLE t2 } + execsql $tbls + + set res 1 + db eval { EXPLAIN INSERT INTO t1 SELECT * FROM t2 } { + if {$opcode == "Column"} { set res 0 } + } + + do_test 2.$tn [list set res] $xfer +} + +finish_test + + From 453e0261dfed94ff6dcccaa6002dbd04499463e4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Apr 2014 17:52:08 +0000 Subject: [PATCH 153/177] Allow the xfer optimization to proceed if the DEFAULT on the very first column of the two tables is different. This is a refinement of the fix for ticket [f67b41381a]. FossilOrigin-Name: 349f483499dd685a8da94923b6bd810a52e5e236 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 6 ++++-- test/tkt-f67b41381a.test | 9 ++++++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 498632b350..3b524ffeaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\stransfering\srecords\sbetween\stables\sunless\sthe\sdefault\svalues\sfor\sall\scolumns\sare\sthe\ssame.\sFix\sfor\s[f67b41381a]. -D 2014-04-26T14:07:57.099 +C Allow\sthe\sxfer\soptimization\sto\sproceed\sif\sthe\sDEFAULT\son\sthe\svery\sfirst\ncolumn\sof\sthe\stwo\stables\sis\sdifferent.\s\sThis\sis\sa\srefinement\sof\sthe\nfix\sfor\sticket\s[f67b41381a]. +D 2014-04-26T17:52:08.640 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 08de23111439c650cac52861a5b80d10d9d40560 +F src/insert.c ab34bea5af4fee9f956a0805a32463fb7f674d00 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -906,7 +906,7 @@ F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 -F test/tkt-f67b41381a.test 18d3477b6fe82e6c5ec5e233d79642d7e6d0de77 +F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5ada136f43ce744ae8c349eff39838eb44611b6e -R 72b6179d3d1a0fd8f077c8cde285a253 -U dan -Z 7eecc39cd60ba5323d907c2532e97144 +P f8c4c495e6de1f124d205383d4bafa46accbff5c +R efe95b9fe528a9f472987911752f5742 +U drh +Z 366467002d461344ec8032510917cbba diff --git a/manifest.uuid b/manifest.uuid index 6aa5d7fb27..26e36016ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8c4c495e6de1f124d205383d4bafa46accbff5c \ No newline at end of file +349f483499dd685a8da94923b6bd810a52e5e236 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 3c77761b32..909bc27474 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1876,8 +1876,10 @@ static int xferOptimization( if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } - if( (pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) - || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)) + /* Default values for second and subsequent columns need to match. */ + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) ){ return 0; /* Default values must be the same for all columns */ } diff --git a/test/tkt-f67b41381a.test b/test/tkt-f67b41381a.test index d15d985342..1ddec988cd 100644 --- a/test/tkt-f67b41381a.test +++ b/test/tkt-f67b41381a.test @@ -30,8 +30,13 @@ foreach {tn tbls xfer} { 2 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b) } 0 3 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b DEFAULT 'x') } 1 4 { CREATE TABLE t1(a, b DEFAULT NULL); CREATE TABLE t2(a, b) } 0 - 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 0 + 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 6 { CREATE TABLE t1(a DEFAULT 1, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 + 7 { CREATE TABLE t1(a DEFAULT 1, b DEFAULT 1); + CREATE TABLE t2(a DEFAULT 3, b DEFAULT 1) } 1 + 8 { CREATE TABLE t1(a DEFAULT 1, b DEFAULT 1); + CREATE TABLE t2(a DEFAULT 3, b DEFAULT 3) } 0 + } { execsql { DROP TABLE t1; DROP TABLE t2 } @@ -46,5 +51,3 @@ foreach {tn tbls xfer} { } finish_test - - From 1f8bb4b0a84986977b1e4d6712b57cd266c04c95 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Apr 2014 19:23:14 +0000 Subject: [PATCH 154/177] Update requirements marks to fix typos in the requirements text. No changes to code. FossilOrigin-Name: f5a263658187250044afc1a74000e6f6962733ca --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/e_createtable.test | 5 +++-- test/e_fkey.test | 11 ++++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3b524ffeaf..fea1e4c44c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sxfer\soptimization\sto\sproceed\sif\sthe\sDEFAULT\son\sthe\svery\sfirst\ncolumn\sof\sthe\stwo\stables\sis\sdifferent.\s\sThis\sis\sa\srefinement\sof\sthe\nfix\sfor\sticket\s[f67b41381a]. -D 2014-04-26T17:52:08.640 +C Update\srequirements\smarks\sto\sfix\stypos\sin\sthe\srequirements\stext.\nNo\schanges\sto\scode. +D 2014-04-26T19:23:14.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -433,12 +433,12 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test ee95d48664503d40f6cc9ef4a7d03216188e2ada +F test/e_createtable.test ed82efcedc4b3656b27a5fcd12335cdb7e20eeee F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 F test/e_expr.test 5c71d183fbf519a4769fd2e2124afdc70b5b1f42 -F test/e_fkey.test 630597377549af579d34faaf64c6959a5a68ef76 +F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 1e44f84d2abe44d66e4fbf198be4b20e3cc724a0 F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f8c4c495e6de1f124d205383d4bafa46accbff5c -R efe95b9fe528a9f472987911752f5742 +P 349f483499dd685a8da94923b6bd810a52e5e236 +R 310b081215d2804c69537a1d1f588735 U drh -Z 366467002d461344ec8032510917cbba +Z 3788b514428a54594de7c9ac65d6befd diff --git a/manifest.uuid b/manifest.uuid index 26e36016ef..b7714159cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -349f483499dd685a8da94923b6bd810a52e5e236 \ No newline at end of file +f5a263658187250044afc1a74000e6f6962733ca \ No newline at end of file diff --git a/test/e_createtable.test b/test/e_createtable.test index 2cd63280df..42fc017e2a 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -884,9 +884,10 @@ do_execsql_test e_createtable-3.3.1 { ); } {} -# EVIDENCE-OF: R-10288-43169 For the purposes of the DEFAULT clause, an +# EVIDENCE-OF: R-36381-62919 For the purposes of the DEFAULT clause, an # expression is considered constant provided that it does not contain -# any sub-queries or string constants enclosed in double quotes. +# any sub-queries, column or table references, or string literals +# enclosed in double-quotes instead of single-quotes. # do_createtable_tests 3.4.1 -error { default value of column [x] is not constant diff --git a/test/e_fkey.test b/test/e_fkey.test index 921e967176..09756505c3 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -135,9 +135,9 @@ reset_db # # This also tests that foreign key constraints are disabled by default. # -# EVIDENCE-OF: R-59578-04990 Foreign key constraints are disabled by +# EVIDENCE-OF: R-44261-39702 Foreign key constraints are disabled by # default (for backwards compatibility), so must be enabled separately -# for each database connection separately. +# for each database connection. # drop_all_tables do_test e_fkey-4.1 { @@ -163,9 +163,10 @@ do_test e_fkey-4.2 { } {world} #------------------------------------------------------------------------- -# EVIDENCE-OF: R-15278-54456 The application can can also use a PRAGMA +# EVIDENCE-OF: R-08013-37737 The application can also use a PRAGMA # foreign_keys statement to determine if foreign keys are currently # enabled. + # # This also tests the example code in section 2 of foreignkeys.in. # @@ -2990,8 +2991,8 @@ if {[clang_sanitize_address]==0} { # The setting of the recursive_triggers pragma does not affect foreign # key actions. # -# EVIDENCE-OF: R-51769-32730 The PRAGMA recursive_triggers setting does -# not not affect the operation of foreign key actions. +# EVIDENCE-OF: R-44355-00270 The PRAGMA recursive_triggers setting does +# not affect the operation of foreign key actions. # foreach recursive_triggers_setting [list 0 1 ON OFF] { drop_all_tables From 2dd3cdcfab89b47ee8b47260d0cdf73801094018 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 20:21:14 +0000 Subject: [PATCH 155/177] Fix an sqlite3_stmt_status() problem caused by recent changs on this branch. FossilOrigin-Name: dee204092421a239f9f60ab83c3a5b3e24d1baea --- manifest | 15 +++++----- manifest.uuid | 2 +- src/where.c | 10 ++++--- test/cost.test | 68 ++++++++++++++++++++++++++++++++++++++++++++++ test/orderby5.test | 4 +-- 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 test/cost.test diff --git a/manifest b/manifest index afdc8889fe..c32bc38c65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. -D 2014-04-25T20:22:45.291 +C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. +D 2014-04-26T20:21:14.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,6 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -722,7 +723,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 -R 804d3aff8175381c892eb4eebbb86307 +P c51efaa5d29ee0a91b9e6a83a8dd82530670811a +R 95df5cff60bc1a7189139d237d77a9d7 U dan -Z 6dd1c2a2c429274ab108441094bd0963 +Z c6c47e040169e1387f5404e0d25c212f diff --git a/manifest.uuid b/manifest.uuid index 71ddf4b2d3..22512678d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file +dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 73d3a22b7c..6d924e0782 100644 --- a/src/where.c +++ b/src/where.c @@ -4147,14 +4147,15 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_ONEROW; } } - + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; }else if( eOp & (WO_GT|WO_GE) ){ testcase( eOp & WO_GT ); testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else if( (eOp & WO_ISNULL)==0 ){ + }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); testcase( eOp & WO_LE ); @@ -5225,9 +5226,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** scans instead? */ LogEst rScale, rSortCost; - assert( nOrderBy>0 ); + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ diff --git a/test/cost.test b/test/cost.test new file mode 100644 index 0000000000..8df32d8ed1 --- /dev/null +++ b/test/cost.test @@ -0,0 +1,68 @@ +# 2014-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cost + + +do_execsql_test 1.1 { + CREATE TABLE t3(id INTEGER PRIMARY KEY, b NOT NULL); + CREATE TABLE t4(c, d, e); + CREATE UNIQUE INDEX i3 ON t3(b); + CREATE UNIQUE INDEX i4 ON t4(c, d); +} +do_eqp_test 1.2 { + SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d; +} { + 0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} + 0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +# It is better to use an index for ORDER BY than sort externally, even +# if the index is a non-covering index. +do_eqp_test 2.2 { + SELECT * FROM t1 ORDER BY a; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} +} + +do_execsql_test 3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t5b ON t5(b); + CREATE INDEX t5c ON t5(c); + CREATE INDEX t5d ON t5(d); + CREATE INDEX t5e ON t5(e); + CREATE INDEX t5f ON t5(f); + CREATE INDEX t5g ON t5(g); +} + +do_eqp_test 3.2 { + SELECT a FROM t5 + WHERE b IS NULL OR c IS NULL OR d IS NULL + ORDER BY a; +} { + 0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + + +finish_test diff --git a/test/orderby5.test b/test/orderby5.test index bccd469f25..c9cce703ba 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -80,12 +80,12 @@ do_execsql_test 2.1a { EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; } {~/B-TREE/} + do_execsql_test 2.1b { EXPLAIN QUERY PLAN - SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; + SELECT * FROM t1 WHERE likelihood(a=0, 0.05) ORDER BY a, b, c; } {/B-TREE/} - do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; From 440e6ff3b023b66ecefa545990d5eeb85bda03ea Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 08:49:54 +0000 Subject: [PATCH 156/177] Fix an error in estimating of the number of rows visited by a range scan. FossilOrigin-Name: d491de62fce69d93e89f65f7713972f7c2c451f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 7 ++++++- test/cost.test | 27 +++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c32bc38c65..2a1d6b7303 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. -D 2014-04-26T20:21:14.140 +C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. +D 2014-04-28T08:49:54.584 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a +F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 +F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c51efaa5d29ee0a91b9e6a83a8dd82530670811a -R 95df5cff60bc1a7189139d237d77a9d7 +P dee204092421a239f9f60ab83c3a5b3e24d1baea +R 3b87cec4307de74fea71def903d4f518 U dan -Z c6c47e040169e1387f5404e0d25c212f +Z dce0ef4becf30f1f5b6dab0637076369 diff --git a/manifest.uuid b/manifest.uuid index 22512678d7..94e48b42ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file +d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6d924e0782..dd80ab4ccb 100644 --- a/src/where.c +++ b/src/where.c @@ -4241,7 +4241,12 @@ static int whereLoopAddBtreeIndex( pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = nOutUnadjusted; + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) diff --git a/test/cost.test b/test/cost.test index 8df32d8ed1..faaa7bdf16 100644 --- a/test/cost.test +++ b/test/cost.test @@ -63,6 +63,33 @@ do_eqp_test 3.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } +#------------------------------------------------------------------------- +# If there is no likelihood() or stat3 data, SQLite assumes that a closed +# range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) +# visits 1/16 of the rows in a table. +# +# Note: 1/17 =~ 0.058 +# Note: 1/15 =~ 0.067 +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); +} +do_eqp_test 4.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} +} +do_eqp_test 4.3 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b Date: Mon, 28 Apr 2014 09:35:31 +0000 Subject: [PATCH 157/177] Modify internal function whereLoopAdjustCost() so that it does not prefer a skip-scan over a regular index scan even if the regular scan uses a subset of the WHERE terms used by the skip-scan. FossilOrigin-Name: 88a5758dcce891eb7be15432ebdc9f80071d413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2a1d6b7303..b301a439d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-04-28T08:49:54.584 +C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. +D 2014-04-28T09:35:31.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b +F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dee204092421a239f9f60ab83c3a5b3e24d1baea -R 3b87cec4307de74fea71def903d4f518 +P d491de62fce69d93e89f65f7713972f7c2c451f7 +R a36e9b880c1d76e2d8cd1072dd5871be U dan -Z dce0ef4becf30f1f5b6dab0637076369 +Z 07995bb3734998fef6b8bca04739721a diff --git a/manifest.uuid b/manifest.uuid index 94e48b42ff..8be55d6794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file +88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file diff --git a/src/where.c b/src/where.c index dd80ab4ccb..4c1d81e183 100644 --- a/src/where.c +++ b/src/where.c @@ -3779,9 +3779,11 @@ static int whereLoopCheaperProperSubset( */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ From b51926e67c4921137cac5e3b66257a445f23ad5f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 10:00:59 +0000 Subject: [PATCH 158/177] Update test script analyze3.test to account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze3.test | 42 ++++++++++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b301a439d1..06eee5c7d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. -D 2014-04-28T09:35:31.541 +C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T10:00:59.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -305,7 +305,7 @@ F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 -F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d +F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d491de62fce69d93e89f65f7713972f7c2c451f7 -R a36e9b880c1d76e2d8cd1072dd5871be +P 88a5758dcce891eb7be15432ebdc9f80071d413b +R a1e1c4a3f11d62d0bb62f198394034d8 U dan -Z 07995bb3734998fef6b8bca04739721a +Z fd306a356911acfe444509c2efb0dade diff --git a/manifest.uuid b/manifest.uuid index 8be55d6794..5b247aefe3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file +35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file diff --git a/test/analyze3.test b/test/analyze3.test index fb26303ee3..e7416d5730 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -103,12 +103,21 @@ do_test analyze3-1.1.1 { } } {1} +do_execsql_test analyze3-1.1.x { + SELECT count(*) FROM t1 WHERE x>200 AND x<300; + SELECT count(*) FROM t1 WHERE x>0 AND x<1100; +} {99 1000} + +# The first of the following two SELECT statements visits 99 rows. So +# it is better to use the index. But the second visits every row in +# the table (1000 in total) so it is better to do a full-table scan. +# do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x200 AND x<300 } @@ -125,17 +134,17 @@ do_test analyze3-1.1.6 { } {199 0 14850} do_test analyze3-1.1.7 { sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} # The following tests are similar to the block above. The difference is @@ -152,12 +161,17 @@ do_test analyze3-1.2.1 { ANALYZE; } } {} +do_execsql_test analyze3-2.1.x { + SELECT count(*) FROM t2 WHERE x>1 AND x<2; + SELECT count(*) FROM t2 WHERE x>0 AND x<99; +} {200 990} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 } {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x12 AND x<20 } } {161 0 4760} @@ -173,17 +187,17 @@ do_test analyze3-1.2.6 { } {161 0 integer integer 4760} do_test analyze3-1.2.7 { sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 } -} {1981 0 490555} +} {999 999 490555} do_test analyze3-1.2.8 { set l [string range "0" 0 end] set u [string range "99" 0 end] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 text text 490555} +} {999 999 text text 490555} do_test analyze3-1.2.9 { set l [expr int(0)] set u [expr int(99)] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 integer integer 490555} +} {999 999 integer integer 490555} # Same tests a third time. This time, column x has INTEGER affinity and # is not the leftmost column of the table. This triggered a bug causing @@ -199,12 +213,16 @@ do_test analyze3-1.3.1 { ANALYZE; } } {} +do_execsql_test analyze3-1.3.x { + SELECT count(*) FROM t3 WHERE x>200 AND x<300; + SELECT count(*) FROM t3 WHERE x>0 AND x<1100 +} {99 1000} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x200 AND x<300 } @@ -221,17 +239,17 @@ do_test analyze3-1.3.6 { } {199 0 14850} do_test analyze3-1.3.7 { sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} #------------------------------------------------------------------------- # Test that the values of bound SQL variables may be used for the LIKE From 9881d60d17451055282b2bde12454fa42aeaabc8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 12:08:23 +0000 Subject: [PATCH 159/177] Add an extra column to a table in analyze9.test to give the planner a little more reason to select an index. FossilOrigin-Name: 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 06eee5c7d2..04a8c1374d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T10:00:59.075 +C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. +D 2014-04-28T12:08:23.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e +F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 88a5758dcce891eb7be15432ebdc9f80071d413b -R a1e1c4a3f11d62d0bb62f198394034d8 +P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 +R 8aefc1341eaf7a6ece9efb278712f5a6 U dan -Z fd306a356911acfe444509c2efb0dade +Z ba7221d22e50ff31dad54b0ec475658f diff --git a/manifest.uuid b/manifest.uuid index 5b247aefe3..7dba22fdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file +1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 125cecf182..f25e5924e6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -566,7 +566,7 @@ foreach {tn schema} { drop_all_tables do_test 13.1 { execsql { - CREATE TABLE t1(a, b, c); + CREATE TABLE t1(a, b, c, d); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b, c); } From 6b6828625b2843a719fb123dc46feb5666b2cda3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 15:11:25 +0000 Subject: [PATCH 160/177] Update unordered.test to take into account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 20f468dfbcb247e51446fad411a6e6cc0d130411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/unordered.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 04a8c1374d..05f3d798c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. -D 2014-04-28T12:08:23.821 +C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T15:11:25.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1023,7 +1023,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c +F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 -R 8aefc1341eaf7a6ece9efb278712f5a6 +P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 +R 307957877e44688fb3b83b922c0183e6 U dan -Z ba7221d22e50ff31dad54b0ec475658f +Z 085ae09cb0f2460f340245b5be0529cf diff --git a/manifest.uuid b/manifest.uuid index 7dba22fdc4..8fd0bd92ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file +20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file diff --git a/test/unordered.test b/test/unordered.test index d8f7aa6add..147e91f0d9 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -42,7 +42,7 @@ foreach idxmode {ordered unordered} { 1 "SELECT * FROM t1 ORDER BY a" {0 0 0 {SCAN TABLE t1 USING INDEX i1}} {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} - 2 "SELECT * FROM t1 WHERE a >?" + 2 "SELECT * FROM t1 WHERE a > 100" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" From 65e6b0dd1266cf5178b1e7b432cdd9165ef2a1f6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Apr 2014 17:56:19 +0000 Subject: [PATCH 161/177] Add the sqlite3_rtree_query_callback() API to the RTree virtual table. (Cherrypick from the sessions branch.) FossilOrigin-Name: af2cbe64adab5f9e3b0f3da00d06428088589d7f --- ext/rtree/rtree.c | 1496 ++++++++++++++++---------------- ext/rtree/rtree1.test | 19 +- ext/rtree/rtree6.test | 18 +- ext/rtree/rtreeB.test | 2 +- ext/rtree/rtreeC.test | 13 +- ext/rtree/rtreeE.test | 129 +++ ext/rtree/sqlite3rtree.h | 67 +- main.mk | 2 +- manifest | 33 +- manifest.uuid | 2 +- src/test_rtree.c | 200 ++++- src/test_vfstrace.c | 2 +- test/show_speedtest1_rtree.tcl | 57 ++ test/speedtest1.c | 186 +++- 14 files changed, 1425 insertions(+), 801 deletions(-) create mode 100644 ext/rtree/rtreeE.test create mode 100644 test/show_speedtest1_rtree.tcl diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index cefb9a8b2a..7b540b4be1 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -54,48 +54,6 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) -/* -** This file contains an implementation of a couple of different variants -** of the r-tree algorithm. See the README file for further details. The -** same data-structure is used for all, but the algorithms for insert and -** delete operations vary. The variants used are selected at compile time -** by defining the following symbols: -*/ - -/* Either, both or none of the following may be set to activate -** r*tree variant algorithms. -*/ -#define VARIANT_RSTARTREE_CHOOSESUBTREE 0 -#define VARIANT_RSTARTREE_REINSERT 1 - -/* -** Exactly one of the following must be set to 1. -*/ -#define VARIANT_GUTTMAN_QUADRATIC_SPLIT 0 -#define VARIANT_GUTTMAN_LINEAR_SPLIT 0 -#define VARIANT_RSTARTREE_SPLIT 1 - -#define VARIANT_GUTTMAN_SPLIT \ - (VARIANT_GUTTMAN_LINEAR_SPLIT||VARIANT_GUTTMAN_QUADRATIC_SPLIT) - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT - #define PickNext QuadraticPickNext - #define PickSeeds QuadraticPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_GUTTMAN_LINEAR_SPLIT - #define PickNext LinearPickNext - #define PickSeeds LinearPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_RSTARTREE_SPLIT - #define AssignCells splitNodeStartree -#endif - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - #ifndef SQLITE_CORE #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -105,11 +63,13 @@ #include #include +#include #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; +typedef unsigned short u16; typedef unsigned int u32; #endif @@ -127,6 +87,7 @@ typedef struct RtreeConstraint RtreeConstraint; typedef struct RtreeMatchArg RtreeMatchArg; typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -135,7 +96,7 @@ typedef union RtreeCoord RtreeCoord; ** ever contain very many entries, so a fixed number of buckets is ** used. */ -#define HASHSIZE 128 +#define HASHSIZE 97 /* The xBestIndex method of this virtual table requires an estimate of ** the number of rows in the virtual table to calculate the costs of @@ -151,15 +112,15 @@ typedef union RtreeCoord RtreeCoord; ** An rtree virtual-table object. */ struct Rtree { - sqlite3_vtab base; + sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ - int nDim; /* Number of dimensions */ - int nBytesPerCell; /* Bytes consumed per cell */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ @@ -186,10 +147,10 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - int eCoordType; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; -/* Possible values for eCoordType: */ +/* Possible values for Rtree.eCoordType: */ #define RTREE_COORD_REAL32 0 #define RTREE_COORD_INT32 1 @@ -201,11 +162,30 @@ struct Rtree { #ifdef SQLITE_RTREE_INT_ONLY typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 #else typedef double RtreeDValue; /* High accuracy coordinate */ typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 #endif +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + /* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: @@ -228,21 +208,44 @@ struct Rtree { */ #define RTREE_MAX_DEPTH 40 + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + /* ** An rtree cursor object. */ struct RtreeCursor { - sqlite3_vtab_cursor base; - RtreeNode *pNode; /* Node cursor is currently pointing at */ - int iCell; /* Index of current cell in pNode */ + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ }; +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ union RtreeCoord { - RtreeValue f; - int i; + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ }; /* @@ -267,38 +270,67 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - RtreeDValue rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 -#define RTREE_MATCH 0x46 +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + /* ** An rtree structure node. */ struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; - int nRef; - int isDirty; - u8 *zData; - RtreeNode *pNext; /* Next node in this hash chain */ + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ }; + +/* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) /* -** Structure to store a deserialized rtree record. +** A single cell from a node, deserialized */ struct RtreeCell { - i64 iRowid; - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; }; @@ -310,29 +342,16 @@ struct RtreeCell { #define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** An instance of this structure must be supplied as a blob argument to -** the right-hand-side of an SQL MATCH operator used to constrain an -** r-tree query. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *); - void *pContext; - int nParam; - RtreeDValue aParam[1]; -}; - -/* -** When a geometry callback is created (see sqlite3_rtree_geometry_callback), -** a single instance of the following structure is allocated. It is used -** as the context for the user-function created by by s_r_g_c(). The object -** is eventually deleted by the destructor mechanism provided by -** sqlite3_create_function_v2() (which is called by s_r_g_c() to create -** the geometry callback function). -*/ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - void *pContext; + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; #ifndef MAX @@ -426,10 +445,7 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** in the Rtree.aHash table. */ static int nodeHash(i64 iNode){ - return ( - (iNode>>56) ^ (iNode>>48) ^ (iNode>>40) ^ (iNode>>32) ^ - (iNode>>24) ^ (iNode>>16) ^ (iNode>> 8) ^ (iNode>> 0) - ) % HASHSIZE; + return iNode % HASHSIZE; } /* @@ -489,8 +505,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ /* ** Obtain a reference to an r-tree node. */ -static int -nodeAcquire( +static int nodeAcquire( Rtree *pRtree, /* R-tree structure */ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ @@ -579,10 +594,10 @@ nodeAcquire( ** Overwrite cell iCell of node pNode with the contents of pCell. */ static void nodeOverwriteCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ ){ int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -594,7 +609,7 @@ static void nodeOverwriteCell( } /* -** Remove cell the cell with index iCell from node pNode. +** Remove the cell with index iCell from node pNode. */ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -611,11 +626,10 @@ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ ** ** If there is not enough free space in pNode, return SQLITE_FULL. */ -static int -nodeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ int nCell; /* Current number of cells in pNode */ int nMaxCell; /* Maximum number of cells for pNode */ @@ -636,8 +650,7 @@ nodeInsertCell( /* ** If the node is dirty, write it out to the database. */ -static int -nodeWrite(Rtree *pRtree, RtreeNode *pNode){ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode->isDirty ){ sqlite3_stmt *p = pRtree->pWriteNode; @@ -662,8 +675,7 @@ nodeWrite(Rtree *pRtree, RtreeNode *pNode){ ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. */ -static int -nodeRelease(Rtree *pRtree, RtreeNode *pNode){ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); @@ -691,9 +703,9 @@ nodeRelease(Rtree *pRtree, RtreeNode *pNode){ ** an internal node, then the 64-bit integer is a child page number. */ static i64 nodeGetRowid( - Rtree *pRtree, - RtreeNode *pNode, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); @@ -703,11 +715,11 @@ static i64 nodeGetRowid( ** Return coordinate iCoord from cell iCell in node pNode. */ static void nodeGetCoord( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - int iCoord, - RtreeCoord *pCoord /* Space to write result to */ + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ ){ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -717,15 +729,20 @@ static void nodeGetCoord( ** to by pCell with the results. */ static void nodeGetCell( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - RtreeCell *pCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ ){ - int ii; + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - for(ii=0; iinDim*2; ii++){ - nodeGetCoord(pRtree, pNode, iCell, ii, &pCell->aCoord[ii]); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDataaConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ - sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; - if( pGeom ){ - if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); - sqlite3_free(pGeom); + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); } } sqlite3_free(pCsr->aConstraint); @@ -867,12 +884,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int rc; + int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; freeCursorConstraints(pCsr); - rc = nodeRelease(pRtree, pCsr->pNode); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); - return rc; + return SQLITE_OK; } /* @@ -883,196 +901,166 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; - return (pCsr->pNode==0); + return pCsr->atEOF; } /* -** The r-tree constraint passed as the second argument to this function is -** guaranteed to be a MATCH constraint. -*/ -static int testRtreeGeom( - Rtree *pRtree, /* R-Tree object */ - RtreeConstraint *pConstraint, /* MATCH constraint to test */ - RtreeCell *pCell, /* Cell to test */ - int *pbRes /* OUT: Test result */ -){ - int i; - RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2]; - int nCoord = pRtree->nDim*2; - - assert( pConstraint->op==RTREE_MATCH ); - assert( pConstraint->pGeom ); - - for(i=0; iaCoord[i]); - } - return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); -} - -/* -** Cursor pCursor currently points to a cell in a non-leaf page. -** Set *pbEof to true if the sub-tree headed by the cell is filtered -** (excluded) by the constraints in the pCursor->aConstraint[] -** array, or false otherwise. +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". ** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - int bRes = 0; - int rc = SQLITE_OK; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; bRes==0 && iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - - switch( p->op ){ - case RTREE_LE: case RTREE_LT: - bRes = p->rValuerValue>cell_max; - break; - - case RTREE_EQ: - bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &bRes); - bRes = !bRes; - break; - } - } - } - - *pbEof = bRes; - return rc; +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } - -/* -** Test if the cell that cursor pCursor currently points to -** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array. If so, set *pbEof to true before -** returning. If the cell is not filtered (excluded) by the constraints, -** set pbEof to zero. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -** -** This function assumes that the cell is part of a leaf node. -*/ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - *pbEof = 0; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]); - int res; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - switch( p->op ){ - case RTREE_LE: res = (coord<=p->rValue); break; - case RTREE_LT: res = (coordrValue); break; - case RTREE_GE: res = (coord>=p->rValue); break; - case RTREE_GT: res = (coord>p->rValue); break; - case RTREE_EQ: res = (coord==p->rValue); break; - default: { - int rc; - assert( p->op==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &res); - if( rc!=SQLITE_OK ){ - return rc; - } - break; - } - } - - if( !res ){ - *pbEof = 1; - return SQLITE_OK; - } - } - - return SQLITE_OK; +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif /* -** Cursor pCursor currently points at a node that heads a sub-tree of -** height iHeight (if iHeight==0, then the node is a leaf). Descend -** to point to the left-most cell of the sub-tree that matches the -** configured constraints. +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. */ -static int descendToCell( - Rtree *pRtree, - RtreeCursor *pCursor, - int iHeight, - int *pEof /* OUT: Set to true if cannot descend */ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ ){ - int isEof; - int rc; - int ii; - RtreeNode *pChild; - sqlite3_int64 iRowid; + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - RtreeNode *pSavedNode = pCursor->pNode; - int iSavedCell = pCursor->iCell; + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); - assert( iHeight>=0 ); - - if( iHeight==0 ){ - rc = testRtreeEntry(pRtree, pCursor, &isEof); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; }else{ - rc = testRtreeCell(pRtree, pCursor, &isEof); - } - if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - goto descend_to_cell_out; - } - - iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); - rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - - nodeRelease(pRtree, pCursor->pNode); - pCursor->pNode = pChild; - isEof = 1; - for(ii=0; isEof && iiiCell = ii; - rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - if( isEof ){ - assert( pCursor->pNode==pChild ); - nodeReference(pSavedNode); - nodeRelease(pRtree, pChild); - pCursor->pNode = pSavedNode; - pCursor->iCell = iSavedCell; - } - -descend_to_cell_out: - *pEof = isEof; return rc; } +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + /* ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. @@ -1085,6 +1073,7 @@ static int nodeRowidIndex( ){ int ii; int nCell = NCELL(pNode); + assert( nCell<200 ); for(ii=0; iirScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + /* ** Rtree virtual table module xNext method. */ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ - Rtree *pRtree = (Rtree *)(pVtabCursor->pVtab); RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; - /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is - ** already at EOF. It is against the rules to call the xNext() method of - ** a cursor that has already reached EOF. - */ - assert( pCsr->pNode ); - - if( pCsr->iStrategy==1 ){ - /* This "scan" is a direct lookup by rowid. There is no next entry. */ - nodeRelease(pRtree, pCsr->pNode); - pCsr->pNode = 0; - }else{ - /* Move to the next entry that matches the configured constraints. */ - int iHeight = 0; - while( pCsr->pNode ){ - RtreeNode *pNode = pCsr->pNode; - int nCell = NCELL(pNode); - for(pCsr->iCell++; pCsr->iCelliCell++){ - int isEof; - rc = descendToCell(pRtree, pCsr, iHeight, &isEof); - if( rc!=SQLITE_OK || !isEof ){ - return rc; - } - } - pCsr->pNode = pNode->pParent; - rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); - if( rc!=SQLITE_OK ){ - return rc; - } - nodeReference(pCsr->pNode); - nodeRelease(pRtree, pNode); - iHeight++; - } - } - + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); return rc; } @@ -1156,13 +1399,14 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - - assert(pCsr->pNode); - *pRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - - return SQLITE_OK; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; } /* @@ -1171,13 +1415,18 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ Rtree *pRtree = (Rtree *)cur->pVtab; RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; if( i==0 ){ - i64 iRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - sqlite3_result_int64(ctx, iRowid); + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - RtreeCoord c; - nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); @@ -1188,7 +1437,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ sqlite3_result_int(ctx, c.i); } } - return SQLITE_OK; } @@ -1199,12 +1447,18 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf ** to zero and return an SQLite error code. */ -static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ int rc; *ppLeaf = 0; sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); sqlite3_reset(pRtree->pReadRowid); }else{ @@ -1220,9 +1474,10 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *p; - sqlite3_rtree_geometry *pGeom; - int nBlob; + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ /* Check that value is actually a blob. */ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; @@ -1235,27 +1490,29 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_ERROR; } - pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( - sizeof(sqlite3_rtree_geometry) + nBlob - ); - if( !pGeom ) return SQLITE_NOMEM; - memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); - p = (RtreeMatchArg *)&pGeom[1]; + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; - memcpy(p, sqlite3_value_blob(pValue), nBlob); - if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue)) - ){ - sqlite3_free(pGeom); + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); return SQLITE_ERROR; } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; - pGeom->pContext = p->pContext; - pGeom->nParam = p->nParam; - pGeom->aParam = p->aParam; - - pCons->xGeom = p->xGeom; - pCons->pGeom = pGeom; + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; return SQLITE_OK; } @@ -1269,10 +1526,10 @@ static int rtreeFilter( ){ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; int ii; int rc = SQLITE_OK; + int iCell = 0; rtreeReference(pRtree); @@ -1282,31 +1539,42 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); - rc = findLeafNode(pRtree, iRowid, &pLeaf); - pCsr->pNode = pLeaf; - if( pLeaf ){ - assert( rc==SQLITE_OK ); - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ - if( argc>0 ){ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); assert( (idxStr==0 && argc==0) || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'a'; - if( p->op==RTREE_MATCH ){ + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ /* A MATCH operator. The right-hand-side must be a blob that ** can be cast into an RtreeMatchArg object. One created using ** an sqlite3_rtree_geometry_callback() SQL user function. @@ -1315,41 +1583,35 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ #ifdef SQLITE_RTREE_INT_ONLY - p->rValue = sqlite3_value_int64(argv[ii]); + p->u.rValue = sqlite3_value_int64(argv[ii]); #else - p->rValue = sqlite3_value_double(argv[ii]); + p->u.rValue = sqlite3_value_double(argv[ii]); #endif } } } } - if( rc==SQLITE_OK ){ - pCsr->pNode = 0; - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - } - if( rc==SQLITE_OK ){ - int isEof = 1; - int nCell = NCELL(pRoot); - pCsr->pNode = pRoot; - for(pCsr->iCell=0; rc==SQLITE_OK && pCsr->iCelliCell++){ - assert( pCsr->pNode==pRoot ); - rc = descendToCell(pRtree, pCsr, pRtree->iDepth, &isEof); - if( !isEof ){ - break; - } - } - if( rc==SQLITE_OK && isEof ){ - assert( pCsr->pNode==pRoot ); - nodeRelease(pRtree, pRoot); - pCsr->pNode = 0; - } - assert( rc!=SQLITE_OK || !pCsr->pNode || pCsr->iCellpNode) ); + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } + nodeRelease(pRtree, pRoot); rtreeRelease(pRtree); return rc; } @@ -1451,7 +1713,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -1544,62 +1806,32 @@ static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ return (cellArea(pRtree, &cell)-area); } -#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, - int nCell, - int iExclude + int nCell ){ int ii; - RtreeDValue overlap = 0.0; + RtreeDValue overlap = RTREE_ZERO; for(ii=0; iinDim*2); jj+=2){ - RtreeDValue x1, x2; - - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - - if( x2nDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-1) ){ - int jj; - aCell = sqlite3_malloc(sizeof(RtreeCell)*nCell); - if( !aCell ){ - rc = SQLITE_NOMEM; - nodeRelease(pRtree, pNode); - pNode = 0; - continue; - } - for(jj=0; jjiDepth-1) ){ - overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); - }else{ - overlap = 0.0; - } - if( (iCell==0) - || (overlapnDim; i++){ - RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]); - RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]); - RtreeDValue x3 = x1; - RtreeDValue x4 = x2; - int jj; - - int iCellLeft = 0; - int iCellRight = 0; - - for(jj=1; jjx4 ) x4 = right; - if( left>x3 ){ - x3 = left; - iCellRight = jj; - } - if( rightmaxNormalInnerWidth ){ - iLeftSeed = iCellLeft; - iRightSeed = iCellRight; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_LINEAR_SPLIT */ - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT -/* -** Implementation of the quadratic variant of the PickNext() function from -** Guttman[84]. -*/ -static RtreeCell *QuadraticPickNext( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeCell *pLeftBox, - RtreeCell *pRightBox, - int *aiUsed -){ - #define FABS(a) ((a)<0.0?-1.0*(a):(a)) - - int iSelect = -1; - RtreeDValue fDiff; - int ii; - for(ii=0; iifDiff ){ - fDiff = diff; - iSelect = ii; - } - } - } - aiUsed[iSelect] = 1; - return &aCell[iSelect]; -} - -/* -** Implementation of the quadratic variant of the PickSeeds() function from -** Guttman[84]. -*/ -static void QuadraticPickSeeds( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - int *piLeftSeed, - int *piRightSeed -){ - int ii; - int jj; - - int iLeftSeed = 0; - int iRightSeed = 1; - RtreeDValue fWaste = 0.0; - - for(ii=0; iifWaste ){ - iLeftSeed = ii; - iRightSeed = jj; - fWaste = waste; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_QUADRATIC_SPLIT */ /* ** Arguments aIdx, aDistance and aSpare all point to arrays of size @@ -2040,7 +2086,6 @@ static void SortByDimension( } } -#if VARIANT_RSTARTREE_SPLIT /* ** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. */ @@ -2059,7 +2104,7 @@ static int splitNodeStartree( int iBestDim = 0; int iBestSplit = 0; - RtreeDValue fBestMargin = 0.0; + RtreeDValue fBestMargin = RTREE_ZERO; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -2080,9 +2125,9 @@ static int splitNodeStartree( } for(ii=0; iinDim; ii++){ - RtreeDValue margin = 0.0; - RtreeDValue fBestOverlap = 0.0; - RtreeDValue fBestArea = 0.0; + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; int iBestLeft = 0; int nLeft; @@ -2108,7 +2153,7 @@ static int splitNodeStartree( } margin += cellMargin(pRtree, &left); margin += cellMargin(pRtree, &right); - overlap = cellOverlap(pRtree, &left, &right, 1, -1); + overlap = cellOverlap(pRtree, &left, &right, 1); area = cellArea(pRtree, &left) + cellArea(pRtree, &right); if( (nLeft==RTREE_MINCELLS(pRtree)) || (overlap0; i--){ - RtreeCell *pNext; - pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - RtreeDValue diff = - cellGrowth(pRtree, pBboxLeft, pNext) - - cellGrowth(pRtree, pBboxRight, pNext) - ; - if( (RTREE_MINCELLS(pRtree)-NCELL(pRight)==i) - || (diff>0.0 && (RTREE_MINCELLS(pRtree)-NCELL(pLeft)!=i)) - ){ - nodeInsertCell(pRtree, pRight, pNext); - cellUnion(pRtree, pBboxRight, pNext); - }else{ - nodeInsertCell(pRtree, pLeft, pNext); - cellUnion(pRtree, pBboxLeft, pNext); - } - } - - sqlite3_free(aiUsed); - return SQLITE_OK; -} -#endif static int updateMapping( Rtree *pRtree, @@ -2274,7 +2263,8 @@ static int SplitNode( memset(pLeft->zData, 0, pRtree->iNodeSize); memset(pRight->zData, 0, pRtree->iNodeSize); - rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } @@ -2557,7 +2547,7 @@ static int Reinsert( } for(ii=0; iinDim; iDim++){ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - DCOORD(aCell[ii].aCoord[iDim*2])); @@ -2623,16 +2613,12 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ -#if VARIANT_RSTARTREE_REINSERT if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ pRtree->iReinsertHeight = iHeight; rc = Reinsert(pRtree, pNode, pCell, iHeight); } -#else - rc = SplitNode(pRtree, pNode, pCell, iHeight); -#endif }else{ rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ @@ -2702,7 +2688,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ ** about to be deleted. */ if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } /* Delete the cell in question from the leaf node. */ @@ -3039,7 +3025,8 @@ static int rtreeSqlInit( char *zCreate = sqlite3_mprintf( "CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY, parentnode INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" "INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize ); @@ -3253,10 +3240,10 @@ static int rtreeInit( ** Implementation of a scalar function that decodes r-tree nodes to ** human readable strings. This can be used for debugging and analysis. ** -** The scalar function takes two arguments, a blob of data containing -** an r-tree node, and the number of dimensions the r-tree indexes. -** For a two-dimensional r-tree structure called "rt", to deserialize -** all nodes, a statement like: +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** ** SELECT rtreenode(2, data) FROM rt_node; ** @@ -3289,7 +3276,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nCell = (int)strlen(zCell); for(jj=0; jjxDestructor ) pInfo->xDestructor(pInfo->pContext); sqlite3_free(p); } /* -** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite -** scalar user function. This C function is the callback used for all such -** registered SQL functions. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. ** -** The scalar user functions return a blob that is interpreted by r-tree -** table MATCH operators. +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); @@ -3381,8 +3386,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ }else{ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->xGeom = pGeomCtx->xGeom; - pBlob->pContext = pGeomCtx->pContext; + pBlob->cb = pGeomCtx[0]; pBlob->nParam = nArg; for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); #endif } - sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); } } @@ -3399,10 +3403,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ ** Register a new geometry function for use with the r-tree MATCH operator. */ int sqlite3_rtree_geometry_callback( - sqlite3 *db, - const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *), - void *pContext + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ @@ -3410,12 +3414,36 @@ int sqlite3_rtree_geometry_callback( pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); if( !pGeomCtx ) return SQLITE_NOMEM; pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - - /* Create the new user-function. Register a destructor function to delete - ** the context object when it is no longer required. */ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 275b13264f..9de5362781 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -120,12 +120,13 @@ proc execsql_intout {sql} { # Test that it is possible to open an existing database that contains # r-tree tables. # -do_test rtree-1.4.1 { - execsql { - CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); - INSERT INTO t1 VALUES(1, 5.0, 10.0); - INSERT INTO t1 VALUES(2, 15.0, 20.0); - } +do_execsql_test rtree-1.4.1a { + CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); + INSERT INTO t1 VALUES(1, 5.0, 10.0); + SELECT substr(hex(data),1,40) FROM t1_node; +} {00000001000000000000000140A0000041200000} +do_execsql_test rtree-1.4.1b { + INSERT INTO t1 VALUES(2, 15.0, 20.0); } {} do_test rtree-1.4.2 { db close @@ -435,16 +436,18 @@ do_test rtree-11.2 { # Test on-conflict clause handling. # db_delete_and_reopen -do_execsql_test 12.0 { +do_execsql_test 12.0.1 { CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2); INSERT INTO t1 VALUES(1, 1, 2, 3, 4); + SELECT substr(hex(data),1,56) FROM t1_node; +} {00000001000000000000000100000001000000020000000300000004} +do_execsql_test 12.0.2 { INSERT INTO t1 VALUES(2, 2, 3, 4, 5); INSERT INTO t1 VALUES(3, 3, 4, 5, 6); CREATE TABLE source(idx, x1, x2, y1, y2); INSERT INTO source VALUES(5, 8, 8, 8, 8); INSERT INTO source VALUES(2, 7, 7, 7, 7); - } db_save_and_close foreach {tn sql_template testdata} { diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index bdc9bb4146..c5a78bb3e3 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -57,31 +57,31 @@ do_test rtree6-1.1 { do_test rtree6-1.2 { rtree_strategy {SELECT * FROM t1 WHERE x1>10} -} {Ea} +} {E0} do_test rtree6-1.3 { rtree_strategy {SELECT * FROM t1 WHERE x1<10} -} {Ca} +} {C0} do_test rtree6-1.4 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=ii AND x1<10} -} {Ca} +} {C0} do_test rtree6-1.5 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10} -} {Ca} +} {C0} do_eqp_test rtree6.2.1 { SELECT * FROM t1,t2 WHERE k=+ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.2 { SELECT * FROM t1,t2 WHERE k=ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } @@ -95,7 +95,7 @@ do_eqp_test rtree6.2.3 { do_eqp_test rtree6.2.4 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} } @@ -126,7 +126,7 @@ do_test rtree6.3.2 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_test rtree6.3.3 { rtree_strategy { SELECT * FROM t3 WHERE @@ -137,7 +137,7 @@ do_test rtree6.3.3 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_execsql_test rtree6-3.4 { SELECT * FROM t3 WHERE x1>0.5 AND x1>0.8 AND x1>1.1 diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test index 7cb445cc4f..aeb308eca7 100644 --- a/ext/rtree/rtreeB.test +++ b/ext/rtree/rtreeB.test @@ -41,7 +41,7 @@ ifcapable rtree_int_only { INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); SELECT rtreenode(2, data) FROM t1_node; } - } {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}} + } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}} } finish_test diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index 23dc607841..94db05a4d1 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -29,7 +29,7 @@ do_eqp_test 1.1 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 1 {SCAN TABLE t} - 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.2 { @@ -37,7 +37,7 @@ do_eqp_test 1.2 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.3 { @@ -45,7 +45,7 @@ do_eqp_test 1.3 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.5 { @@ -82,7 +82,7 @@ do_eqp_test 2.1 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 1 {SCAN TABLE t} - 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.2 { @@ -90,7 +90,7 @@ do_eqp_test 2.2 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.3 { @@ -98,7 +98,7 @@ do_eqp_test 2.3 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.5 { @@ -271,4 +271,3 @@ ifcapable rtree { finish_test - diff --git a/ext/rtree/rtreeE.test b/ext/rtree/rtreeE.test new file mode 100644 index 0000000000..c450623790 --- /dev/null +++ b/ext/rtree/rtreeE.test @@ -0,0 +1,129 @@ +# 2010 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. Specifically, it tests +# that new-style custom r-tree queries (geometry callbacks) work. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } +ifcapable rtree_int_only { finish_test; return } + + +#------------------------------------------------------------------------- +# Test the example 2d "circle" geometry callback. +# +register_circle_geom db + +do_execsql_test rtreeE-1.1 { + PRAGMA page_size=512; + CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1); + + /* A tight pattern of small boxes near 0,0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT x+5*y, x, x+2, y, y+2 FROM x, y; + + /* A looser pattern of small boxes near 100, 0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 100+x+5*y, x*3+100, x*3+102, y*3, y*3+2 FROM x, y; + + /* A looser pattern of larger boxes near 0, 200 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 200+x+5*y, x*7, x*7+15, y*7+200, y*7+215 FROM x, y; +} {} + +# Queries against each of the three clusters */ +do_execsql_test rtreeE-1.1 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 0.0, 50.0, 3) ORDER BY id; +} {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24} +do_execsql_test rtreeE-1.2 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(100.0, 0.0, 50.0, 3) ORDER BY id; +} {100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124} +do_execsql_test rtreeE-1.3 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 200.0, 50.0, 3) ORDER BY id; +} {200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224} + +# The Qcircle geometry function gives a lower score to larger leaf-nodes. +# This causes the 200s to sort before the 100s and the 0s to sort before +# last. +# +do_execsql_test rtreeE-1.4 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,3) AND id%100==0 +} {200 100 0} + +# Exclude odd rowids on a depth-first search +do_execsql_test rtreeE-1.5 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,4) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Exclude odd rowids on a breadth-first search. +do_execsql_test rtreeE-1.6 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,5) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Construct a large 2-D RTree with thousands of random entries. +# +do_test rtreeE-2.1 { + db eval { + CREATE TABLE t2(id,x0,x1,y0,y1); + CREATE VIRTUAL TABLE rt2 USING rtree(id,x0,x1,y0,y1); + BEGIN; + } + expr srand(0) + for {set i 1} {$i<=10000} {incr i} { + set dx [expr {int(rand()*40)+1}] + set dy [expr {int(rand()*40)+1}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set id [expr {$i+10000}] + db eval {INSERT INTO t2 VALUES($id,$x0,$x1,$y0,$y1)} + } + db eval { + INSERT INTO rt2 SELECT * FROM t2; + COMMIT; + } +} {} + +for {set i 1} {$i<=200} {incr i} { + set dx [expr {int(rand()*100)}] + set dy [expr {int(rand()*100)}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set ans [db eval {SELECT id FROM t2 WHERE x1>=$x0 AND x0<=$x1 AND y1>=$y0 AND y0<=$y1 ORDER BY id}] + do_execsql_test rtreeE-2.2.$i { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch($x0,$x1,$y0,$y1) ORDER BY id + } $ans +} + +# Run query that have very deep priority queues +# +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=5000 AND y1>=0 AND y0<=5000 ORDER BY id}] +do_execsql_test rtreeE-2.3 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,5000,0,5000) ORDER BY id +} $ans +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=10000 AND y1>=0 AND y0<=10000 ORDER BY id}] +do_execsql_test rtreeE-2.4 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,10000,0,10000) ORDER BY id +} $ans + +finish_test diff --git a/ext/rtree/sqlite3rtree.h b/ext/rtree/sqlite3rtree.h index c849091f29..5de0508d00 100644 --- a/ext/rtree/sqlite3rtree.h +++ b/ext/rtree/sqlite3rtree.h @@ -21,6 +21,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -31,11 +41,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, -#ifdef SQLITE_RTREE_INT_ONLY - int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), -#else - int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), -#endif + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -47,11 +53,60 @@ int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #ifdef __cplusplus } /* end of the 'extern "C"' block */ diff --git a/main.mk b/main.mk index ebd508a279..fcc5c03db7 100644 --- a/main.mk +++ b/main.mk @@ -476,7 +476,7 @@ parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION +sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c diff --git a/manifest b/manifest index fea1e4c44c..41c2bd4ae0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\srequirements\smarks\sto\sfix\stypos\sin\sthe\srequirements\stext.\nNo\schanges\sto\scode. -D 2014-04-26T19:23:14.120 +C Add\sthe\ssqlite3_rtree_query_callback()\sAPI\sto\sthe\sRTree\svirtual\stable.\n(Cherrypick\sfrom\sthe\ssessions\sbranch.) +D 2014-04-28T17:56:19.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,30 +120,31 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de +F ext/rtree/rtree.c 6f70db93e0e42c369325c5cddcf2024c5a87ca43 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 +F ext/rtree/rtree1.test e2da4aaa426918d27122d1a1066c6ecf8409a514 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test fe0bd377a21c68ce2826129d14354c884cb1f354 +F ext/rtree/rtree6.test 756585abc51727fec97c77852476445c10c0ee95 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf -F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test 16d7aa86ecb6a876d2a38cf590a1471a41b3a46d +F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e +F ext/rtree/rtreeC.test df158dcc81f1a43ce7eef361af03c48ec91f1e06 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca +F ext/rtree/rtreeE.test 388c1c8602c3ce55c15f03b509e9cf545fb7c41f F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea -F ext/rtree/sqlite3rtree.h c34c1e41d1ab80bb8ad09aae402c9c956871a765 +F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 3ae543fa446525c1dec55f58de67f41b78651812 +F main.mk 9546867b42992c554e7af8672549ba13afaadade F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -260,7 +261,7 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 30c64f0ef84734f2231a686df41ed882b0c59bc0 F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_rtree.c f3d1d12538dccb75fd916e3fa58f250edbdd3b47 +F src/test_rtree.c fdd8d29ca5165c7857987a2ba263fac5c69e231f F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e @@ -270,7 +271,7 @@ F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 -F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 +F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb @@ -811,6 +812,7 @@ F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 F test/shell5.test bb755ea9144b8078a752fc56223582627070b5f1 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 +F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f @@ -826,7 +828,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 +F test/speedtest1.c d29c8048beb7ea9254191f3fde9414709166a920 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1163,7 +1165,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 349f483499dd685a8da94923b6bd810a52e5e236 -R 310b081215d2804c69537a1d1f588735 +P f5a263658187250044afc1a74000e6f6962733ca +Q +3dca2809352c6c6d56db74447a814f77011c6220 +R 70a04a84bf76743284b12f147604df6e U drh -Z 3788b514428a54594de7c9ac65d6befd +Z 8441eee0dd2b010346629b18a46aad71 diff --git a/manifest.uuid b/manifest.uuid index b7714159cf..fbb0a3e2f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5a263658187250044afc1a74000e6f6962733ca \ No newline at end of file +af2cbe64adab5f9e3b0f3da00d06428088589d7f \ No newline at end of file diff --git a/src/test_rtree.c b/src/test_rtree.c index e1966c2437..9d19fa0e2c 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -35,6 +35,8 @@ struct Circle { double centerx; double centery; double radius; + double mxArea; + int eScoreType; }; /* @@ -50,11 +52,7 @@ static void circle_del(void *p){ static int circle_geom( sqlite3_rtree_geometry *p, int nCoord, -#ifdef SQLITE_RTREE_INT_ONLY - sqlite3_int64 *aCoord, -#else - double *aCoord, -#endif + sqlite3_rtree_dbl *aCoord, int *pRes ){ int i; /* Iterator variable */ @@ -62,7 +60,12 @@ static int circle_geom( double xmin, xmax; /* X dimensions of box being tested */ double ymin, ymax; /* X dimensions of box being tested */ - if( p->pUser==0 ){ + xmin = aCoord[0]; + xmax = aCoord[1]; + ymin = aCoord[2]; + ymax = aCoord[3]; + pCircle = (Circle *)p->pUser; + if( pCircle==0 ){ /* If pUser is still 0, then the parameter values have not been tested ** for correctness or stored into a Circle structure yet. Do this now. */ @@ -108,14 +111,9 @@ static int circle_geom( pCircle->aBox[1].xmax = pCircle->centerx - pCircle->radius; pCircle->aBox[1].ymin = pCircle->centery; pCircle->aBox[1].ymax = pCircle->centery; + pCircle->mxArea = (xmax - xmin)*(ymax - ymin) + 1.0; } - pCircle = (Circle *)p->pUser; - xmin = aCoord[0]; - xmax = aCoord[1]; - ymin = aCoord[2]; - ymax = aCoord[3]; - /* Check if any of the 4 corners of the bounding-box being tested lie ** inside the circular region. If they do, then the bounding-box does ** intersect the region of interest. Set the output variable to true and @@ -154,6 +152,170 @@ static int circle_geom( return SQLITE_OK; } +/* +** Implementation of "circle" r-tree geometry callback using the +** 2nd-generation interface that allows scoring. +*/ +static int circle_query_func(sqlite3_rtree_query_info *p){ + int i; /* Iterator variable */ + Circle *pCircle; /* Structure defining circular region */ + double xmin, xmax; /* X dimensions of box being tested */ + double ymin, ymax; /* X dimensions of box being tested */ + int nWithin = 0; /* Number of corners inside the circle */ + + xmin = p->aCoord[0]; + xmax = p->aCoord[1]; + ymin = p->aCoord[2]; + ymax = p->aCoord[3]; + pCircle = (Circle *)p->pUser; + if( pCircle==0 ){ + /* If pUser is still 0, then the parameter values have not been tested + ** for correctness or stored into a Circle structure yet. Do this now. */ + + /* This geometry callback is for use with a 2-dimensional r-tree table. + ** Return an error if the table does not have exactly 2 dimensions. */ + if( p->nCoord!=4 ) return SQLITE_ERROR; + + /* Test that the correct number of parameters (4) have been supplied, + ** and that the parameters are in range (that the radius of the circle + ** radius is greater than zero). */ + if( p->nParam!=4 || p->aParam[2]<0.0 ) return SQLITE_ERROR; + + /* Allocate a structure to cache parameter data in. Return SQLITE_NOMEM + ** if the allocation fails. */ + pCircle = (Circle *)(p->pUser = sqlite3_malloc(sizeof(Circle))); + if( !pCircle ) return SQLITE_NOMEM; + p->xDelUser = circle_del; + + /* Record the center and radius of the circular region. One way that + ** tested bounding boxes that intersect the circular region are detected + ** is by testing if each corner of the bounding box lies within radius + ** units of the center of the circle. */ + pCircle->centerx = p->aParam[0]; + pCircle->centery = p->aParam[1]; + pCircle->radius = p->aParam[2]; + pCircle->eScoreType = (int)p->aParam[3]; + + /* Define two bounding box regions. The first, aBox[0], extends to + ** infinity in the X dimension. It covers the same range of the Y dimension + ** as the circular region. The second, aBox[1], extends to infinity in + ** the Y dimension and is constrained to the range of the circle in the + ** X dimension. + ** + ** Then imagine each box is split in half along its short axis by a line + ** that intersects the center of the circular region. A bounding box + ** being tested can be said to intersect the circular region if it contains + ** points from each half of either of the two infinite bounding boxes. + */ + pCircle->aBox[0].xmin = pCircle->centerx; + pCircle->aBox[0].xmax = pCircle->centerx; + pCircle->aBox[0].ymin = pCircle->centery + pCircle->radius; + pCircle->aBox[0].ymax = pCircle->centery - pCircle->radius; + pCircle->aBox[1].xmin = pCircle->centerx + pCircle->radius; + pCircle->aBox[1].xmax = pCircle->centerx - pCircle->radius; + pCircle->aBox[1].ymin = pCircle->centery; + pCircle->aBox[1].ymax = pCircle->centery; + pCircle->mxArea = 200.0*200.0; + } + + /* Check if any of the 4 corners of the bounding-box being tested lie + ** inside the circular region. If they do, then the bounding-box does + ** intersect the region of interest. Set the output variable to true and + ** return SQLITE_OK in this case. */ + for(i=0; i<4; i++){ + double x = (i&0x01) ? xmax : xmin; + double y = (i&0x02) ? ymax : ymin; + double d2; + + d2 = (x-pCircle->centerx)*(x-pCircle->centerx); + d2 += (y-pCircle->centery)*(y-pCircle->centery); + if( d2<(pCircle->radius*pCircle->radius) ) nWithin++; + } + + /* Check if the bounding box covers any other part of the circular region. + ** See comments above for a description of how this test works. If it does + ** cover part of the circular region, set the output variable to true + ** and return SQLITE_OK. */ + if( nWithin==0 ){ + for(i=0; i<2; i++){ + if( xmin<=pCircle->aBox[i].xmin + && xmax>=pCircle->aBox[i].xmax + && ymin<=pCircle->aBox[i].ymin + && ymax>=pCircle->aBox[i].ymax + ){ + nWithin = 1; + break; + } + } + } + + if( pCircle->eScoreType==1 ){ + /* Depth first search */ + p->rScore = p->iLevel; + }else if( pCircle->eScoreType==2 ){ + /* Breadth first search */ + p->rScore = 100 - p->iLevel; + }else if( pCircle->eScoreType==3 ){ + /* Depth-first search, except sort the leaf nodes by area with + ** the largest area first */ + if( p->iLevel==1 ){ + p->rScore = 1.0 - (xmax-xmin)*(ymax-ymin)/pCircle->mxArea; + if( p->rScore<0.01 ) p->rScore = 0.01; + }else{ + p->rScore = 0.0; + } + }else if( pCircle->eScoreType==4 ){ + /* Depth-first search, except exclude odd rowids */ + p->rScore = p->iLevel; + if( p->iRowid&1 ) nWithin = 0; + }else{ + /* Breadth-first search, except exclude odd rowids */ + p->rScore = 100 - p->iLevel; + if( p->iRowid&1 ) nWithin = 0; + } + if( nWithin==0 ){ + p->eWithin = NOT_WITHIN; + }else if( nWithin>=4 ){ + p->eWithin = FULLY_WITHIN; + }else{ + p->eWithin = PARTLY_WITHIN; + } + return SQLITE_OK; +} +/* +** Implementation of "breadthfirstsearch" r-tree geometry callback using the +** 2nd-generation interface that allows scoring. +** +** ... WHERE id MATCH breadthfirstsearch($x0,$x1,$y0,$y1) ... +** +** It returns all entries whose bounding boxes overlap with $x0,$x1,$y0,$y1. +*/ +static int bfs_query_func(sqlite3_rtree_query_info *p){ + double x0,x1,y0,y1; /* Dimensions of box being tested */ + double bx0,bx1,by0,by1; /* Boundary of the query function */ + + if( p->nParam!=4 ) return SQLITE_ERROR; + x0 = p->aCoord[0]; + x1 = p->aCoord[1]; + y0 = p->aCoord[2]; + y1 = p->aCoord[3]; + bx0 = p->aParam[0]; + bx1 = p->aParam[1]; + by0 = p->aParam[2]; + by1 = p->aParam[3]; + p->rScore = 100 - p->iLevel; + if( p->eParentWithin==FULLY_WITHIN ){ + p->eWithin = FULLY_WITHIN; + }else if( x0>=bx0 && x1<=bx1 && y0>=by0 && y1<=by1 ){ + p->eWithin = FULLY_WITHIN; + }else if( x1>=bx0 && x0<=bx1 && y1>=by0 && y0<=by1 ){ + p->eWithin = PARTLY_WITHIN; + }else{ + p->eWithin = NOT_WITHIN; + } + return SQLITE_OK; +} + /* END of implementation of "circle" geometry callback. ************************************************************************** *************************************************************************/ @@ -194,11 +356,7 @@ static int gHere = 42; static int cube_geom( sqlite3_rtree_geometry *p, int nCoord, -#ifdef SQLITE_RTREE_INT_ONLY - sqlite3_int64 *aCoord, -#else - double *aCoord, -#endif + sqlite3_rtree_dbl *aCoord, int *piRes ){ Cube *pCube = (Cube *)p->pUser; @@ -293,6 +451,14 @@ static int register_circle_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "circle", circle_geom, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_rtree_query_callback(db, "Qcircle", + circle_query_func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_rtree_query_callback(db, "breadthfirstsearch", + bfs_query_func, 0, 0); + } Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index c1792f116e..d0bc29f0c3 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -678,7 +678,7 @@ static int vfstraceAccess( vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; sqlite3_vfs *pRoot = pInfo->pRootVfs; int rc; - vfstrace_printf(pInfo, "%s.xDelete(\"%s\",%d)", + vfstrace_printf(pInfo, "%s.xAccess(\"%s\",%d)", pInfo->zVfsName, zPath, flags); rc = pRoot->xAccess(pRoot, zPath, flags, pResOut); vfstrace_print_errcode(pInfo, " -> %s", rc); diff --git a/test/show_speedtest1_rtree.tcl b/test/show_speedtest1_rtree.tcl new file mode 100644 index 0000000000..3faa168146 --- /dev/null +++ b/test/show_speedtest1_rtree.tcl @@ -0,0 +1,57 @@ +#!/usr/bin/tclsh +# +# This script displays the field of rectangles used by --testset rtree +# of speedtest1. Run this script as follows: +# +# rm test.db +# ./speedtest1 --testset rtree --size 25 test.db +# sqlite3 --separator ' ' test.db 'SELECT * FROM rt1' >data.txt +# wish show_speedtest1_rtree.tcl +# +# The filename "data.txt" is hard coded into this script and so that name +# must be used on lines 3 and 4 above. Elsewhere, different filenames can +# be used. The --size N parameter can be adjusted as desired. +# +package require Tk +set f [open data.txt rb] +set data [read $f] +close $f +canvas .c +frame .b +button .b.b1 -text X-Y -command refill-xy +button .b.b2 -text X-Z -command refill-xz +button .b.b3 -text Y-Z -command refill-yz +pack .b.b1 .b.b2 .b.b3 -side left +pack .c -side top -fill both -expand 1 +pack .b -side top +proc resize_canvas_to_fit {} { + foreach {x0 y0 x1 y1} [.c bbox all] break + set w [expr {$x1-$x0}] + set h [expr {$y1-$y0}] + .c config -width $w -height $h +} +proc refill-xy {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $x0 $y0 $x1 $y1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +proc refill-xz {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $x0 $z0 $x1 $z1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +proc refill-yz {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $y0 $z0 $y1 $z1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +refill-xy diff --git a/test/speedtest1.c b/test/speedtest1.c index 28c24e0885..383f5809a9 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -29,6 +29,7 @@ static const char zHelp[] = " --trace Turn on SQL tracing\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" + " --verify Run additional verification steps.\n" " --without-rowid Use WITHOUT ROWID where appropriate\n" ; @@ -51,6 +52,7 @@ static struct Global { int bReprepare; /* True to reprepare the SQL on each rerun */ int bSqlOnly; /* True to print the SQL once only */ int bExplain; /* Print SQL with EXPLAIN prefix */ + int bVerify; /* Try to verify that results are correct */ int szTest; /* Scale factor for test iterations */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ @@ -931,6 +933,183 @@ void testset_cte(void){ } +/* Generate two numbers between 1 and mx. The first number is less than +** the second. Usually the numbers are near each other but can sometimes +** be far apart. +*/ +static void twoCoords( + int p1, int p2, /* Parameters adjusting sizes */ + unsigned mx, /* Range of 1..mx */ + unsigned *pX0, unsigned *pX1 /* OUT: write results here */ +){ + unsigned d, x0, x1, span; + + span = mx/100 + 1; + if( speedtest1_random()%3==0 ) span *= p1; + if( speedtest1_random()%p2==0 ) span = mx/2; + d = speedtest1_random()%span + 1; + x0 = speedtest1_random()%(mx-d) + 1; + x1 = x0 + d; + *pX0 = x0; + *pX1 = x1; +} + +/* The following routine is an R-Tree geometry callback. It returns +** true if the object overlaps a slice on the Y coordinate between the +** two values given as arguments. In other words +** +** SELECT count(*) FROM rt1 WHERE id MATCH xslice(10,20); +** +** Is the same as saying: +** +** SELECT count(*) FROM rt1 WHERE y1>=10 AND y0<=20; +*/ +static int xsliceGeometryCallback( + sqlite3_rtree_geometry *p, + int nCoord, + double *aCoord, + int *pRes +){ + *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1]; + return SQLITE_OK; +} + +/* +** A testset for the R-Tree virtual table +*/ +void testset_rtree(int p1, int p2){ + unsigned i, n; + unsigned mxCoord; + unsigned x0, x1, y0, y1, z0, z1; + unsigned iStep; + int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 ); + + mxCoord = 15000; + n = g.szTest*100; + speedtest1_begin_test(100, "%d INSERTs into an r-tree", n); + speedtest1_exec("BEGIN"); + speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)"); + speedtest1_prepare("INSERT INTO rt1(id,x0,x1,y0,y1,z0,z1)" + "VALUES(?1,?2,?3,?4,?5,?6,?7)"); + for(i=1; i<=n; i++){ + twoCoords(p1, p2, mxCoord, &x0, &x1); + twoCoords(p1, p2, mxCoord, &y0, &y1); + twoCoords(p1, p2, mxCoord, &z0, &z1); + sqlite3_bind_int(g.pStmt, 1, i); + sqlite3_bind_int(g.pStmt, 2, x0); + sqlite3_bind_int(g.pStmt, 3, x1); + sqlite3_bind_int(g.pStmt, 4, y0); + sqlite3_bind_int(g.pStmt, 5, y1); + sqlite3_bind_int(g.pStmt, 6, z0); + sqlite3_bind_int(g.pStmt, 7, z1); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + speedtest1_begin_test(101, "Copy from rtree to a regular table"); + speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); + speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1"); + speedtest1_end_test(); + + n = g.szTest*20; + speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n); + speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND x1<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND y0<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND y0<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND x0<=?2" + " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2"); + iStep = mxCoord/n; + for(i=0; i Date: Mon, 28 Apr 2014 19:34:06 +0000 Subject: [PATCH 162/177] The trunk assumes that an open range constraint on an indexed term (col>?) term matches 1/4 of the indexed rows, and that a closed constraint (col BETWEEN ? AND ?) matches 1/64. Change this branch to do the same. FossilOrigin-Name: 4047ac75e2a8f0b330255501c42e4f04e5ab500d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05f3d798c0..b037731f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T15:11:25.118 +C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. +D 2014-04-28T19:34:06.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 +F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 -R 307957877e44688fb3b83b922c0183e6 +P 20f468dfbcb247e51446fad411a6e6cc0d130411 +R eabc843f8d46cb0d3b0b7dd8faf4ef00 U dan -Z 085ae09cb0f2460f340245b5be0529cf +Z 37bdf577e72e4cd110b1edb4853d30e0 diff --git a/manifest.uuid b/manifest.uuid index 8fd0bd92ad..405d43fc27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file +4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c1d81e183..4b2eef9c52 100644 --- a/src/where.c +++ b/src/where.c @@ -2155,6 +2155,12 @@ static int whereRangeScanEst( assert( pLower || pUpper ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNew Date: Mon, 28 Apr 2014 20:11:20 +0000 Subject: [PATCH 163/177] Do not reduce the number of rows scanned at all for "IS NOT NULL" constraints. Fix a bug in calculating the number of rows visited by scans of partial indicies. FossilOrigin-Name: a8ae93f0cffa116df0ba34d46a53f49d42dace41 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/where.c | 8 ++-- test/cost.test | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ test/index6.test | 5 ++- 5 files changed, 112 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b037731f09..823653ac7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. -D 2014-04-28T19:34:06.281 +C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. +D 2014-04-28T20:11:20.674 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 +F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 +F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -611,7 +611,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 +F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 20f468dfbcb247e51446fad411a6e6cc0d130411 -R eabc843f8d46cb0d3b0b7dd8faf4ef00 +P 4047ac75e2a8f0b330255501c42e4f04e5ab500d +R fc72ae401d0e8a316c9dd5e0ec98069f U dan -Z 37bdf577e72e4cd110b1edb4853d30e0 +Z f6f096240da2e8d5cf26af9c9310dce5 diff --git a/manifest.uuid b/manifest.uuid index 405d43fc27..99d11f9b9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file +a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4b2eef9c52..8abe6bd1a1 100644 --- a/src/where.c +++ b/src/where.c @@ -1992,9 +1992,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ if( pTerm ){ if( pTerm->truthProb<=0 ){ nRet += pTerm->truthProb; - }else if( pTerm->wtFlags & TERM_VNULL ){ - nRet -= 10; assert( 10==sqlite3LogEst(2) ); - }else{ + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ nRet -= 20; assert( 20==sqlite3LogEst(4) ); } } @@ -2153,14 +2151,17 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ if( pLower && pUpper ) nNew -= 20; + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewiTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; diff --git a/test/cost.test b/test/cost.test index faaa7bdf16..0f9314fc8e 100644 --- a/test/cost.test +++ b/test/cost.test @@ -89,6 +89,101 @@ do_eqp_test 4.3 { } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(x, y); + CREATE INDEX t2i1 ON t2(x); +} + +do_eqp_test 5.2 { + SELECT * FROM t2 ORDER BY x, y; +} {} +#exit + +# TODO: Check this one out!! +# set sqlite_where_trace 0xfff +do_eqp_test 5.3 { + SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; +} {} +#exit + +# where7.test, where8.test: +# +do_execsql_test 6.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t3i1 ON t3(b); + CREATE INDEX t3i2 ON t3(c); +} + +#set sqlite_where_trace 0xfff +# eqp.test +do_eqp_test 6.2 { + SELECT a FROM t3 WHERE (b BETWEEN 2 AND 4) OR c=100 ORDER BY a +} { +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + CREATE INDEX t1d ON t1(d); + CREATE INDEX t1e ON t1(e); + CREATE INDEX t1f ON t1(f); + CREATE INDEX t1g ON t1(g); +} + +do_eqp_test 7.2 { + SELECT a FROM t1 + WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) + ORDER BY a +} { +} + +#set sqlite_where_trace 0xfff +do_eqp_test 7.3 { + SELECT rowid FROM t1 + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) +} {} +#exit + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE composer( + cid INTEGER PRIMARY KEY, + cname TEXT + ); + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + aname TEXT + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + cid INTEGER REFERENCES composer, + aid INTEGER REFERENCES album, + title TEXT + ); + CREATE INDEX track_i1 ON track(cid); + CREATE INDEX track_i2 ON track(aid); +} + +do_eqp_test 8.2 { + SELECT DISTINCT aname + FROM album, composer, track + WHERE cname LIKE '%bach%' + AND unlikely(composer.cid=track.cid) + AND unlikely(album.aid=track.aid); +} { +} + finish_test diff --git a/test/index6.test b/test/index6.test index 3451e5c1df..68bdd06c14 100644 --- a/test/index6.test +++ b/test/index6.test @@ -145,11 +145,11 @@ do_test index6-2.1 { execsql { CREATE TABLE t2(a,b); INSERT INTO t2(a,b) SELECT value, value FROM nums WHERE value<1000; - UPDATE t2 SET a=NULL WHERE b%5==0; + UPDATE t2 SET a=NULL WHERE b%2==0; CREATE INDEX t2a1 ON t2(a) WHERE a IS NOT NULL; SELECT count(*) FROM t2 WHERE a IS NOT NULL; } -} {800} +} {500} do_test index6-2.2 { execsql { EXPLAIN QUERY PLAN @@ -157,6 +157,7 @@ do_test index6-2.2 { } } {/.* TABLE t2 USING INDEX t2a1 .*/} ifcapable stat4||stat3 { + execsql ANALYZE do_test index6-2.3stat4 { execsql { EXPLAIN QUERY PLAN From 67a03cfd6e5213f1b7997356d5db620bd21e5329 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 12:01:35 +0000 Subject: [PATCH 164/177] Fix a test case so that it updates sqlite_stat1 consistently. FossilOrigin-Name: 2dc5a0b55567f13f0528ed17242e680cde2f2a29 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/skipscan2.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 823653ac7d..a7e8fbdb50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. -D 2014-04-28T20:11:20.674 +C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. +D 2014-04-29T12:01:35.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -814,7 +814,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f -F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d +F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4047ac75e2a8f0b330255501c42e4f04e5ab500d -R fc72ae401d0e8a316c9dd5e0ec98069f +P a8ae93f0cffa116df0ba34d46a53f49d42dace41 +R 176e01464114f39a9f52a0420836d3f7 U dan -Z f6f096240da2e8d5cf26af9c9310dce5 +Z 9c22069b17d97b138b62636ee980813e diff --git a/manifest.uuid b/manifest.uuid index 99d11f9b9b..5debc1c5f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file +2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file diff --git a/test/skipscan2.test b/test/skipscan2.test index e39b16ed27..27d193e94d 100644 --- a/test/skipscan2.test +++ b/test/skipscan2.test @@ -74,6 +74,7 @@ do_execsql_test skipscan2-1.4 { -- of a skip-scan. So make a manual adjustment to the stat1 table -- to make it seem like there are many more. UPDATE sqlite_stat1 SET stat='10000 5000 20' WHERE idx='people_idx1'; + UPDATE sqlite_stat1 SET stat='10000 1' WHERE idx='sqlite_autoindex_people_1'; ANALYZE sqlite_master; } db cache flush From 09e1df6c009bc90558ec0250f005bb59fe8f808e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 16:10:22 +0000 Subject: [PATCH 165/177] Ignore likelihood() values on indexed IPK lookups. FossilOrigin-Name: 5bb7757a7b32a74482d3e93e9c9eea02273fe981 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/cost.test | 36 ++++++++++++++++++++++-------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a7e8fbdb50..cf16f91a6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. -D 2014-04-29T12:01:35.119 +C Ignore\slikelihood()\svalues\son\sindexed\sIPK\slookups. +D 2014-04-29T16:10:22.104 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 +F src/where.c 397bd5d4f402238c396d73b3c98b9fa2c312eea4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 +F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a8ae93f0cffa116df0ba34d46a53f49d42dace41 -R 176e01464114f39a9f52a0420836d3f7 +P 2dc5a0b55567f13f0528ed17242e680cde2f2a29 +R b5feca3cdcf90dbcdc4d81625fcb13e8 U dan -Z 9c22069b17d97b138b62636ee980813e +Z d7294cbb283905147264033ecb5a94d9 diff --git a/manifest.uuid b/manifest.uuid index 5debc1c5f8..0097b17163 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file +5bb7757a7b32a74482d3e93e9c9eea02273fe981 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8abe6bd1a1..566c907e55 100644 --- a/src/where.c +++ b/src/where.c @@ -4189,7 +4189,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 ){ + if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; diff --git a/test/cost.test b/test/cost.test index 0f9314fc8e..979815f9f7 100644 --- a/test/cost.test +++ b/test/cost.test @@ -66,10 +66,10 @@ do_eqp_test 3.2 { #------------------------------------------------------------------------- # If there is no likelihood() or stat3 data, SQLite assumes that a closed # range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) -# visits 1/16 of the rows in a table. +# visits 1/64 of the rows in a table. # -# Note: 1/17 =~ 0.058 -# Note: 1/15 =~ 0.067 +# Note: 1/63 =~ 0.016 +# Note: 1/65 =~ 0.015 # reset_db do_execsql_test 4.1 { @@ -78,12 +78,12 @@ do_execsql_test 4.1 { CREATE INDEX i2 ON t1(b); } do_eqp_test 4.2 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } do_eqp_test 4.3 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b? AND x? AND b=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) ORDER BY a } { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b Date: Tue, 29 Apr 2014 19:01:57 +0000 Subject: [PATCH 166/177] Test that the default values used when sqlite_stat1 data is not available are calculated correctly. Fixes for the same. FossilOrigin-Name: e2d42f909de85a0586389f2dc0e654f7af2e351a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 35 +++++++++++++++-------------------- test/cost.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 5dc1bf6a38..f4b5efc137 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-04-29T16:46:24.176 +C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. +D 2014-04-29T19:01:57.481 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 +F src/build.c 02665ca158431a0926b10cbd7d8178a4c9fc4a22 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f +F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5bb7757a7b32a74482d3e93e9c9eea02273fe981 af2cbe64adab5f9e3b0f3da00d06428088589d7f -R 9c74729ef8e10b97c890b0be6d308bd3 +P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e +R e73cccb2b7e246d7f5476a97d396afde U dan -Z e5760c6e9001de9044df542c0c7c165c +Z 19f8efc43d85a395f7ff59d4035b6f55 diff --git a/manifest.uuid b/manifest.uuid index e9d3392789..92aa1b2d7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e \ No newline at end of file +e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file diff --git a/src/build.c b/src/build.c index fb025495dc..a79abed3b6 100644 --- a/src/build.c +++ b/src/build.c @@ -3260,32 +3260,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ -#if 0 - tRowcnt *a = pIdx->aiRowEst; - int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; - } -#endif - /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ - LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; LogEst *a = pIdx->aiRowLogEst; int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 0; + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* diff --git a/test/cost.test b/test/cost.test index 979815f9f7..e04dc9392a 100644 --- a/test/cost.test +++ b/test/cost.test @@ -192,6 +192,51 @@ do_eqp_test 8.2 { 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE t1( + a,b,c,d,e, f,g,h,i,j, + k,l,m,n,o, p,q,r,s,t + ); + CREATE INDEX i1 ON t1(k,l,m,n,o,p,q,r,s,t); +} +do_test 9.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 DEFAULT VALUES } + } + execsql { + ANALYZE; + CREATE INDEX i2 ON t1(a,b,c,d,e,f,g,h,i,j); + } +} {} + +set L [list a=? b=? c=? d=? e=? f=? g=? h=? i=? j=?] +foreach {tn nTerm nRow} { + 1 1 10 + 2 2 9 + 3 3 8 + 4 4 7 + 5 5 6 + 6 6 5 + 7 7 5 + 8 8 5 + 9 9 5 + 10 10 5 +} { + set w [join [lrange $L 0 [expr $nTerm-1]] " AND "] + set p1 [expr ($nRow-1) / 100.0] + set p2 [expr ($nRow+1) / 100.0] + + set sql1 "SELECT * FROM t1 WHERE likelihood(k=?, $p1) AND $w" + set sql2 "SELECT * FROM t1 WHERE likelihood(k=?, $p2) AND $w" + + do_eqp_test 9.3.$tn.1 $sql1 {/INDEX i1/} + do_eqp_test 9.3.$tn.2 $sql2 {/INDEX i2/} +} + + + finish_test From 224155dd1c165a133279ebaa52f80f771844cedc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 13:19:09 +0000 Subject: [PATCH 167/177] Fix long-standing typos in comments. FossilOrigin-Name: b9f91317c34d07769a95dc2f905a6ccabceb64a3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 4 ++-- src/util.c | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f4b5efc137..44e14c4938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. -D 2014-04-29T19:01:57.481 +C Fix\slong-standing\stypos\sin\scomments. +D 2014-04-30T13:19:09.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da +F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf +F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e -R e73cccb2b7e246d7f5476a97d396afde -U dan -Z 19f8efc43d85a395f7ff59d4035b6f55 +P e2d42f909de85a0586389f2dc0e654f7af2e351a +R cdeda4c4d8ae1ea7276fce5a8779e48d +U drh +Z 6915f97ffb226384426800091a7da39d diff --git a/manifest.uuid b/manifest.uuid index 92aa1b2d7c..42cd607e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file +b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b584ff401d..4bd65a690f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -525,10 +525,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 diff --git a/src/util.c b/src/util.c index d88c17b759..577d552ac3 100644 --- a/src/util.c +++ b/src/util.c @@ -1246,8 +1246,8 @@ LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; From 253666e5203974b28d990e321b243153f3a432dd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 14:22:38 +0000 Subject: [PATCH 168/177] Improved rendering of LogEst values corresponding to real values near 0.0 in the tool/logest.c utility program. FossilOrigin-Name: 32910c8c595858245bb7ecfe3aa0f90eeae641af --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/logest.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 44e14c4938..29bd3a7832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slong-standing\stypos\sin\scomments. -D 2014-04-30T13:19:09.070 +C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. +D 2014-04-30T14:22:38.579 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1128,7 +1128,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 +F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2d42f909de85a0586389f2dc0e654f7af2e351a -R cdeda4c4d8ae1ea7276fce5a8779e48d +P b9f91317c34d07769a95dc2f905a6ccabceb64a3 +R 0e5a317f465f102c0002288cd477a07d U drh -Z 6915f97ffb226384426800091a7da39d +Z 88c640e8cc6f42fa637c9ebb251cd8ac diff --git a/manifest.uuid b/manifest.uuid index 42cd607e10..7421cca676 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file +32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 1ac337d36c..347fa68a4f 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -83,7 +83,8 @@ static LogEst logEstFromDouble(double x){ LogEst e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=0.0 ) return -32768; - if( x<1.0 ) return -logEstFromDouble(1/x); + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); memcpy(&a, &x, 8); @@ -156,8 +157,10 @@ int main(int argc, char **argv){ } } for(i=n-1; i>=0; i--){ - if( a[i]<0 ){ + if( a[i]<-40 ){ printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); From 4a6b8a05cd203606f227263f9146aa7dff36c888 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:47:01 +0000 Subject: [PATCH 169/177] Fix a couple of out-of-date comments in where.c. FossilOrigin-Name: eefeda32d54efbbdf7d20b719299eda48b891fae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 47 ++++++++++++++++++++++++++++++----------------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29bd3a7832..8167b75420 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. -D 2014-04-30T14:22:38.579 +C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. +D 2014-04-30T14:47:01.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9651faf05f900d9235bf3166fa54a9e5f12a40f2 +F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b9f91317c34d07769a95dc2f905a6ccabceb64a3 -R 0e5a317f465f102c0002288cd477a07d -U drh -Z 88c640e8cc6f42fa637c9ebb251cd8ac +P 32910c8c595858245bb7ecfe3aa0f90eeae641af +R 26fff8d56fc56eff56bc1e57cba958ab +U dan +Z b9aa63fbb519903c6b0c5b0a6a8e051f diff --git a/manifest.uuid b/manifest.uuid index 7421cca676..6c9c8ac5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file +eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file diff --git a/src/where.c b/src/where.c index aea0be3042..ec7a908b1a 100644 --- a/src/where.c +++ b/src/where.c @@ -4022,8 +4022,13 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -4085,10 +4090,14 @@ static int whereLoopAddBtreeIndex( /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip @@ -5226,23 +5235,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** also N*log(N) but it has a larger constant of proportionality. - ** Multiply by 3.0. */ + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ rSortCost += 16; } From 75525cbe2423852e551e1843e5fad91acf803024 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:53:21 +0000 Subject: [PATCH 170/177] Update a couple of test cases to account for the fact that this branch prefers an index scan and partial sort over a full-table scan and full external sort. FossilOrigin-Name: 9b975bf33cd8fc28c64183a9642bf9fb436a4746 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/cost.test | 5 ++++- test/eqp.test | 12 ++++++------ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8167b75420..9f27ec6f9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. -D 2014-04-30T14:47:01.628 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. +D 2014-04-30T14:53:21.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de +F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -454,7 +454,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 57c6c604c2807fb5531731c5323133453c24afac +F test/eqp.test 90b56d03a93a2e7bb90f88be6083a8ea53f11a0e F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32910c8c595858245bb7ecfe3aa0f90eeae641af -R 26fff8d56fc56eff56bc1e57cba958ab +P eefeda32d54efbbdf7d20b719299eda48b891fae +R 50e654a1c0b26308b8a927bcd26c6421 U dan -Z b9aa63fbb519903c6b0c5b0a6a8e051f +Z f4d62e136a5d6e05753f5050c6fb0c4a diff --git a/manifest.uuid b/manifest.uuid index 6c9c8ac5b7..00e612df5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file +9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file diff --git a/test/cost.test b/test/cost.test index e04dc9392a..045a4c08d6 100644 --- a/test/cost.test +++ b/test/cost.test @@ -99,7 +99,10 @@ do_execsql_test 5.1 { do_eqp_test 5.2 { SELECT * FROM t2 ORDER BY x, y; -} {} +} { + 0 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} +} do_eqp_test 5.3 { SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; diff --git a/test/eqp.test b/test/eqp.test index 610ebdc5ca..9e2e9505cc 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -312,8 +312,8 @@ do_eqp_test 4.2.3 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.2.4 { @@ -321,8 +321,8 @@ do_eqp_test 4.2.4 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.2.5 { @@ -330,8 +330,8 @@ do_eqp_test 4.2.5 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } From 8164722c5806410eb5443a2498f619475fee3fff Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 15:00:16 +0000 Subject: [PATCH 171/177] Add text to the header comment of whereLoopAddBtree() describing how the costs of various b-tree loops are estimated. FossilOrigin-Name: 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9f27ec6f9b..35f78b46c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. -D 2014-04-30T14:53:21.345 +C Add\stext\sto\sthe\sheader\scomment\sof\swhereLoopAddBtree()\sdescribing\show\sthe\scosts\sof\svarious\sb-tree\sloops\sare\sestimated. +D 2014-04-30T15:00:16.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 +F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eefeda32d54efbbdf7d20b719299eda48b891fae -R 50e654a1c0b26308b8a927bcd26c6421 +P 9b975bf33cd8fc28c64183a9642bf9fb436a4746 +R 20e49e26218874ddfe5fb9438df8f580 U dan -Z f4d62e136a5d6e05753f5050c6fb0c4a +Z c1e32968c738a6da213a45755f91539c diff --git a/manifest.uuid b/manifest.uuid index 00e612df5f..6ee1deefef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file +05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ec7a908b1a..7d01db61ee 100644 --- a/src/where.c +++ b/src/where.c @@ -4349,6 +4349,29 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ From 5da73e1a09b89f3605af1e438ac5e269969015f5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 18:11:55 +0000 Subject: [PATCH 172/177] Fix a problem in calculating the costs of "OR" scans. FossilOrigin-Name: 9bbca48b42e4fe16f2188e18dc736da30a96435c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 17 +++++++++++++++-- test/cost.test | 7 ++++++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c2ff19fbfe..88f6a39dac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sway\sthe\scosts\sof\svarious\squery\splans\sare\sestimated.\sIf\sthe\suser\ssupplies\sa\slikelihood()\svalue\s(or\sequivalent)\son\san\sindexed\sWHERE\sconstraint,\suse\sit\sto\sestimate\sthe\snumber\sof\sindex\srows\svisited. -D 2014-04-30T15:22:25.359 +C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. +D 2014-04-30T18:11:55.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa +F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 +F test/cost.test 04842adec34311d70c3f3c5fd698b22be54138fd F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P af2cbe64adab5f9e3b0f3da00d06428088589d7f 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb -R 20e49e26218874ddfe5fb9438df8f580 +P 90e36676476e8db00658772e6c938242f766d306 +R cf0f5a4a25fa17ee63b35d342904cb95 U dan -Z 75da520dd73c7e9f7e5ee96ec52c5c59 +Z 15897e765882c5d27e5f655ffd8d3c94 diff --git a/manifest.uuid b/manifest.uuid index 437f1a2753..274547b53f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90e36676476e8db00658772e6c938242f766d306 \ No newline at end of file +9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7d01db61ee..3b96e5fbea 100644 --- a/src/where.c +++ b/src/where.c @@ -4738,7 +4738,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur, sPrev; + WhereOrSet sSum, sCur; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -4794,6 +4794,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ whereOrMove(&sSum, &sCur); once = 0; }else{ + WhereOrSet sPrev; whereOrMove(&sPrev, &sSum); sSum.n = 0; for(i=0; iiSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun; + /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs + ** of all sub-scans required by the OR-scan. However, due to rounding + ** errors, it may be that the cost of the OR-scan is equal to its + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that + ** this does not happen. Otherwise, for WHERE clauses such as the + ** following where there is an index on "y": + ** + ** WHERE likelihood(x=?, 0.99) OR y=? + ** + ** the planner may elect to "OR" together a full-table scan and an + ** index lookup. And other similarly odd results. */ + pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/cost.test b/test/cost.test index 045a4c08d6..c413c3cdfa 100644 --- a/test/cost.test +++ b/test/cost.test @@ -150,7 +150,6 @@ do_eqp_test 7.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } -#set sqlite_where_trace 0xfff do_eqp_test 7.3 { SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) @@ -160,6 +159,12 @@ do_eqp_test 7.3 { 0 0 0 {SCAN TABLE t1} } +do_eqp_test 7.4 { + SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL) OR c IS NULL +} { + 0 0 0 {SCAN TABLE t1} +} + #------------------------------------------------------------------------- # reset_db From e03d76254c850555579aa203af78d3e75671383b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 10:19:16 +0000 Subject: [PATCH 173/177] Update a test case in wal2.test that explicitly corrupts a checksum in the wal file to account for the fact that the first byte of said checksum may initially be 0xFF. FossilOrigin-Name: 2b935bdea1452505f36dc8c7aad49e6c42f4eceb --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal2.test | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 88f6a39dac..c24d03cdc7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. -D 2014-04-30T18:11:55.566 +C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. +D 2014-05-01T10:19:16.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1057,7 +1057,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc -F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af +F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 90e36676476e8db00658772e6c938242f766d306 -R cf0f5a4a25fa17ee63b35d342904cb95 +P 9bbca48b42e4fe16f2188e18dc736da30a96435c +R 82092de2428e80a3b03a937e23116b0e U dan -Z 15897e765882c5d27e5f655ffd8d3c94 +Z f6ffad5e4f0572fc8e2909e59c29de84 diff --git a/manifest.uuid b/manifest.uuid index 274547b53f..98d646217d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file +2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file diff --git a/test/wal2.test b/test/wal2.test index b331d5ed10..9d45444d6a 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -811,7 +811,13 @@ do_test wal2-7.1.1 { do_test wal2-7.1.2 { forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal - hexio_write test2.db-wal 48 FF + # The first 32 bytes of the WAL file contain the WAL header. Offset 48 + # is the first byte of the checksum for the first frame in the WAL. + # The following three lines replaces the contents of that byte with + # a different value. + set newval FF + if {$newval == [hexio_read test2.db-wal 48 1]} { set newval 00 } + hexio_write test2.db-wal 48 $newval } {1} do_test wal2-7.1.3 { sqlite3 db2 test2.db From c5f246eb3747ce3ab28aff821774493560e84611 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 20:24:21 +0000 Subject: [PATCH 174/177] Add #ifdefs for test coverage. Add a testcase(). FossilOrigin-Name: be2702ce35c713b33c9b7689643b45fb0de6af2a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 8 +++++++- src/where.c | 1 + 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c24d03cdc7..a856daec13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. -D 2014-05-01T10:19:16.340 +C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). +D 2014-05-01T20:24:21.190 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -159,7 +159,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d +F src/analyze.c 3596f863bb80126fe56ba217df5932749271efc8 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 +F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9bbca48b42e4fe16f2188e18dc736da30a96435c -R 82092de2428e80a3b03a937e23116b0e -U dan -Z f6ffad5e4f0572fc8e2909e59c29de84 +P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb +R a1dff26029b41608141f15cdd82c4ee9 +U drh +Z 39b43641ac7d3a2ec2ad0946638e023c diff --git a/manifest.uuid b/manifest.uuid index 98d646217d..c97cdf989b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file +be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4fbaaa5173..2952b364c3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1390,9 +1390,15 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( aOut ){ aOut[i] = v; - }else{ + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { aLog[i] = sqlite3LogEst(v); } if( *z==' ' ) z++; diff --git a/src/where.c b/src/where.c index 3b96e5fbea..238d1ef42a 100644 --- a/src/where.c +++ b/src/where.c @@ -4200,6 +4200,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->nOut==saved_nOut ); if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; pNew->wsFlags |= WHERE_LIKELIHOOD; From 38524132466e77081fc2262122d86d655eb36be3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 20:26:48 +0000 Subject: [PATCH 175/177] Fix an obscure problem to do with temp register allocation that could occur if more than one simple SELECT within a compound SELECT uses a partial sort. FossilOrigin-Name: 427409ae106cdab7892a6b50fe30c5f52de5addc --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/select.c | 12 ++++----- src/vdbe.c | 1 + test/selectA.test | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index a856daec13..c7b6bc3a26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). -D 2014-05-01T20:24:21.190 +C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. +D 2014-05-01T20:26:48.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 +F src/select.c 089c4d46f067a5cccae93524c6377f981ba99bd9 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -279,7 +279,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 +F src/vdbe.c 7f359193bf2366cc914a9ece093ebf284e56acdc F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -789,7 +789,7 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test 77adaffe9704cb80e301ebaeff4b107b58d435c5 +F test/selectA.test 64b88a80271c1710966e50e633380696b60a12a4 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb -R a1dff26029b41608141f15cdd82c4ee9 -U drh -Z 39b43641ac7d3a2ec2ad0946638e023c +P be2702ce35c713b33c9b7689643b45fb0de6af2a +R 31df819a20c9dcbfb809510504bd5463 +U dan +Z aa2318c066a07655ea4b90fb2a4008ea diff --git a/manifest.uuid b/manifest.uuid index c97cdf989b..76ebe330ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file +427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file diff --git a/src/select.c b/src/select.c index dfca6d3f83..5fff010f34 100644 --- a/src/select.c +++ b/src/select.c @@ -466,15 +466,17 @@ static void pushOntoSorter( ){ Vdbe *v = pParse->pVdbe; int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; int nOBSat = pSort->nOBSat; int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -511,10 +513,6 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( nOBSat==0 ){ - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); - } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; diff --git a/src/vdbe.c b/src/vdbe.c index 4ef6d0738c..a64e5bf5e1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4232,6 +4232,7 @@ case OP_SorterData: { pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); break; } diff --git a/test/selectA.test b/test/selectA.test index ca2ec38da4..6e593e8e22 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -21,6 +21,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix selectA ifcapable !compound { finish_test @@ -1310,4 +1311,68 @@ do_execsql_test selectA-3.98 { SELECT n FROM xyz ORDER BY +n; } {MAD MAD+ MAD++} +#------------------------------------------------------------------------- +# At one point the following code exposed a temp register reuse problem. +# +proc f {args} { return 1 } +db func f f + +do_execsql_test 4.1.1 { + CREATE TABLE t4(a, b); + CREATE TABLE t5(c, d); + + INSERT INTO t5 VALUES(1, 'x'); + INSERT INTO t5 VALUES(2, 'x'); + INSERT INTO t4 VALUES(3, 'x'); + INSERT INTO t4 VALUES(4, 'x'); + + CREATE INDEX i1 ON t4(a); + CREATE INDEX i2 ON t5(c); +} + +do_eqp_test 4.1.2 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 0 0 {SCAN TABLE t5 USING INDEX i2} + 1 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 2 0 0 {SCAN TABLE t4 USING INDEX i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} +} + +do_execsql_test 4.1.3 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 x 2 x 3 x 4 x +} + +do_execsql_test 4.2.1 { + CREATE TABLE t6(a, b); + CREATE TABLE t7(c, d); + + INSERT INTO t7 VALUES(2, 9); + INSERT INTO t6 VALUES(3, 0); + INSERT INTO t6 VALUES(4, 1); + INSERT INTO t7 VALUES(5, 6); + INSERT INTO t6 VALUES(6, 0); + INSERT INTO t7 VALUES(7, 6); + + CREATE INDEX i6 ON t6(a); + CREATE INDEX i7 ON t7(c); +} + +do_execsql_test 4.2.2 { + SELECT c, f(d,c,d,c,d) FROM t7 + UNION ALL + SELECT a, b FROM t6 + ORDER BY 1,2 +} {/2 . 3 . 4 . 5 . 6 . 7 ./} + + finish_test From e724d3d26a6db490377d85b6015d6f85768b26dc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 00:09:40 +0000 Subject: [PATCH 176/177] Add a comment explaining why WhereLoop cost adjustments are omitted for skip-scan loops. FossilOrigin-Name: 3bc43594aaeee9225c0590677fcce480bedcb37b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c7b6bc3a26..af2b3b36ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. -D 2014-05-01T20:26:48.793 +C Add\sa\scomment\sexplaining\swhy\sWhereLoop\scost\sadjustments\sare\somitted\sfor\nskip-scan\sloops. +D 2014-05-02T00:09:40.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f +F src/where.c 91dfd382273c3f67fcdc0ac4728f9140f91c6ab3 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P be2702ce35c713b33c9b7689643b45fb0de6af2a -R 31df819a20c9dcbfb809510504bd5463 -U dan -Z aa2318c066a07655ea4b90fb2a4008ea +P 427409ae106cdab7892a6b50fe30c5f52de5addc +R ee65c8e54ebb299c9693aee246a7e200 +U drh +Z 5a37e5aacc1e047352493d523b6fb849 diff --git a/manifest.uuid b/manifest.uuid index 76ebe330ab..6c6f1f805d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file +3bc43594aaeee9225c0590677fcce480bedcb37b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 238d1ef42a..c5892590bc 100644 --- a/src/where.c +++ b/src/where.c @@ -3783,6 +3783,17 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; From a153643b0731de848cb07cbe697ebef0e1ae4e61 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 14:54:56 +0000 Subject: [PATCH 177/177] Fix a broken test case in fuzz.test. FossilOrigin-Name: faa469355eabb2c407f24638b090725448aac37f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fuzz.test | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index af2b3b36ec..c958eec031 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sexplaining\swhy\sWhereLoop\scost\sadjustments\sare\somitted\sfor\nskip-scan\sloops. -D 2014-05-02T00:09:40.134 +C Fix\sa\sbroken\stest\scase\sin\sfuzz.test. +D 2014-05-02T14:54:56.266 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -582,7 +582,7 @@ F test/func3.test dbccee9133cfef1473c59ec07b5f0262b9d72f9a F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 -F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 +F test/fuzz.test 96083052bf5765e4518c1ba686ce2bab785670d1 F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test efd384b896c647b61a2c1848ba70d42aad60a7b3 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 427409ae106cdab7892a6b50fe30c5f52de5addc -R ee65c8e54ebb299c9693aee246a7e200 -U drh -Z 5a37e5aacc1e047352493d523b6fb849 +P 3bc43594aaeee9225c0590677fcce480bedcb37b +R cb5319eb84ea1cdb4a55d1e33cf3d763 +U dan +Z 412ddec016208716851032f358e0939e diff --git a/manifest.uuid b/manifest.uuid index 6c6f1f805d..52fb89ae57 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bc43594aaeee9225c0590677fcce480bedcb37b \ No newline at end of file +faa469355eabb2c407f24638b090725448aac37f \ No newline at end of file diff --git a/test/fuzz.test b/test/fuzz.test index e1b22ae804..0deed3b636 100644 --- a/test/fuzz.test +++ b/test/fuzz.test @@ -285,7 +285,7 @@ do_test fuzz-1.18 { ) )) } -} {0 -4294967298} +} {0 {{}}} # At one point the following INSERT statement caused an assert() to fail. #