From 0fbb50ee153e654b24c44612056442b377b4763a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Nov 2012 10:54:12 +0000 Subject: [PATCH 1/8] When available, use posix_fallocate() rather than ftruncate() to allocate space for mmap()ed -shm files, since posix_fallocate() gives an error if no disk space is available whereas ftruncate() is silent and leaves the system vulnerable to a SIGBUS upon first write to the mmap()ed region. Ticket [5eaa61ea1881040b17449ca043b6f8fd9ca55dc3] FossilOrigin-Name: 356259617cfad04492a02912fdf781f54a2b4494 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 8 ++++++++ test/wal9.test | 3 +-- 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 028b6852d1..ef2f6d3b59 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\slog\sunlink()\serrors\sif\sthe\serror\sis\ssomething\sother\sthan\nSQLITE_IOERR_DELETE_NOENT.\s\sThe\serror\sis\sstill\sreported\sup\sthe\sstack,\sit\nis\ssimply\snot\sadded\sto\sthe\ssqlite3_log(). -D 2012-11-09T21:40:02.069 +C When\savailable,\suse\sposix_fallocate()\srather\sthan\sftruncate()\sto\sallocate\nspace\sfor\smmap()ed\s-shm\sfiles,\ssince\sposix_fallocate()\sgives\san\serror\sif\nno\sdisk\sspace\sis\savailable\swhereas\sftruncate()\sis\ssilent\sand\sleaves\sthe\ssystem\nvulnerable\sto\sa\sSIGBUS\supon\sfirst\swrite\sto\sthe\smmap()ed\sregion.\nTicket\s[5eaa61ea1881040b17449ca043b6f8fd9ca55dc3] +D 2012-11-13T10:54:12.768 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f0753566e1125d8b2eef6dd080b48ed91a83d424 +F src/os_unix.c fad4c9cbf89aa5e5de1f5972458c39bd75418b16 F src/os_win.c 43ec1285357e5d5d919cb0492eac775c58ad7d12 F src/pager.c ed53fe75a269c1d67645fe079ea0f3f0ce6492d5 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 @@ -943,7 +943,7 @@ F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 -F test/wal9.test b4eb5d27170c65ee9c8ff9c9e76babd902a13cfc +F test/wal9.test 48c40803faf6849515c81213697e9f3376835981 F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test b9f68e39646375c2b877be906babcc15d38b4877 F test/walbig.test f437473a16cfb314867c6b5d1dbcd519e73e3434 @@ -1024,7 +1024,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bed9c172ce624ab7b5b9de9ad42444891717ad9a -R 6c7c8f8d3a3d00e4614865ddb6f518c4 +P 5a3b07f0f5dfae7eea870303f52f37d6a17f1da2 +R 457e538f0280386531160ea4b104ea9e +T *branch * tkt-5eaa61ea18 +T *sym-tkt-5eaa61ea18 * +T -sym-trunk * U drh -Z 81d94dfa08446549954e693bf8235ef9 +Z 97e3b0ff73b00a5f41754576708d0771 diff --git a/manifest.uuid b/manifest.uuid index eae5c05106..4ca6494d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5a3b07f0f5dfae7eea870303f52f37d6a17f1da2 \ No newline at end of file +356259617cfad04492a02912fdf781f54a2b4494 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index ca62139430..21ec0fda9e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4168,11 +4168,19 @@ static int unixShmMap( ** the requested memory region. */ if( !bExtend ) goto shmpage_out; +#if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE + if( osFallocate(pShmNode->h, sStat.st_size, nByte)!=0 ){ + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", + pShmNode->zFilename); + goto shmpage_out; + } +#else if( robust_ftruncate(pShmNode->h, nByte) ){ rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", pShmNode->zFilename); goto shmpage_out; } +#endif } } diff --git a/test/wal9.test b/test/wal9.test index a219e30884..f483eb615c 100644 --- a/test/wal9.test +++ b/test/wal9.test @@ -62,7 +62,7 @@ do_execsql_test 1.2 { # the *shm file is now more than one chunk (>32KiB). do_test 1.3 { file size test.db } {1024} do_test 1.4 { file size test.db-wal } {15421352} -do_test 1.5 { file size test.db-shm } {131072} +do_test 1.5 { expr {[file size test.db-shm]>32768} } {1} do_execsql_test 1.6 { PRAGMA wal_checkpoint } {0 14715 14715} @@ -87,4 +87,3 @@ do_test 1.7 { db2 close finish_test - From 6033e15aa25012e67e94054fb844f0acef1b69c7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Nov 2012 11:08:49 +0000 Subject: [PATCH 2/8] Use preprocessor macros to automatically detect whether or not posix_allocate() is available. (It is generally available on Linux but not on Mac.) Ticket [5eaa61ea1881040b17449ca043b6f8fd9ca55dc3] FossilOrigin-Name: 597333f1024092b94bcd8772541e19a0f707bd40 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/os_unix.c | 7 +++++++ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ef2f6d3b59..b522c75e00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\savailable,\suse\sposix_fallocate()\srather\sthan\sftruncate()\sto\sallocate\nspace\sfor\smmap()ed\s-shm\sfiles,\ssince\sposix_fallocate()\sgives\san\serror\sif\nno\sdisk\sspace\sis\savailable\swhereas\sftruncate()\sis\ssilent\sand\sleaves\sthe\ssystem\nvulnerable\sto\sa\sSIGBUS\supon\sfirst\swrite\sto\sthe\smmap()ed\sregion.\nTicket\s[5eaa61ea1881040b17449ca043b6f8fd9ca55dc3] -D 2012-11-13T10:54:12.768 +C Use\spreprocessor\smacros\sto\sautomatically\sdetect\swhether\sor\snot\nposix_allocate()\sis\savailable.\s\s(It\sis\sgenerally\savailable\son\sLinux\nbut\snot\son\sMac.)\s\sTicket\s[5eaa61ea1881040b17449ca043b6f8fd9ca55dc3] +D 2012-11-13T11:08:49.725 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c fad4c9cbf89aa5e5de1f5972458c39bd75418b16 +F src/os_unix.c b75d9b0876ad3fde151843ad389b4c3dd727c662 F src/os_win.c 43ec1285357e5d5d919cb0492eac775c58ad7d12 F src/pager.c ed53fe75a269c1d67645fe079ea0f3f0ce6492d5 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 @@ -1024,10 +1024,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5a3b07f0f5dfae7eea870303f52f37d6a17f1da2 -R 457e538f0280386531160ea4b104ea9e -T *branch * tkt-5eaa61ea18 -T *sym-tkt-5eaa61ea18 * -T -sym-trunk * +P 356259617cfad04492a02912fdf781f54a2b4494 +R 1da6d6b008a37af0f12b5b83b4203af6 U drh -Z 97e3b0ff73b00a5f41754576708d0771 +Z 2b21d116614b6c7fcc6d8aed3f12096b diff --git a/manifest.uuid b/manifest.uuid index 4ca6494d31..f7349a29f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -356259617cfad04492a02912fdf781f54a2b4494 \ No newline at end of file +597333f1024092b94bcd8772541e19a0f707bd40 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 21ec0fda9e..a3a012126b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -46,6 +46,13 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX /* This file is used on unix only */ +/* Use posix_fallocate() if it is available +*/ +#if !defined(HAVE_POSIX_FALLOCATE) \ + && (_XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L) +# define HAVE_POSIX_FALLOCATE 1 +#endif + /* ** There are various methods for file locking used for concurrency ** control: From 6d405c2cb7e19cc9ef250159f1fc4a9aecc5b714 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 Nov 2012 15:06:57 +0000 Subject: [PATCH 3/8] Have the windows implementation of xDelete return SQLITE_IOERR_DELETE_NOENT if the file to be deleted does not exist. The unix implementation was previously modified to behave this way. The current changes simply brings the two implementations into alignment. FossilOrigin-Name: d4c36d4991b048133efb21b251ab57fa66764d9d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ea35017ae9..42664c084e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Strive\sto\suse\sposix_fallocate()\srather\sthan\sftruncate()\swhen\s\nposix_fallocate()\sis\savailable.\s\sTicket\s[5eaa61ea18]. -D 2012-11-13T11:16:04.644 +C Have\sthe\swindows\simplementation\sof\sxDelete\sreturn\sSQLITE_IOERR_DELETE_NOENT\nif\sthe\sfile\sto\sbe\sdeleted\sdoes\snot\sexist.\s\sThe\sunix\simplementation\swas\npreviously\smodified\sto\sbehave\sthis\sway.\s\sThe\scurrent\schanges\ssimply\sbrings\nthe\stwo\simplementations\sinto\salignment. +D 2012-11-20T15:06:57.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c b75d9b0876ad3fde151843ad389b4c3dd727c662 -F src/os_win.c 43ec1285357e5d5d919cb0492eac775c58ad7d12 +F src/os_win.c 1003f13a187bdab1d2cb54d7a930ac875dc7cf08 F src/pager.c ed53fe75a269c1d67645fe079ea0f3f0ce6492d5 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1024,7 +1024,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5a3b07f0f5dfae7eea870303f52f37d6a17f1da2 597333f1024092b94bcd8772541e19a0f707bd40 -R 1da6d6b008a37af0f12b5b83b4203af6 +P 29980b08ec591f695386b715df72d4afb6ffc3fb +R b34ddf6d9c41f9627ecb66c931c8da70 U drh -Z 308917846cd590cd935ef45fbc502739 +Z d5d273e9e92a86e1ab421c06a18a3cfd diff --git a/manifest.uuid b/manifest.uuid index d6910efb93..6a072bfe59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29980b08ec591f695386b715df72d4afb6ffc3fb \ No newline at end of file +d4c36d4991b048133efb21b251ab57fa66764d9d \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 3c92b43d4e..38f6597ca4 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3888,14 +3888,14 @@ static int winDelete( &sAttrData) ){ attr = sAttrData.dwFileAttributes; }else{ - rc = SQLITE_OK; /* Already gone? */ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ break; } #else attr = osGetFileAttributesW(zConverted); #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3935,7 +3935,7 @@ static int winDelete( } while(1); } #endif - if( rc ){ + if( rc && rc!=SQLITE_IOERR_DELETE_NOENT ){ rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ From 55fbc86981b181488f65bc2599a33451c0cba60f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 21 Nov 2012 02:10:20 +0000 Subject: [PATCH 4/8] In winDelete, determine that a file does not exist by checking for a last error of ERROR_FILE_NOT_FOUND or ERROR_PATH_NOT_FOUND. FossilOrigin-Name: 692ad3c02b1af83f0419283fab9b800e361cdf31 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 21 ++++++++++++++++++--- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 42664c084e..0d6e00bd0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\swindows\simplementation\sof\sxDelete\sreturn\sSQLITE_IOERR_DELETE_NOENT\nif\sthe\sfile\sto\sbe\sdeleted\sdoes\snot\sexist.\s\sThe\sunix\simplementation\swas\npreviously\smodified\sto\sbehave\sthis\sway.\s\sThe\scurrent\schanges\ssimply\sbrings\nthe\stwo\simplementations\sinto\salignment. -D 2012-11-20T15:06:57.977 +C In\swinDelete,\sdetermine\sthat\sa\sfile\sdoes\snot\sexist\sby\schecking\sfor\sa\slast\serror\sof\sERROR_FILE_NOT_FOUND\sor\sERROR_PATH_NOT_FOUND. +D 2012-11-21T02:10:20.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c b75d9b0876ad3fde151843ad389b4c3dd727c662 -F src/os_win.c 1003f13a187bdab1d2cb54d7a930ac875dc7cf08 +F src/os_win.c 6e55b48f793d0c0d0e086d3f1482a0882530eeeb F src/pager.c ed53fe75a269c1d67645fe079ea0f3f0ce6492d5 F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1024,7 +1024,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 29980b08ec591f695386b715df72d4afb6ffc3fb -R b34ddf6d9c41f9627ecb66c931c8da70 -U drh -Z d5d273e9e92a86e1ab421c06a18a3cfd +P d4c36d4991b048133efb21b251ab57fa66764d9d +R 2bf66cd8bb02066607c81d87d184d12d +T *branch * winNotFound +T *sym-winNotFound * +T -sym-trunk * +U mistachkin +Z 28cfcb44d275b9d2825a3e2f3bb8894b diff --git a/manifest.uuid b/manifest.uuid index 6a072bfe59..e17b22c91e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4c36d4991b048133efb21b251ab57fa66764d9d \ No newline at end of file +692ad3c02b1af83f0419283fab9b800e361cdf31 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 38f6597ca4..6f49257705 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3888,14 +3888,24 @@ static int winDelete( &sAttrData) ){ attr = sAttrData.dwFileAttributes; }else{ - rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } #else attr = osGetFileAttributesW(zConverted); #endif if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ @@ -3917,7 +3927,12 @@ static int winDelete( do { attr = osGetFileAttributesA(zConverted); if ( attr==INVALID_FILE_ATTRIBUTES ){ - rc = SQLITE_OK; /* Already gone? */ + lastErrno = osGetLastError(); + if( lastErrno==ERROR_FILE_NOT_FOUND || lastErrno==ERROR_PATH_NOT_FOUND ){ + rc = SQLITE_IOERR_DELETE_NOENT; /* Already gone? */ + }else{ + rc = SQLITE_ERROR; + } break; } if ( attr&FILE_ATTRIBUTE_DIRECTORY ){ From ac4559393136c11be004555abafef53f7e9a6b7a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Nov 2012 19:50:41 +0000 Subject: [PATCH 5/8] Add an option to register global hooks used for logging all SQL executed by an application. FossilOrigin-Name: cd501bbccf3e62b002317592cc331770b32c129a --- main.mk | 1 + manifest | 28 +-- manifest.uuid | 2 +- src/global.c | 4 + src/main.c | 29 ++++ src/sqlite.h.in | 9 + src/sqliteInt.h | 4 + src/test_sqllog.c | 433 ++++++++++++++++++++++++++++++++++++++++++++++ src/vdbeaux.c | 24 ++- 9 files changed, 520 insertions(+), 14 deletions(-) create mode 100644 src/test_sqllog.c diff --git a/main.mk b/main.mk index 76a9f406a2..cbae400a50 100644 --- a/main.mk +++ b/main.mk @@ -257,6 +257,7 @@ TESTSRC = \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ $(TOP)/src/test_stat.c \ + $(TOP)/src/test_sqllog.c \ $(TOP)/src/test_superlock.c \ $(TOP)/src/test_syscall.c \ $(TOP)/src/test_tclvar.c \ diff --git a/manifest b/manifest index 42664c084e..56f3032e28 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\swindows\simplementation\sof\sxDelete\sreturn\sSQLITE_IOERR_DELETE_NOENT\nif\sthe\sfile\sto\sbe\sdeleted\sdoes\snot\sexist.\s\sThe\sunix\simplementation\swas\npreviously\smodified\sto\sbehave\sthis\sway.\s\sThe\scurrent\schanges\ssimply\sbrings\nthe\stwo\simplementations\sinto\salignment. -D 2012-11-20T15:06:57.977 +C Add\san\soption\sto\sregister\sglobal\shooks\sused\sfor\slogging\sall\sSQL\sexecuted\sby\san\sapplication. +D 2012-11-26T19:50:41.133 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 615aeaf7474cb2f141547e0c3ac08bcb75dd2b55 +F main.mk a0d170ae1a8a8683688e281194e09d47a68eaa3f F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -134,7 +134,7 @@ F src/expr.c 3b25a95f3d309403940ba4a3212f197b8b6251d5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af F src/func.c 1755cafdb8f2a291681f1ea55e0215518ebd6e52 -F src/global.c fb44b11e02e06c995e6ed6642509edd23599d584 +F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -143,7 +143,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c 5249486037f88e765cce7c4870846dc5a2cd2ed1 +F src/main.c c0b8cfbe397801dfae1f63b56b6dd30b8726b199 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -176,10 +176,10 @@ F src/resolve.c 7b986a715ac281643309c29257bb58cfae7aa810 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 3a8baf4719f9723b4e0b43f2baa60692d0d921f8 F src/shell.c 24cd0aa74aff73ea08594629faead564c4c2a286 -F src/sqlite.h.in 2a0d1234242d75e19e297db99f0925fc9f021e85 +F src/sqlite.h.in c0ab54580c16be8618fd9af48041dfc56cf071b2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 79c00e24d84541c3117ef34ce09c5749dcdcba25 +F src/sqliteInt.h 68f23dcfc3fb6559c07b327ab6d8d2e63e4bb69e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -221,6 +221,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 76dd8d3111d2f5354c374f71fa23b752bd0b029c +F src/test_sqllog.c 5e8e25829947eee74ebb68a484d22a71c54b9c2f F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -240,7 +241,7 @@ F src/vdbe.c e1b2f881f85598b8e5d3366a3cdba60526d40144 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h 79abf9b31be406d35ca77d6999cb2d42aaf91e78 F src/vdbeapi.c 4c2418161cf45392ba76a7ca92f9a5f06b96f89c -F src/vdbeaux.c 8c8cfd30063e9b3664e2faa0e3785102bf131a10 +F src/vdbeaux.c 552a878f5d9d3484ff6fd09acc3d566403f5db4b F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74 F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 @@ -1024,7 +1025,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 29980b08ec591f695386b715df72d4afb6ffc3fb -R b34ddf6d9c41f9627ecb66c931c8da70 -U drh -Z d5d273e9e92a86e1ab421c06a18a3cfd +P d4c36d4991b048133efb21b251ab57fa66764d9d +R a9a22eaa36217a482e6b6184420d709e +T *branch * sqllog +T *sym-sqllog * +T -sym-trunk * +U dan +Z 0aa6ee962f53d4f3c0dddb8a9d2ec33e diff --git a/manifest.uuid b/manifest.uuid index 6a072bfe59..25b4f98be5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4c36d4991b048133efb21b251ab57fa66764d9d \ No newline at end of file +cd501bbccf3e62b002317592cc331770b32c129a \ No newline at end of file diff --git a/src/global.c b/src/global.c index dc86e1e081..f5da7e7f1f 100644 --- a/src/global.c +++ b/src/global.c @@ -175,6 +175,10 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xLog */ 0, /* pLogArg */ 0, /* bLocaltimeFault */ +#ifdef SQLITE_ENABLE_SQLLOG + 0, /* xSqllog */ + 0 /* pSqllogArg */ +#endif }; diff --git a/src/main.c b/src/main.c index b2826c0c7b..7ef10605b5 100644 --- a/src/main.c +++ b/src/main.c @@ -132,6 +132,13 @@ int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; +#ifdef SQLITE_ENABLE_SQLLOG + { + extern sqlite3_init_sqllog(void); + sqlite3_init_sqllog(); + } +#endif + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, @@ -480,6 +487,15 @@ int sqlite3_config(int op, ...){ break; } +#ifdef SQLITE_ENABLE_SQLLOG + case SQLITE_CONFIG_SQLLOG: { + typedef void(*SQLLOGFUNC_t)(void*, sqlite3*, const char*, int); + sqlite3GlobalConfig.xSqllog = va_arg(ap, SQLLOGFUNC_t); + sqlite3GlobalConfig.pSqllogArg = va_arg(ap, void *); + break; + } +#endif + default: { rc = SQLITE_ERROR; break; @@ -819,6 +835,12 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_BUSY; } +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 0); + } +#endif + /* Convert the connection into a zombie and then close it. */ db->magic = SQLITE_MAGIC_ZOMBIE; @@ -2451,6 +2473,13 @@ opendb_out: db->magic = SQLITE_MAGIC_SICK; } *ppDb = db; +#ifdef SQLITE_ENABLE_SQLLOG + if( sqlite3GlobalConfig.xSqllog ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, db, zFilename, -1 + ); + } +#endif return sqlite3ApiExit(0, rc); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 520cccd042..9770a7fc03 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1593,6 +1593,14 @@ struct sqlite3_mem_methods { **
These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. ** +** +** [[SQLITE_CONFIG_SQLLOG]] +**
SQLITE_CONFIG_SQLLOG +**
This option is only available if sqlite is compiled with the +** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1614,6 +1622,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index eb779253c9..14dce8ed7c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2515,6 +2515,10 @@ struct Sqlite3Config { void (*xLog)(void*,int,const char*); /* Function for logging */ void *pLogArg; /* First argument to xLog() */ int bLocaltimeFault; /* True to fail localtime() calls */ +#ifdef SQLITE_ENABLE_SQLLOG + void(*xSqllog)(void*,sqlite3*,const char*, int); + void *pSqllogArg; +#endif }; /* diff --git a/src/test_sqllog.c b/src/test_sqllog.c new file mode 100644 index 0000000000..ba0367039c --- /dev/null +++ b/src/test_sqllog.c @@ -0,0 +1,433 @@ +/* +** 2012 November 26 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** OVERVIEW +** +** This file contains experimental code used to record data from live +** SQLite applications that may be useful for offline analysis. Specifically: +** +** 1) The initial contents of all database files opened by the +** application, and +** +** 2) All SQL statements executed by the application. +** +** USAGE +** +** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG +** pre-processor symbol defined and this file linked into the application +** somehow. +** +** At runtime, logging is enabled by setting environment variable +** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged +** data. The directory must already exist. +** +** Usually, if the application opens the same database file more than once +** (either by attaching it or by using more than one database handle), only +** a single copy is made. This behaviour may be overridden (so that a +** separate copy is taken each time the database file is opened or attached) +** by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0. +** +** OUTPUT: +** +** The SQLITE_SQLLOG_DIR is populated with three types of files: +** +** sqllog_N.db - Copies of database files. N may be any integer. +** +** sqllog_N.sql - A list of SQL statements executed by a single +** connection. N may be any integer. +** +** sqllog.idx - An index mapping from integer N to a database +** file name - indicating the full path of the +** database from which sqllog_N.db was copied. +*/ + +#include "sqlite3.h" +#include "stdio.h" +#include "stdlib.h" +#include "string.h" +#include "assert.h" + +#define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" +#define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" + +/* Assume that all database and database file names are shorted than this. */ +#define SQLLOG_NAMESZ 512 + +/* Maximum number of simultaneous database connections the process may +** open (if any more are opened an error is logged using sqlite3_log() +** and processing is halted). +*/ +#define MAX_CONNECTIONS 256 + +struct SLConn { + int isErr; /* True if an error has occurred */ + sqlite3 *db; /* Connection handle */ + int iLog; /* First integer value used in file names */ + FILE *fd; /* File descriptor for log file */ +}; + +struct SLGlobal { + /* Protected by MUTEX_STATIC_MASTER */ + sqlite3_mutex *mutex; /* Recursive mutex */ + int nConn; /* Size of aConn[] array */ + + /* Protected by SLGlobal.mutex */ + int bReuse; /* True to avoid extra copies of db files */ + char zDir[SQLLOG_NAMESZ]; /* Directory to create files in */ + char zIdx[SQLLOG_NAMESZ]; /* Full path to *.idx file */ + int iNextLog; /* Used to allocate file names */ + int iNextDb; /* Used to allocate database file names */ + int bRec; /* True if testSqllog() is called rec. */ + int iClock; /* Clock value */ + struct SLConn aConn[MAX_CONNECTIONS]; +} sqllogglobal; + +/* +** Return true if c is an ASCII whitespace character. +*/ +static int sqllog_isspace(char c){ + return (c==' ' || c=='\t' || c=='\n' || c=='\v' || c=='\f' || c=='\r'); +} + +/* +** The first argument points to a nul-terminated string containing an SQL +** command. Before returning, this function sets *pz to point to the start +** of the first token in this command, and *pn to the number of bytes in +** the token. This is used to check if the SQL command is an "ATTACH" or +** not. +*/ +static void sqllogTokenize(const char *z, const char **pz, int *pn){ + const char *p = z; + int n; + + /* Skip past any whitespace */ + while( sqllog_isspace(*p) ){ + p++; + } + + /* Figure out how long the first token is */ + *pz = p; + n = 0; + while( (p[n]>='a' && p[n]<='z') || (p[n]>='A' && p[n]<='Z') ) n++; + *pn = n; +} + +/* +** Check if the logs directory already contains a copy of database file +** zFile. If so, return a pointer to the full path of the copy. Otherwise, +** return NULL. +** +** If a non-NULL value is returned, then the caller must arrange to +** eventually free it using sqlite3_free(). +*/ +static char *sqllogFindFile(const char *zFile){ + char *zRet = 0; + FILE *fd = 0; + + /* Open the index file for reading */ + fd = fopen(sqllogglobal.zIdx, "r"); + + while( feof(fd)==0 ){ + char *z; + int n; + char zLine[SQLLOG_NAMESZ*2+5]; + + fgets(zLine, sizeof(zLine), fd); + z = zLine; + while( *z>='0' && *z<='9' ) z++; + while( *z==' ' ) z++; + + n = strlen(z); + while( n>0 && sqllog_isspace(z[n-1]) ) n--; + if( n==strlen(zFile) && 0==memcmp(zFile, z, n) ){ + char zBuf[16]; + z = zLine; + while( *z>='0' && *z<='9' ){ + zBuf[z-zLine] = *z; + z++; + } + zRet = sqlite3_mprintf("%s/sqllog_%s.db", sqllogglobal.zDir, zBuf); + break; + } + } + + fclose(fd); + return zRet; +} + +static void sqllogFindAttached( + struct SLConn *p, /* Database connection */ + const char *zSearch, /* Name to search for (or NULL) */ + char *zName, /* OUT: Name of attached database */ + char *zFile /* OUT: Name of attached file */ +){ + sqlite3_stmt *pStmt; + int rc; + + /* The "PRAGMA database_list" command returns a list of databases in the + ** order that they were attached. So a newly attached database is + ** described by the last row returned. */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zVal1; int nVal1; + char *zVal2; int nVal2; + + zVal1 = sqlite3_column_text(pStmt, 1); + nVal1 = sqlite3_column_bytes(pStmt, 1); + memcpy(zName, zVal1, nVal1+1); + + zVal2 = sqlite3_column_text(pStmt, 2); + nVal2 = sqlite3_column_bytes(pStmt, 2); + memcpy(zFile, zVal2, nVal2+1); + + if( zSearch && strlen(zSearch)==nVal1 + && 0==sqlite3_strnicmp(zSearch, zVal1, nVal1) + ){ + break; + } + } + rc = sqlite3_finalize(pStmt); + } + sqllogglobal.bRec = 0; +} + + +/* +** Parameter zSearch is the name of a database attached to the database +** connection associated with the first argument. This function creates +** a backup of this database in the logs directory. +** +** The name used for the backup file is automatically generated. Call +** it zFile. +** +** If the bLog parameter is true, then a statement of the following form +** is written to the log file associated with *p: +** +** ATTACH 'zFile' AS 'zName'; +** +** Otherwise, if bLog is false, a comment is added to the log file: +** +** -- Main database file is 'zFile' +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ + char zName[SQLLOG_NAMESZ]; /* Attached database name */ + char zFile[SQLLOG_NAMESZ]; /* Database file name */ + char *zFree; + char *zInit = 0; + + sqllogFindAttached(p, zSearch, zName, zFile); + if( zFile[0]=='\0' ){ + zInit = sqlite3_mprintf(""); + }else{ + if( sqllogglobal.bReuse ){ + zInit = sqllogFindFile(zFile); + }else{ + zInit = 0; + } + if( zInit==0 ){ + int rc; + sqlite3 *copy = 0; + FILE *fd; + int iDb; + + /* Generate a file-name to use for the copy of this database */ + iDb = sqllogglobal.iNextDb++; + zInit = sqlite3_mprintf("%s/sqllog_%d.db", sqllogglobal.zDir, iDb); + + /* Create the backup */ + assert( sqllogglobal.bRec==0 ); + sqllogglobal.bRec = 1; + rc = sqlite3_open(zInit, ©); + if( rc==SQLITE_OK ){ + sqlite3_backup *pBak; + sqlite3_exec(copy, "PRAGMA synchronous = 0", 0, 0, 0); + pBak = sqlite3_backup_init(copy, "main", p->db, zName); + if( pBak ){ + sqlite3_backup_step(pBak, -1); + sqlite3_backup_finish(pBak); + } + sqlite3_close(copy); + } + sqllogglobal.bRec = 0; + + /* Write an entry into the database index file */ + fd = fopen(sqllogglobal.zIdx, "a"); + fprintf(fd, "%d %s\n", iDb, zFile); + fclose(fd); + } + } + + if( bLog ){ + zFree = sqlite3_mprintf("ATTACH '%q' AS '%q'; -- clock=%d steps=1\n", + zInit, zName, sqllogglobal.iClock++ + ); + }else{ + zFree = sqlite3_mprintf("-- Main database is '%q'\n", zInit); + } + fprintf(p->fd, "%s", zFree); + sqlite3_free(zFree); + + sqlite3_free(zInit); +} + +/* +** If it is not already open, open the log file for connection *p. +** +** The SLGlobal.mutex mutex is always held when this function is called. +*/ +static void sqllogOpenlog(struct SLConn *p){ + /* If the log file has not yet been opened, open it now. */ + if( p->fd==0 ){ + char *zLog; + + /* If it is still NULL, have global.zDir point to a copy of environment + ** variable $ENVIRONMENT_VARIABLE1_NAME. */ + if( sqllogglobal.zDir[0]==0 ){ + FILE *fd; + char *zVar = getenv(ENVIRONMENT_VARIABLE1_NAME); + if( zVar==0 || strlen(zVar)>=(sizeof(sqllogglobal.zDir)) ) return; + memcpy(sqllogglobal.zDir, zVar, strlen(zVar)+1); + sprintf(sqllogglobal.zIdx, "%s/sqllog.idx", sqllogglobal.zDir); + if( getenv(ENVIRONMENT_VARIABLE2_NAME) ){ + sqllogglobal.bReuse = atoi(getenv(ENVIRONMENT_VARIABLE2_NAME)); + } + fd = fopen(sqllogglobal.zIdx, "w"); + close(fd); + } + + /* Open the log file */ + zLog = sqlite3_mprintf("%s/sqllog_%d.sql", sqllogglobal.zDir, p->iLog); + p->fd = fopen(zLog, "w"); + assert( p->fd ); + sqlite3_free(zLog); + } +} + +/* +** This function is called if the SQLLOG callback is invoked to report +** execution of an SQL statement. Parameter p is the connection the statement +** was executed by, parameter zSql is the text of the statement itself and +** parameter nStep is the number of times sqlite3_step() was called. +*/ +static void testSqllogStmt(struct SLConn *p, const char *zSql, int nStep){ + const char *zFirst; /* Pointer to first token in zSql */ + int nFirst; /* Size of token zFirst in bytes */ + + assert( nStep>0 ); + + sqllogTokenize(zSql, &zFirst, &nFirst); + if( nFirst!=6 || 0!=sqlite3_strnicmp("ATTACH", zFirst, 6) ){ + /* Not an ATTACH statement. Write this directly to the log. */ + fprintf(p->fd, "%s; -- clock=%d steps=%d\n", + zSql, sqllogglobal.iClock++, nStep + ); + }else{ + /* This is an ATTACH statement. Copy the database. */ + sqllogCopydb(p, 0, 1); + } +} + +/* +** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +*/ +static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int nStep){ + struct SLConn *p; + sqlite3_mutex *master = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); + +#if 0 + if( sqllogglobal.isErr && zSql ){ + /* If an error has already occurred, ignore all callbacks except + ** those reporting calls to sqlite3_close(). */ + return; + } +#endif + + /* This is a database open command. */ + if( zSql && nStep<0 ){ + sqlite3_mutex_enter(master); + if( sqllogglobal.mutex==0 ){ + sqllogglobal.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + } + p = &sqllogglobal.aConn[sqllogglobal.nConn++]; + p->fd = 0; + p->db = db; + p->iLog = sqllogglobal.iNextLog++; + sqlite3_mutex_leave(master); + + /* Open the log and take a copy of the main database file */ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + sqllogOpenlog(p); + sqllogCopydb(p, "main", 0); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + + else{ + + int i; + for(i=0; idb==db ) break; + } + if( i==sqllogglobal.nConn ) return; + + /* A database handle close command */ + if( zSql==0 ){ + sqlite3_mutex_enter(master); + if( p->fd ) fclose(p->fd); + p->db = 0; + p->fd = 0; + + sqllogglobal.nConn--; + if( sqllogglobal.nConn==0 ){ + sqlite3_mutex_free(sqllogglobal.mutex); + sqllogglobal.mutex = 0; + }else{ + int nShift = &sqllogglobal.aConn[sqllogglobal.nConn] - p; + if( nShift>0 ){ + memmove(p, &p[1], nShift*sizeof(struct SLConn)); + } + } + sqlite3_mutex_leave(master); + + /* An ordinary SQL command. */ + }else{ + sqlite3_mutex_enter(sqllogglobal.mutex); + if( sqllogglobal.bRec==0 ){ + testSqllogStmt(p, zSql, nStep); + } + sqlite3_mutex_leave(sqllogglobal.mutex); + } + } +} + +/* +** This function is called either before sqlite3_initialized() or by it. +** It checks if the SQLITE_SQLLOG_DIR variable is defined, and if so +** registers an SQLITE_CONFIG_SQLLOG callback to record the applications +** database activity. +*/ +void sqlite3_init_sqllog(void){ + if( getenv(ENVIRONMENT_VARIABLE1_NAME) ){ + if( SQLITE_OK==sqlite3_config(SQLITE_CONFIG_SQLLOG, testSqllog, 0) ){ + memset(&sqllogglobal, 0, sizeof(sqllogglobal)); + sqllogglobal.bReuse = 1; + } + } +} + diff --git a/src/vdbeaux.c b/src/vdbeaux.c index efcc7476c0..286c733592 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -53,7 +53,7 @@ Vdbe *sqlite3VdbeCreate(sqlite3 *db){ void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ assert( isPrepareV2==1 || isPrepareV2==0 ); if( p==0 ) return; -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) if( !isPrepareV2 ) return; #endif assert( p->zSql==0 ); @@ -2326,6 +2326,27 @@ int sqlite3VdbeTransferError(Vdbe *p){ return rc; } +#ifdef SQLITE_ENABLE_SQLLOG +/* +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** invoke it. +*/ +static void vdbeInvokeSqllog(Vdbe *v){ + if( sqlite3GlobalConfig.xSqllog && v->rc==SQLITE_OK && v->zSql && v->pc>=0 ){ + char *zExpanded = sqlite3VdbeExpandSql(v, v->zSql); + assert( v->db->init.busy==0 ); + if( zExpanded ){ + sqlite3GlobalConfig.xSqllog( + sqlite3GlobalConfig.pSqllogArg, v->db, zExpanded, 1 + ); + sqlite3DbFree(v->db, zExpanded); + } + } +} +#else +# define vdbeInvokeSqllog(x) +#endif + /* ** Clean up a VDBE after execution but do not delete the VDBE just yet. ** Write any error messages into *pzErrMsg. Return the result code. @@ -2353,6 +2374,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** instructions yet, leave the main database error information unchanged. */ if( p->pc>=0 ){ + vdbeInvokeSqllog(p); sqlite3VdbeTransferError(p); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; From 71ba10d3c9e8f8c8137d0efaaa549f38f9a8233c Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Nov 2012 10:56:39 +0000 Subject: [PATCH 6/8] Fix problems in test_sqllog.c. Clarify the experimental SQLITE_CONFIG_SQLLOG interface. Handle at least the more likely error conditions in test_sqllog.c. FossilOrigin-Name: 429c5b2056d7b7c644ca53bc97b8e0b9cb89ab04 --- manifest | 19 +++--- manifest.uuid | 2 +- src/main.c | 11 ++-- src/sqlite.h.in | 10 +++- src/test_sqllog.c | 150 ++++++++++++++++++++++++++++------------------ 5 files changed, 117 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index 56f3032e28..3de91a4d81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\soption\sto\sregister\sglobal\shooks\sused\sfor\slogging\sall\sSQL\sexecuted\sby\san\sapplication. -D 2012-11-26T19:50:41.133 +C Fix\sproblems\sin\stest_sqllog.c.\sClarify\sthe\sexperimental\sSQLITE_CONFIG_SQLLOG\sinterface.\sHandle\sat\sleast\sthe\smore\slikely\serror\sconditions\sin\stest_sqllog.c. +D 2012-11-27T10:56:39.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -143,7 +143,7 @@ F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c f20382fbaeec832438a1ba7797bee3d3c8a6d51d -F src/main.c c0b8cfbe397801dfae1f63b56b6dd30b8726b199 +F src/main.c 448564ee3795b13c373bbc5c8590df8fbfe1d64c F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -176,7 +176,7 @@ F src/resolve.c 7b986a715ac281643309c29257bb58cfae7aa810 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 3a8baf4719f9723b4e0b43f2baa60692d0d921f8 F src/shell.c 24cd0aa74aff73ea08594629faead564c4c2a286 -F src/sqlite.h.in c0ab54580c16be8618fd9af48041dfc56cf071b2 +F src/sqlite.h.in 4e71a210f383b6d060bd3fdf81d850f0f8c4eca3 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 F src/sqliteInt.h 68f23dcfc3fb6559c07b327ab6d8d2e63e4bb69e @@ -221,7 +221,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 76dd8d3111d2f5354c374f71fa23b752bd0b029c -F src/test_sqllog.c 5e8e25829947eee74ebb68a484d22a71c54b9c2f +F src/test_sqllog.c 426eb95f41d06f9a45af6cb58f80f516434edc9d F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -1025,10 +1025,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d4c36d4991b048133efb21b251ab57fa66764d9d -R a9a22eaa36217a482e6b6184420d709e -T *branch * sqllog -T *sym-sqllog * -T -sym-trunk * +P cd501bbccf3e62b002317592cc331770b32c129a +R ad919a174cc92f00b19fd1b031dd4d29 U dan -Z 0aa6ee962f53d4f3c0dddb8a9d2ec33e +Z 5c76c382e687bb7efe7c77eaf9918880 diff --git a/manifest.uuid b/manifest.uuid index 25b4f98be5..ae47c04099 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd501bbccf3e62b002317592cc331770b32c129a \ No newline at end of file +429c5b2056d7b7c644ca53bc97b8e0b9cb89ab04 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 7ef10605b5..b52d474443 100644 --- a/src/main.c +++ b/src/main.c @@ -134,7 +134,7 @@ int sqlite3_initialize(void){ #ifdef SQLITE_ENABLE_SQLLOG { - extern sqlite3_init_sqllog(void); + extern void sqlite3_init_sqllog(void); sqlite3_init_sqllog(); } #endif @@ -837,7 +837,8 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ #ifdef SQLITE_ENABLE_SQLLOG if( sqlite3GlobalConfig.xSqllog ){ - sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 0); + /* Closing the handle. Fourth parameter is passed the value 2. */ + sqlite3GlobalConfig.xSqllog(sqlite3GlobalConfig.pSqllogArg, db, 0, 2); } #endif @@ -2475,9 +2476,9 @@ opendb_out: *ppDb = db; #ifdef SQLITE_ENABLE_SQLLOG if( sqlite3GlobalConfig.xSqllog ){ - sqlite3GlobalConfig.xSqllog( - sqlite3GlobalConfig.pSqllogArg, db, zFilename, -1 - ); + /* Opening a db handle. Fourth parameter is passed 0. */ + void *pArg = sqlite3GlobalConfig.pSqllogArg; + sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } #endif return sqlite3ApiExit(0, rc); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9770a7fc03..ba97cd7d4e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1599,7 +1599,15 @@ struct sqlite3_mem_methods { **
This option is only available if sqlite is compiled with the ** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). -** The second should be of type (void*). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ diff --git a/src/test_sqllog.c b/src/test_sqllog.c index ba0367039c..52aa3a6326 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -48,6 +48,13 @@ ** sqllog.idx - An index mapping from integer N to a database ** file name - indicating the full path of the ** database from which sqllog_N.db was copied. +** +** ERROR HANDLING: +** +** This module attempts to make a best effort to continue logging if an +** IO or other error is encountered. For example, if a log file cannot +** be opened logs are not collected for that connection, but other +** logging proceeds as expected. Errors are logged by calling sqlite3_log(). */ #include "sqlite3.h" @@ -56,6 +63,13 @@ #include "string.h" #include "assert.h" +#include "sys/types.h" +#include "unistd.h" +static int getProcessId(void){ + return (int)getpid(); +} + + #define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" #define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" @@ -82,7 +96,7 @@ struct SLGlobal { /* Protected by SLGlobal.mutex */ int bReuse; /* True to avoid extra copies of db files */ - char zDir[SQLLOG_NAMESZ]; /* Directory to create files in */ + char zPrefix[SQLLOG_NAMESZ]; /* Prefix for all created files */ char zIdx[SQLLOG_NAMESZ]; /* Full path to *.idx file */ int iNextLog; /* Used to allocate file names */ int iNextDb; /* Used to allocate database file names */ @@ -135,36 +149,51 @@ static char *sqllogFindFile(const char *zFile){ /* Open the index file for reading */ fd = fopen(sqllogglobal.zIdx, "r"); + if( fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error in fopen()"); + return 0; + } + /* Loop through each entry in the index file. If zFile is not NULL and the + ** entry is a match, then set zRet to point to the filename of the existing + ** copy and break out of the loop. */ while( feof(fd)==0 ){ - char *z; - int n; char zLine[SQLLOG_NAMESZ*2+5]; + if( fgets(zLine, sizeof(zLine), fd) ){ + int n; + char *z; - fgets(zLine, sizeof(zLine), fd); - z = zLine; - while( *z>='0' && *z<='9' ) z++; - while( *z==' ' ) z++; - - n = strlen(z); - while( n>0 && sqllog_isspace(z[n-1]) ) n--; - if( n==strlen(zFile) && 0==memcmp(zFile, z, n) ){ - char zBuf[16]; + zLine[sizeof(zLine)-1] = '\0'; z = zLine; - while( *z>='0' && *z<='9' ){ - zBuf[z-zLine] = *z; - z++; + while( *z>='0' && *z<='9' ) z++; + while( *z==' ' ) z++; + + n = strlen(z); + while( n>0 && sqllog_isspace(z[n-1]) ) n--; + + if( n==strlen(zFile) && 0==memcmp(zFile, z, n) ){ + char zBuf[16]; + memset(zBuf, 0, sizeof(zBuf)); + z = zLine; + while( *z>='0' && *z<='9' ){ + zBuf[z-zLine] = *z; + z++; + } + zRet = sqlite3_mprintf("%s_%s.db", sqllogglobal.zPrefix, zBuf); + break; } - zRet = sqlite3_mprintf("%s/sqllog_%s.db", sqllogglobal.zDir, zBuf); - break; } } + if( ferror(fd) ){ + sqlite3_log(SQLITE_IOERR, "sqllogFindFile(): error reading index file"); + } + fclose(fd); return zRet; } -static void sqllogFindAttached( +static int sqllogFindAttached( struct SLConn *p, /* Database connection */ const char *zSearch, /* Name to search for (or NULL) */ char *zName, /* OUT: Name of attached database */ @@ -201,6 +230,11 @@ static void sqllogFindAttached( rc = sqlite3_finalize(pStmt); } sqllogglobal.bRec = 0; + + if( rc!=SQLITE_OK ){ + sqlite3_log(rc, "sqllogFindAttached(): error in \"PRAGMA database_list\""); + } + return rc; } @@ -228,8 +262,11 @@ static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ char zFile[SQLLOG_NAMESZ]; /* Database file name */ char *zFree; char *zInit = 0; + int rc; + + rc = sqllogFindAttached(p, zSearch, zName, zFile); + if( rc!=SQLITE_OK ) return; - sqllogFindAttached(p, zSearch, zName, zFile); if( zFile[0]=='\0' ){ zInit = sqlite3_mprintf(""); }else{ @@ -241,12 +278,11 @@ static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ if( zInit==0 ){ int rc; sqlite3 *copy = 0; - FILE *fd; int iDb; /* Generate a file-name to use for the copy of this database */ iDb = sqllogglobal.iNextDb++; - zInit = sqlite3_mprintf("%s/sqllog_%d.db", sqllogglobal.zDir, iDb); + zInit = sqlite3_mprintf("%s_%d.db", sqllogglobal.zPrefix, iDb); /* Create the backup */ assert( sqllogglobal.bRec==0 ); @@ -258,21 +294,29 @@ static void sqllogCopydb(struct SLConn *p, const char *zSearch, int bLog){ pBak = sqlite3_backup_init(copy, "main", p->db, zName); if( pBak ){ sqlite3_backup_step(pBak, -1); - sqlite3_backup_finish(pBak); + rc = sqlite3_backup_finish(pBak); + }else{ + rc = sqlite3_errcode(copy); } sqlite3_close(copy); } sqllogglobal.bRec = 0; - /* Write an entry into the database index file */ - fd = fopen(sqllogglobal.zIdx, "a"); - fprintf(fd, "%d %s\n", iDb, zFile); - fclose(fd); + if( rc==SQLITE_OK ){ + /* Write an entry into the database index file */ + FILE *fd = fopen(sqllogglobal.zIdx, "a"); + if( fd ){ + fprintf(fd, "%d %s\n", iDb, zFile); + fclose(fd); + } + }else{ + sqlite3_log(rc, "sqllogCopydb(): error backing up database"); + } } } if( bLog ){ - zFree = sqlite3_mprintf("ATTACH '%q' AS '%q'; -- clock=%d steps=1\n", + zFree = sqlite3_mprintf("ATTACH '%q' AS '%q'; -- clock=%d\n", zInit, zName, sqllogglobal.iClock++ ); }else{ @@ -294,47 +338,44 @@ static void sqllogOpenlog(struct SLConn *p){ if( p->fd==0 ){ char *zLog; - /* If it is still NULL, have global.zDir point to a copy of environment - ** variable $ENVIRONMENT_VARIABLE1_NAME. */ - if( sqllogglobal.zDir[0]==0 ){ + /* If it is still NULL, have global.zPrefix point to a copy of + ** environment variable $ENVIRONMENT_VARIABLE1_NAME. */ + if( sqllogglobal.zPrefix[0]==0 ){ FILE *fd; char *zVar = getenv(ENVIRONMENT_VARIABLE1_NAME); - if( zVar==0 || strlen(zVar)>=(sizeof(sqllogglobal.zDir)) ) return; - memcpy(sqllogglobal.zDir, zVar, strlen(zVar)+1); - sprintf(sqllogglobal.zIdx, "%s/sqllog.idx", sqllogglobal.zDir); + if( zVar==0 || strlen(zVar)+10>=(sizeof(sqllogglobal.zPrefix)) ) return; + sprintf(sqllogglobal.zPrefix, "%s/sqllog_%d", zVar, getProcessId()); + sprintf(sqllogglobal.zIdx, "%s.idx", sqllogglobal.zPrefix); if( getenv(ENVIRONMENT_VARIABLE2_NAME) ){ sqllogglobal.bReuse = atoi(getenv(ENVIRONMENT_VARIABLE2_NAME)); } fd = fopen(sqllogglobal.zIdx, "w"); - close(fd); + if( fd ) fclose(fd); } /* Open the log file */ - zLog = sqlite3_mprintf("%s/sqllog_%d.sql", sqllogglobal.zDir, p->iLog); + zLog = sqlite3_mprintf("%s_%d.sql", sqllogglobal.zPrefix, p->iLog); p->fd = fopen(zLog, "w"); - assert( p->fd ); sqlite3_free(zLog); + if( p->fd==0 ){ + sqlite3_log(SQLITE_IOERR, "sqllogOpenlog(): Failed to open log file"); + } } } /* ** This function is called if the SQLLOG callback is invoked to report ** execution of an SQL statement. Parameter p is the connection the statement -** was executed by, parameter zSql is the text of the statement itself and -** parameter nStep is the number of times sqlite3_step() was called. +** was executed by and parameter zSql is the text of the statement itself. */ -static void testSqllogStmt(struct SLConn *p, const char *zSql, int nStep){ +static void testSqllogStmt(struct SLConn *p, const char *zSql){ const char *zFirst; /* Pointer to first token in zSql */ int nFirst; /* Size of token zFirst in bytes */ - assert( nStep>0 ); - sqllogTokenize(zSql, &zFirst, &nFirst); if( nFirst!=6 || 0!=sqlite3_strnicmp("ATTACH", zFirst, 6) ){ /* Not an ATTACH statement. Write this directly to the log. */ - fprintf(p->fd, "%s; -- clock=%d steps=%d\n", - zSql, sqllogglobal.iClock++, nStep - ); + fprintf(p->fd, "%s; -- clock=%d\n", zSql, sqllogglobal.iClock++); }else{ /* This is an ATTACH statement. Copy the database. */ sqllogCopydb(p, 0, 1); @@ -344,20 +385,15 @@ static void testSqllogStmt(struct SLConn *p, const char *zSql, int nStep){ /* ** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). */ -static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int nStep){ +static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ struct SLConn *p; sqlite3_mutex *master = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); -#if 0 - if( sqllogglobal.isErr && zSql ){ - /* If an error has already occurred, ignore all callbacks except - ** those reporting calls to sqlite3_close(). */ - return; - } -#endif + assert( eType==0 || eType==1 || eType==2 ); + assert( (eType==2)==(zSql==0) ); /* This is a database open command. */ - if( zSql && nStep<0 ){ + if( eType==0 ){ sqlite3_mutex_enter(master); if( sqllogglobal.mutex==0 ){ sqllogglobal.mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); @@ -372,7 +408,7 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int nStep){ sqlite3_mutex_enter(sqllogglobal.mutex); if( sqllogglobal.bRec==0 ){ sqllogOpenlog(p); - sqllogCopydb(p, "main", 0); + if( p->fd ) sqllogCopydb(p, "main", 0); } sqlite3_mutex_leave(sqllogglobal.mutex); } @@ -387,7 +423,7 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int nStep){ if( i==sqllogglobal.nConn ) return; /* A database handle close command */ - if( zSql==0 ){ + if( eType==2 ){ sqlite3_mutex_enter(master); if( p->fd ) fclose(p->fd); p->db = 0; @@ -406,10 +442,10 @@ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int nStep){ sqlite3_mutex_leave(master); /* An ordinary SQL command. */ - }else{ + }else if( p->fd ){ sqlite3_mutex_enter(sqllogglobal.mutex); if( sqllogglobal.bRec==0 ){ - testSqllogStmt(p, zSql, nStep); + testSqllogStmt(p, zSql); } sqlite3_mutex_leave(sqllogglobal.mutex); } From bba44d0d1b5bbbee22b2f5327f967c89f6cf721a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Nov 2012 15:56:38 +0000 Subject: [PATCH 7/8] Fix a problem causing the "number-of-documents" field maintained by FTS4 to be set incorrectly. FossilOrigin-Name: e38fb02d5ea5daa6992df4dfbbeec92bf7b525f6 --- ext/fts3/fts3_write.c | 63 ++++++++++++++++++++++++------------------- manifest | 16 +++++------ manifest.uuid | 2 +- test/fts3conf.test | 42 +++++++++++++++++++++++++++++ 4 files changed, 86 insertions(+), 37 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 7869e638c5..2cfdd2a66c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -776,7 +776,7 @@ static int fts3PendingTermsAdd( int iLangid, /* Language id to use */ const char *zText, /* Text of document to be inserted */ int iCol, /* Column into which text is being inserted */ - u32 *pnWord /* OUT: Number of tokens inserted */ + u32 *pnWord /* IN/OUT: Incr. by number tokens inserted */ ){ int rc; int iStart = 0; @@ -840,7 +840,7 @@ static int fts3PendingTermsAdd( } pModule->xClose(pCsr); - *pnWord = nWord; + *pnWord += nWord; return (rc==SQLITE_DONE ? SQLITE_OK : rc); } @@ -1044,11 +1044,13 @@ static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value *pRowid, /* The docid to be deleted */ - u32 *aSz /* Sizes of deleted document written here */ + u32 *aSz, /* Sizes of deleted document written here */ + int *pbFound /* OUT: Set to true if row really does exist */ ){ int rc; sqlite3_stmt *pSelect; + assert( *pbFound==0 ); if( *pRC ) return; rc = fts3SqlStmt(p, SQL_SELECT_CONTENT_BY_ROWID, &pSelect, &pRowid); if( rc==SQLITE_OK ){ @@ -1066,6 +1068,7 @@ static void fts3DeleteTerms( *pRC = rc; return; } + *pbFound = 1; } rc = sqlite3_reset(pSelect); }else{ @@ -3290,7 +3293,7 @@ static int fts3DoRebuild(Fts3Table *p){ int iCol; int iLangid = langidFromSelect(p, pStmt); rc = fts3PendingTermsDocid(p, iLangid, sqlite3_column_int64(pStmt, 0)); - aSz[p->nColumn] = 0; + memset(aSz, 0, sizeof(aSz[0]) * (p->nColumn+1)); for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ const char *z = (const char *) sqlite3_column_text(pStmt, iCol+1); rc = fts3PendingTermsAdd(p, iLangid, z, iCol, &aSz[iCol]); @@ -5194,28 +5197,32 @@ int sqlite3Fts3DeferToken( static int fts3DeleteByRowid( Fts3Table *p, sqlite3_value *pRowid, - int *pnDoc, + int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ - int isEmpty = 0; - int rc = fts3IsEmpty(p, pRowid, &isEmpty); - if( rc==SQLITE_OK ){ - if( isEmpty ){ - /* Deleting this row means the whole table is empty. In this case - ** delete the contents of all three tables and throw away any - ** data in the pendingTerms hash table. */ - rc = fts3DeleteAll(p, 1); - *pnDoc = *pnDoc - 1; - }else{ - fts3DeleteTerms(&rc, p, pRowid, aSzDel); - if( p->zContentTbl==0 ){ - fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); - if( sqlite3_changes(p->db) ) *pnDoc = *pnDoc - 1; + int rc = SQLITE_OK; /* Return code */ + int bFound = 0; /* True if *pRowid really is in the table */ + + fts3DeleteTerms(&rc, p, pRowid, aSzDel, &bFound); + if( bFound && rc==SQLITE_OK ){ + int isEmpty = 0; /* Deleting *pRowid leaves the table empty */ + rc = fts3IsEmpty(p, pRowid, &isEmpty); + if( rc==SQLITE_OK ){ + if( isEmpty ){ + /* Deleting this row means the whole table is empty. In this case + ** delete the contents of all three tables and throw away any + ** data in the pendingTerms hash table. */ + rc = fts3DeleteAll(p, 1); + *pnChng = 0; + memset(aSzDel, 0, sizeof(u32) * (p->nColumn+1) * 2); }else{ - *pnDoc = *pnDoc - 1; - } - if( p->bHasDocsize ){ - fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + *pnChng = *pnChng - 1; + if( p->zContentTbl==0 ){ + fts3SqlExec(&rc, p, SQL_DELETE_CONTENT, &pRowid); + } + if( p->bHasDocsize ){ + fts3SqlExec(&rc, p, SQL_DELETE_DOCSIZE, &pRowid); + } } } } @@ -5274,13 +5281,13 @@ int sqlite3Fts3UpdateMethod( } /* Allocate space to hold the change in document sizes */ - aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ){ + aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; } - aSzDel = &aSzIns[p->nColumn+1]; - memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); + aSzIns = &aSzDel[p->nColumn+1]; + memset(aSzDel, 0, sizeof(aSzDel[0])*(p->nColumn+1)*2); /* If this is an INSERT operation, or an UPDATE that modifies the rowid ** value, then this operation requires constraint handling. @@ -5365,7 +5372,7 @@ int sqlite3Fts3UpdateMethod( } update_out: - sqlite3_free(aSzIns); + sqlite3_free(aSzDel); sqlite3Fts3SegmentsClose(p); return rc; } diff --git a/manifest b/manifest index 42664c084e..85a64b7abf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\swindows\simplementation\sof\sxDelete\sreturn\sSQLITE_IOERR_DELETE_NOENT\nif\sthe\sfile\sto\sbe\sdeleted\sdoes\snot\sexist.\s\sThe\sunix\simplementation\swas\npreviously\smodified\sto\sbehave\sthis\sway.\s\sThe\scurrent\schanges\ssimply\sbrings\nthe\stwo\simplementations\sinto\salignment. -D 2012-11-20T15:06:57.977 +C Fix\sa\sproblem\scausing\sthe\s"number-of-documents"\sfield\smaintained\sby\sFTS4\sto\sbe\sset\sincorrectly. +D 2012-11-27T15:56:38.822 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -72,7 +72,7 @@ F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 49e36e6ba59f79e6bd6a8bfe434570fe48d20559 F ext/fts3/fts3_unicode2.c a863f05f758af36777dffc2facc898bc73fec896 -F ext/fts3/fts3_write.c ba0bb0a91ca792fba5101bd82fa14d8a00a96365 +F ext/fts3/fts3_write.c a432433a706bd065e8bb0f8b3b33ce7cf9d7f21d F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -471,7 +471,7 @@ F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c -F test/fts3conf.test 8e65ea56f88ced6cdd2252bdddb1a8327ae5af7e +F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 @@ -1024,7 +1024,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 29980b08ec591f695386b715df72d4afb6ffc3fb -R b34ddf6d9c41f9627ecb66c931c8da70 -U drh -Z d5d273e9e92a86e1ab421c06a18a3cfd +P d4c36d4991b048133efb21b251ab57fa66764d9d +R 2a6f06cab2a0521f17074582d293668f +U dan +Z e9d30bbbbbbe7245e672e07e58ec0924 diff --git a/manifest.uuid b/manifest.uuid index 6a072bfe59..8a978c3936 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4c36d4991b048133efb21b251ab57fa66764d9d \ No newline at end of file +e38fb02d5ea5daa6992df4dfbbeec92bf7b525f6 \ No newline at end of file diff --git a/test/fts3conf.test b/test/fts3conf.test index ce410277ca..e91efbefbe 100644 --- a/test/fts3conf.test +++ b/test/fts3conf.test @@ -136,4 +136,46 @@ do_execsql_test 2.2.2 { COMMIT } do_execsql_test 2.2.3 { SELECT * FROM t1 } {{a b c} {a b c}} fts3_integrity 2.2.4 db t1 +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t3 USING fts4; + REPLACE INTO t3(docid, content) VALUES (1, 'one two'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0100000002000000'} + +do_execsql_test 3.2 { + REPLACE INTO t3(docid, content) VALUES (2, 'one two three four'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'four' +} {X'0200000003000000'} + +do_execsql_test 3.3 { + REPLACE INTO t3(docid, content) VALUES (1, 'one two three four five six'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0200000005000000'} + +do_execsql_test 3.4 { + UPDATE OR REPLACE t3 SET docid = 2 WHERE docid=1; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0100000006000000'} + +do_execsql_test 3.5 { + UPDATE OR REPLACE t3 SET docid = 3 WHERE docid=2; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'six' +} {X'0100000006000000'} + +do_execsql_test 3.6 { + REPLACE INTO t3(docid, content) VALUES (3, 'one two'); + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0100000002000000'} + +do_execsql_test 3.7 { + REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four'); + REPLACE INTO t3(docid, content) VALUES (NULL, 'one two three four five six'); + SELECT docid FROM t3; +} {3 4 5} + +do_execsql_test 3.8 { + UPDATE OR REPLACE t3 SET docid = 5, content='three four' WHERE docid = 4; + SELECT quote(matchinfo(t3, 'na')) FROM t3 WHERE t3 MATCH 'one' +} {X'0200000002000000'} + finish_test From 7d1f0c79cfd5ee2fe8f8fce703a0d031786502b4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Nov 2012 16:39:31 +0000 Subject: [PATCH 8/8] Get SQLLOG working on windows. Fix a couple of compiler warnings. FossilOrigin-Name: b3809c937b230b34e5bc6ce4909c04ceef72fb85 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_sqllog.c | 13 ++++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 3de91a4d81..dcae596364 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\stest_sqllog.c.\sClarify\sthe\sexperimental\sSQLITE_CONFIG_SQLLOG\sinterface.\sHandle\sat\sleast\sthe\smore\slikely\serror\sconditions\sin\stest_sqllog.c. -D 2012-11-27T10:56:39.734 +C Get\sSQLLOG\sworking\son\swindows.\s\sFix\sa\scouple\sof\scompiler\swarnings. +D 2012-11-27T16:39:31.650 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 76dd8d3111d2f5354c374f71fa23b752bd0b029c -F src/test_sqllog.c 426eb95f41d06f9a45af6cb58f80f516434edc9d +F src/test_sqllog.c 7813b47021a6d4e39bb7b1b328a8893dc59885cb F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -1025,7 +1025,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P cd501bbccf3e62b002317592cc331770b32c129a -R ad919a174cc92f00b19fd1b031dd4d29 -U dan -Z 5c76c382e687bb7efe7c77eaf9918880 +P 429c5b2056d7b7c644ca53bc97b8e0b9cb89ab04 +R 80bd85027f5a447d832a60beb6eca377 +U drh +Z 7ed00ed04a2be724512bb24e3fa5a2b6 diff --git a/manifest.uuid b/manifest.uuid index ae47c04099..8592a4a12d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -429c5b2056d7b7c644ca53bc97b8e0b9cb89ab04 \ No newline at end of file +b3809c937b230b34e5bc6ce4909c04ceef72fb85 \ No newline at end of file diff --git a/src/test_sqllog.c b/src/test_sqllog.c index 52aa3a6326..49569a39fb 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -66,7 +66,11 @@ #include "sys/types.h" #include "unistd.h" static int getProcessId(void){ +#if SQLITE_OS_WIN + return (int)_getpid(); +#else return (int)getpid(); +#endif } @@ -210,14 +214,14 @@ static int sqllogFindAttached( rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0); if( rc==SQLITE_OK ){ while( SQLITE_ROW==sqlite3_step(pStmt) ){ - char *zVal1; int nVal1; - char *zVal2; int nVal2; + const char *zVal1; int nVal1; + const char *zVal2; int nVal2; - zVal1 = sqlite3_column_text(pStmt, 1); + zVal1 = (const char*)sqlite3_column_text(pStmt, 1); nVal1 = sqlite3_column_bytes(pStmt, 1); memcpy(zName, zVal1, nVal1+1); - zVal2 = sqlite3_column_text(pStmt, 2); + zVal2 = (const char*)sqlite3_column_text(pStmt, 2); nVal2 = sqlite3_column_bytes(pStmt, 2); memcpy(zFile, zVal2, nVal2+1); @@ -466,4 +470,3 @@ void sqlite3_init_sqllog(void){ } } } -