diff --git a/Makefile.in b/Makefile.in index 53ad95df70..8f1f65dcd0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -775,7 +775,7 @@ tclsqlite3$(TEXE): tclsqlite-shell.lo libsqlite3.la # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | $(NAWK) -f $(TOP)/mkopcodeh.awk >opcodes.h diff --git a/Makefile.msc b/Makefile.msc index 022939f2fa..a6c5d5a75b 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -809,7 +809,7 @@ tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c + $(NAWK) -f $(TOP)\mkopcodec.awk opcodes.h > opcodes.c opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h @@ -938,7 +938,7 @@ sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h /link $(LTLINKOPTS) /LIBPATH:$(TCLLIBDIR) $(LIBTCL) $(TLIBS) clean: - del /Q *.lo *.lib *.obj sqlite3.exe libsqlite3.lib + del /Q *.lo *.ilk *.lib *.obj *.pdb sqlite3.exe libsqlite3.lib del /Q sqlite3.h opcodes.c opcodes.h del /Q lemon.exe lempar.c parse.* del /Q mkkeywordhash.exe keywordhash.h diff --git a/Makefile.vxworks b/Makefile.vxworks index 993e557959..8d57da7283 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -517,7 +517,7 @@ tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | \ diff --git a/main.mk b/main.mk index 4837d4e2fb..aab3706272 100644 --- a/main.mk +++ b/main.mk @@ -429,7 +429,7 @@ tclsqlite.o: $(TOP)/src/tclsqlite.c $(HDR) # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)/mkopcodec.awk - sort -n -b -k 3 opcodes.h | $(NAWK) -f $(TOP)/mkopcodec.awk >opcodes.c + $(NAWK) -f $(TOP)/mkopcodec.awk opcodes.h >opcodes.c opcodes.h: parse.h $(TOP)/src/vdbe.c $(TOP)/mkopcodeh.awk cat parse.h $(TOP)/src/vdbe.c | \ diff --git a/manifest b/manifest index 9eaa287209..7c53646216 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Merge\sthe\sstat3-enhancement\sbranch\swith\strunk,\sbut\skeep\sthe\sresulting\smerge\nin\sa\sseparate\sbranch\sfor\snow. -D 2011-08-26T13:16:33.563 +C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sstat3-trunk\sbranch. +D 2011-09-13T19:09:28.865 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315 +F Makefile.in d314143fa6be24828021d3f583ad37d9afdce505 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 55fe94bf23b4c1ff035f19b0ae2ea486350f8d01 -F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 +F Makefile.msc 25da409ce0c7799e57f48a729a8e153b23027adc +F Makefile.vxworks 1deb39c8bb047296c30161ffa10c1b5423e632f9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -104,11 +104,11 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 201b1d81aa4ac0af21b243151e79477c14ce3722 +F main.mk 8744cb76517817170f7fd2c78fbf0006fabb93c1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac -F mkopcodec.awk 3fb9bf077053c968451f4dd03d11661ac373f9d1 +F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F publish.sh 313c5b2425f2cf5e547db7549a9796acc4508f22 @@ -124,16 +124,16 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3 F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ed13fdefdbe671d5777773dcfb3a162ddb4623ae -F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7 +F src/btree.c 4cb13cf019d1978b62494b970d20e9555211094b +F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c ea854c9d5e86098c721574e7c0fda3552227f54f +F src/build.c 57e3b27c26e6754393cd63a2774a6c673df1e804 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 3e961190276d1a48c5224e9558253f8e1f31bf50 +F src/ctime.c dde6a9b835b9e8cc067b713de1a7f6a0c9ff807e F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 -F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 +F src/expr.c cbcd8c2f1588a9862291a081699854c5e1cb28ab F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 9f00ea98f6b360d477b5a78b5b59a1fbde82431c F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 @@ -163,12 +163,12 @@ F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 -F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 +F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 1a34ca3794ced80e4a4ebcc3ba1f4c516762e534 -F src/os_win.c 86bcb5bd0386c761c764c3383879469346da3a14 -F src/pager.c 817f7f7140c9fa2641f28e6330e924708ddd870d -F src/pager.h 2bab1b2ea4eac58663b5833e3522e36b5ff63447 +F src/os_unix.c 10e0c4dcdbec8d4189890fdf3e71b32efae194e3 +F src/os_win.c 33b7b7b48939af5cef2305f5ded19d45c025e2c7 +F src/pager.c 5545863e4e246e1744cfb6993821c6e4b63ffb64 +F src/pager.h 6bea8d1949db33768de1c5b4133b267b40845f8b F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 F src/pcache.c 49e718c095810c6b3334e3a6d89970aceaddefce F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -179,16 +179,16 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d +F src/select.c bf7b7ea6befb483619da5f597b0864668b828c3c F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 3f531daa04457e83bc13765c98c06c7d71c27fa5 F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 6dedc0587dff7dede458000557ac8fc5fbedf5aa +F src/sqliteInt.h b7f41c29477dfa18b18c0aa3e9230c7a4c775de2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3ef1dda2f1dc207c792eaadebf9d8adc44648581 -F src/test1.c 7439efb86c1022f19a39a8e61de2cbac23ffab03 +F src/test1.c 0f41b7c67719207a5de24b009e172c4dcf189827 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -227,7 +227,7 @@ F src/test_stat.c f682704b5d1ba8e1d4e7e882a6d7922e2dcf066c F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa -F src/test_thread.c dc77f920d24f2f515bd315b87942b6396332a414 +F src/test_thread.c 35022393dd54d147b998b6b7f7e945b01114d666 F src/test_vfs.c b0baec983bd6f872715a4b44c8f39104fec333af F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 @@ -238,14 +238,14 @@ F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 4a7191c0f8e918b74e8c84cbdd77746d6b7e3bcf -F src/vdbe.h 2bf6ec77d8b9980fc19da6e0b0a36d0dbf884ce4 -F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096 +F src/vdbe.c 7008edbf8f608d82c035dcb1c56367ad85e68aaa +F src/vdbe.h a10b360bf69474babc8aba8fcc64b824c5e97d38 +F src/vdbeInt.h 693d6ac6810298fc6b4c503cfbe3f99a240f40af F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 -F src/vdbeaux.c 1bd0fbeadede0c07d78e25008db61bc90ac3e412 +F src/vdbeaux.c 316409c6878363525a2835aab15e0505c951a16d F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 -F src/vdbemem.c 74410d1639869b309d6fe1e8cbc02a557157a7c2 -F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e +F src/vdbemem.c 2fc78b3e0fabcc1eaa23cd79dd2e30e6dcfe1e56 +F src/vdbesort.c 667bc65254a9ebaf7bc466ebb7e58884933e351f F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 @@ -370,7 +370,7 @@ F test/descidx1.test 533dcbda614b0463b0ea029527fd27e5a9ab2d66 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 8c4d951fc40aba84421060e07b16099d2f4c2fdf +F test/distinct.test df5b11ad606439129c88720a86787bc9ca181f31 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -512,7 +512,7 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637 F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097 F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 -F test/index4.test c82a59c9ae2ac01804bdb100162dca057318f40f +F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -593,7 +593,7 @@ F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/misc1.test 55cb2bfbf4a8cd61f4be1effc30426ad41696bff F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d -F test/misc3.test 72c5dc87a78e7865c5ec7a969fc572913dbe96b6 +F test/misc3.test 8e42d54b772a23b3c573672d3e0894d15b05221d F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 @@ -610,7 +610,7 @@ F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/oserror.test 3fe52e0bd2891a9bf7cdeb639554992453d46301 -F test/pager1.test 70c94c895ffaf4dc34ee4b66e6e4cd713af41edc +F test/pager1.test 2d3a7c6facd899d8879d23f31454cc53f49358b9 F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 @@ -662,7 +662,7 @@ F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 0d072c5846071b569766e6cd7f923f646a8b2bfa F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 -F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c +F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 34945a516532b11182c3eb26e31247eee3c9ae48 F test/shared2.test 8f71d4eb4d5261280de92284df74172545c852cc F test/shared3.test ebf77f023f4bdaa8f74f65822b559e86ce5c6257 @@ -702,9 +702,9 @@ F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d F test/tester.tcl 0b2999b578964297663de4870babbbee29225622 -F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f -F test/thread002.test 716631b06cccf33b368ab7f6dd3cad92907b8928 -F test/thread003.test 33d2d46e6a53ccb2ff8dc4d0c4e3b3aaee36dcd1 +F test/thread001.test 7cc2ce08f9cde95964736d11e91f9ab610f82f91 +F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 +F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 F test/thread004.test f51dfc3936184aaf73ee85f315224baad272a87f F test/thread005.test 50d10b5684399676174bd96c94ad4250b1a2c8b6 F test/thread1.test df115faa10a4ba1d456e9d4d9ec165016903eae4 @@ -889,7 +889,7 @@ F test/wal.test e11da8d5ea8a38a247339455098357e9adf63d76 F test/wal2.test ad6412596815f553cd30f271d291ab003092bc7e F test/wal3.test 18da4e65c30c43c646ad40e145e9a074e4062fc9 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 08e145a352b1223930c7f0a1de82a8747a99c322 +F test/wal5.test 1bbfaa316dc2a1d0d1fac3f4500c38a90055a41b F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe @@ -929,7 +929,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c d51c68d405ff7f9bad99268ca3c20a198eb983ed +F tool/lemon.c 949328f67cac94969d3112b105b8457edf27f44e F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -952,7 +952,7 @@ F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 F tool/showwal.c f09e5a80a293919290ec85a6a37c85a5ddcf37d9 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b -F tool/spaceanal.tcl b91879d52bf77a1ff5382493284f429d32a63490 +F tool/spaceanal.tcl 1ee4df4e190675ba67b8c60cf304496d0021cfb4 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff @@ -960,13 +960,10 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 -F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 +F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 1dada5158215d1816edb69ff2610f9d2259ce19d eaf447ea87b0ff29ae06283204f522fcd005b284 -R eae2c984d609130c06d9d7e2a5d61977 -T *branch * stat3-trunk -T *sym-stat3-trunk * -T -sym-trunk * +P 63f2c7859fa6e5d0e2cdd218ff52a3ec2d44c61d a9db247b752bcda0131b8f01c6f0182f3101d154 +R 2e9e259494c37eae338f4a7f0467a2a7 U drh -Z cd6356200476d143c6be69191b0e280a +Z 16b18ce61af4e2a2021001a6e69294e0 diff --git a/manifest.uuid b/manifest.uuid index 55b7b89a87..48802d32a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63f2c7859fa6e5d0e2cdd218ff52a3ec2d44c61d \ No newline at end of file +11ca4ed8bf850dae1a24b7182f70039f32bd8dd1 \ No newline at end of file diff --git a/mkopcodec.awk b/mkopcodec.awk index ec80953009..2ef77d4cca 100644 --- a/mkopcodec.awk +++ b/mkopcodec.awk @@ -17,13 +17,18 @@ BEGIN { print " || defined(SQLITE_DEBUG)" print "const char *sqlite3OpcodeName(int i){" print " static const char *const azName[] = { \"?\"," + mx = 0 } /define OP_/ { sub("OP_","",$2) - i++ - printf " /* %3d */ \"%s\",\n", $3, $2 + i = $3+0 + label[i] = $2 + if( mxpKeyInfo, (int)nKey, pKey, - aSpace, sizeof(aSpace)); + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree + ); if( pIdxKey==0 ) return SQLITE_NOMEM; + sqlite3VdbeRecordUnpack(pCur->pKeyInfo, nKey, pKey, pIdxKey); }else{ pIdxKey = 0; } rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); - if( pKey ){ - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + if( pFree ){ + sqlite3DbFree(pCur->pKeyInfo->db, pFree); } return rc; } @@ -1734,22 +1737,11 @@ int sqlite3BtreeOpen( /* A BTREE_SINGLE database is always a temporary and/or ephemeral */ assert( (flags & BTREE_SINGLE)==0 || isTempDb ); - /* The BTREE_SORTER flag is only used if SQLITE_OMIT_MERGE_SORT is undef */ -#ifdef SQLITE_OMIT_MERGE_SORT - assert( (flags & BTREE_SORTER)==0 ); -#endif - - /* BTREE_SORTER is always on a BTREE_SINGLE, BTREE_OMIT_JOURNAL */ - assert( (flags & BTREE_SORTER)==0 || - (flags & (BTREE_SINGLE|BTREE_OMIT_JOURNAL)) - ==(BTREE_SINGLE|BTREE_OMIT_JOURNAL) ); - if( db->flags & SQLITE_NoReadlock ){ flags |= BTREE_NO_READLOCK; } if( isMemdb ){ flags |= BTREE_MEMORY; - flags &= ~BTREE_SORTER; } if( (vfsFlags & SQLITE_OPEN_MAIN_DB)!=0 && (isMemdb || isTempDb) ){ vfsFlags = (vfsFlags & ~SQLITE_OPEN_MAIN_DB) | SQLITE_OPEN_TEMP_DB; @@ -2754,11 +2746,12 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; btreeParseCellPtr(pPage, pCell, &info); - if( info.iOverflow ){ - if( iFrom==get4byte(&pCell[info.iOverflow]) ){ - put4byte(&pCell[info.iOverflow], iTo); - break; - } + if( info.iOverflow + && pCell+info.iOverflow+3<=pPage->aData+pPage->maskPage + && iFrom==get4byte(&pCell[info.iOverflow]) + ){ + put4byte(&pCell[info.iOverflow], iTo); + break; } }else{ if( get4byte(pCell)==iFrom ){ @@ -3479,7 +3472,8 @@ static int btreeCursor( return SQLITE_READONLY; } if( iTable==1 && btreePagecount(pBt)==0 ){ - return SQLITE_EMPTY; + assert( wrFlag==0 ); + iTable = 0; } /* Now that no other errors can occur, finish filling in the BtCursor @@ -4233,6 +4227,9 @@ static int moveToRoot(BtCursor *pCur){ releasePage(pCur->apPage[i]); } pCur->iPage = 0; + }else if( pCur->pgnoRoot==0 ){ + pCur->eState = CURSOR_INVALID; + return SQLITE_OK; }else{ rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); if( rc!=SQLITE_OK ){ @@ -4342,7 +4339,7 @@ int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); *pRes = 1; }else{ assert( pCur->apPage[pCur->iPage]->nCell>0 ); @@ -4381,7 +4378,7 @@ int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); *pRes = 1; }else{ assert( pCur->eState==CURSOR_VALID ); @@ -4454,12 +4451,12 @@ int sqlite3BtreeMovetoUnpacked( if( rc ){ return rc; } - assert( pCur->apPage[pCur->iPage] ); - assert( pCur->apPage[pCur->iPage]->isInit ); - assert( pCur->apPage[pCur->iPage]->nCell>0 || pCur->eState==CURSOR_INVALID ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit ); + assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 ); if( pCur->eState==CURSOR_INVALID ){ *pRes = -1; - assert( pCur->apPage[pCur->iPage]->nCell==0 ); + assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); return SQLITE_OK; } assert( pCur->apPage[0]->intKey || pIdxKey ); @@ -5186,6 +5183,9 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } + if( pCell+info.iOverflow+3 > pPage->aData+pPage->maskPage ){ + return SQLITE_CORRUPT; /* Cell extends past end of page */ + } ovflPgno = get4byte(&pCell[info.iOverflow]); assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; @@ -7288,16 +7288,9 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ - BtShared *pBt = p->pBt; int rc; sqlite3BtreeEnter(p); - if( (pBt->openFlags&BTREE_SINGLE) ){ - pBt->nPage = 0; - sqlite3PagerTruncateImage(pBt->pPager, 1); - rc = newDatabase(pBt); - }else{ - rc = btreeDropTable(p, iTable, piMoved); - } + rc = btreeDropTable(p, iTable, piMoved); sqlite3BtreeLeave(p); return rc; } @@ -7376,6 +7369,11 @@ int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ i64 nEntry = 0; /* Value to return in *pnEntry */ int rc; /* Return code */ + + if( pCur->pgnoRoot==0 ){ + *pnEntry = 0; + return SQLITE_OK; + } rc = moveToRoot(pCur); /* Unless an error occurs, the following loop runs one iteration for each diff --git a/src/btree.h b/src/btree.h index ce19826ad8..9e3a73b3b6 100644 --- a/src/btree.h +++ b/src/btree.h @@ -61,7 +61,6 @@ int sqlite3BtreeOpen( #define BTREE_MEMORY 4 /* This is an in-memory DB */ #define BTREE_SINGLE 8 /* The file contains at most 1 b-tree */ #define BTREE_UNORDERED 16 /* Use of a hash implementation is OK */ -#define BTREE_SORTER 32 /* Used as accumulator in external merge sort */ int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); diff --git a/src/build.c b/src/build.c index 72b3eebab8..8318cb1ff2 100644 --- a/src/build.c +++ b/src/build.c @@ -2343,6 +2343,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iIdx = pParse->nTab++; /* Btree cursor used for pIndex */ int iSorter = iTab; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ + int addr2; /* Address to jump to for next iteration */ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ @@ -2351,15 +2352,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); - /* Set bUseSorter to use OP_OpenSorter, or clear it to insert directly - ** into the index. The sorter is used unless either OMIT_MERGE_SORT is - ** defined or the system is configured to store temp files in-memory. */ -#ifdef SQLITE_OMIT_MERGE_SORT - static const int bUseSorter = 0; -#else - const int bUseSorter = !sqlite3TempInMemory(pParse->db); -#endif - #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, db->aDb[iDb].zName ) ){ @@ -2385,28 +2377,40 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeChangeP5(v, 1); } +#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ - if( bUseSorter ){ - iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_OpenSorter, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); - sqlite3VdbeChangeP5(v, BTREE_SORTER); - } + iSorter = pParse->nTab++; + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); +#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); + addr2 = addr1 + 1; regRecord = sqlite3GetTempReg(pParse); regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - if( bUseSorter ){ - sqlite3VdbeAddOp2(v, OP_IdxInsert, iSorter, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); - sqlite3VdbeJumpHere(v, addr1); - addr1 = sqlite3VdbeAddOp2(v, OP_Sort, iSorter, 0); - sqlite3VdbeAddOp2(v, OP_RowKey, iSorter, regRecord); +#ifndef SQLITE_OMIT_MERGE_SORT + sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); + sqlite3VdbeJumpHere(v, addr1); + addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); + if( pIndex->onError!=OE_None ){ + int j2 = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); + addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_SorterCompare, iSorter, j2, regRecord); + sqlite3HaltConstraint( + pParse, OE_Abort, "indexed columns are not unique", P4_STATIC + ); + }else{ + addr2 = sqlite3VdbeCurrentAddr(v); } - + sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#else if( pIndex->onError!=OE_None ){ const int regRowid = regIdxKey + pIndex->nColumn; const int j2 = sqlite3VdbeCurrentAddr(v) + 2; @@ -2425,10 +2429,11 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3HaltConstraint( pParse, OE_Abort, "indexed columns are not unique", P4_STATIC); } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, bUseSorter); + sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); +#endif sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3VdbeAddOp2(v, OP_Next, iSorter, addr1+1); + sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp1(v, OP_Close, iTab); diff --git a/src/ctime.c b/src/ctime.c index ca372571fc..7e854ee009 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -147,6 +147,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif #ifdef SQLITE_MEMDEBUG "MEMDEBUG", #endif diff --git a/src/expr.c b/src/expr.c index ab4547db9c..ab218078db 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2287,7 +2287,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ inReg = pCol->iMem; break; }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdx, + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, pCol->iSorterColumn, target); break; } diff --git a/src/os_common.h b/src/os_common.h index aa3e18a8c5..f6c3e7ff89 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -29,7 +29,7 @@ # error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." #endif -#ifdef SQLITE_DEBUG +#if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) # ifndef SQLITE_DEBUG_OS_TRACE # define SQLITE_DEBUG_OS_TRACE 0 # endif diff --git a/src/os_unix.c b/src/os_unix.c index 868e029f82..d85a7949b5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -2525,11 +2525,12 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ int rc = SQLITE_OK; int reserved = 0; unixFile *pFile = (unixFile*)id; + afpLockingContext *context; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); assert( pFile ); - afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; + context = (afpLockingContext *) pFile->lockingContext; if( context->reserved ){ *pResOut = 1; return SQLITE_OK; @@ -2669,7 +2670,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ ** operating system calls for the specified lock. */ if( eFileLock==SHARED_LOCK ){ - int lrc1, lrc2, lrc1Errno; + int lrc1, lrc2, lrc1Errno = 0; long lk, mask; assert( pInode->nShared==0 ); @@ -3145,11 +3146,11 @@ int sqlite3_fullsync_count = 0; /* ** We do not trust systems to provide a working fdatasync(). Some do. -** Others do no. To be safe, we will stick with the (slower) fsync(). -** If you know that your system does support fdatasync() correctly, +** Others do no. To be safe, we will stick with the (slightly slower) +** fsync(). If you know that your system does support fdatasync() correctly, ** then simply compile with -Dfdatasync=fdatasync */ -#if !defined(fdatasync) && !defined(__linux__) +#if !defined(fdatasync) # define fdatasync fsync #endif @@ -3439,26 +3440,18 @@ static int proxyFileControl(sqlite3_file*,int,void*); /* ** This function is called to handle the SQLITE_FCNTL_SIZE_HINT -** file-control operation. -** -** If the user has configured a chunk-size for this file, it could be -** that the file needs to be extended at this point. Otherwise, the -** SQLITE_FCNTL_SIZE_HINT operation is a no-op for Unix. +** file-control operation. Enlarge the database to nBytes in size +** (rounded up to the next chunk-size). If the database is already +** nBytes or larger, this routine is a no-op. */ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ - { /* preserve indentation of removed "if" */ + if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ - i64 szChunk; /* Chunk size */ struct stat buf; /* Used to hold return values of fstat() */ if( osFstat(pFile->h, &buf) ) return SQLITE_IOERR_FSTAT; - szChunk = pFile->szChunk; - if( szChunk==0 ){ - nSize = nByte; - }else{ - nSize = ((nByte+szChunk-1) / szChunk) * szChunk; - } + nSize = ((nByte+pFile->szChunk-1) / pFile->szChunk) * pFile->szChunk; if( nSize>(i64)buf.st_size ){ #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -3647,11 +3640,9 @@ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ -#ifdef SQLITE_DEBUG - u8 id; /* Id of this connection within its unixShmNode */ -#endif }; /* @@ -4957,6 +4948,9 @@ static int unixOpen( #if SQLITE_ENABLE_LOCKING_STYLE int isAutoProxy = (flags & SQLITE_OPEN_AUTOPROXY); #endif +#if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE + struct statfs fsInfo; +#endif /* If creating a master or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() @@ -5089,7 +5083,6 @@ static int unixOpen( #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE - struct statfs fsInfo; if( fstatfs(fd, &fsInfo) == -1 ){ ((unixFile*)pFile)->lastErrno = errno; robust_close(p, fd, __LINE__); @@ -5113,7 +5106,6 @@ static int unixOpen( if( envforce!=NULL ){ useProxy = atoi(envforce)>0; }else{ - struct statfs fsInfo; if( statfs(zPath, &fsInfo) == -1 ){ /* In theory, the close(fd) call is sub-optimal. If the file opened ** with fd is a database file, and there are other connections open @@ -5854,6 +5846,8 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ return SQLITE_IOERR; } } +#else + UNUSED_PARAMETER(pError); #endif #ifdef SQLITE_TEST /* simulate multiple hosts by creating unique hostid file paths */ @@ -5946,6 +5940,7 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ int nTries = 0; struct timespec conchModTime; + memset(&conchModTime, 0, sizeof(conchModTime)); do { rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); nTries ++; @@ -6177,11 +6172,12 @@ static int proxyTakeConch(unixFile *pFile){ end_takeconch: OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ + int fd; if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); } pFile->h = -1; - int fd = robust_open(pCtx->dbPath, pFile->openFlags, + fd = robust_open(pCtx->dbPath, pFile->openFlags, SQLITE_DEFAULT_FILE_PERMISSIONS); OSTRACE(("TRANSPROXY: OPEN %d\n", fd)); if( fd>=0 ){ diff --git a/src/os_win.c b/src/os_win.c index 02a7a0c622..c16198bd5d 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1216,7 +1216,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ ** actual file size after the operation may be larger than the requested ** size). */ - if( pFile->szChunk ){ + if( pFile->szChunk>0 ){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } @@ -1603,18 +1603,20 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - winFile *pFile = (winFile*)id; - sqlite3_int64 oldSz; - int rc = winFileSize(id, &oldSz); - if( rc==SQLITE_OK ){ - sqlite3_int64 newSz = *(sqlite3_int64*)pArg; - if( newSz>oldSz ){ - SimulateIOErrorBenign(1); - rc = winTruncate(id, newSz); - SimulateIOErrorBenign(0); + if( pFile->szChunk>0 ){ + sqlite3_int64 oldSz; + int rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK ){ + sqlite3_int64 newSz = *(sqlite3_int64*)pArg; + if( newSz>oldSz ){ + SimulateIOErrorBenign(1); + rc = winTruncate(id, newSz); + SimulateIOErrorBenign(0); + } } + return rc; } - return rc; + return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { int bPersist = *(int*)pArg; diff --git a/src/pager.c b/src/pager.c index 373d06aeca..f40d2af74b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -621,7 +621,6 @@ struct Pager { u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ u8 hasSeenStress; /* pagerStress() called one or more times */ - u8 isSorter; /* True for a PAGER_SORTER */ /************************************************************************** ** The following block contains those class members that change during @@ -845,15 +844,6 @@ static int assert_pager_state(Pager *p){ assert( pagerUseWal(p)==0 ); } - /* A sorter is a temp file that never spills to disk and always has - ** the doNotSpill flag set - */ - if( p->isSorter ){ - assert( p->tempFile ); - assert( p->doNotSpill ); - assert( p->fd->pMethods==0 ); - } - /* If changeCountDone is set, a RESERVED lock or greater must be held ** on the file. */ @@ -4557,12 +4547,6 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ -#ifndef SQLITE_OMIT_MERGE_SORT - if( flags & PAGER_SORTER ){ - pPager->doNotSpill = 1; - pPager->isSorter = 1; - } -#endif *ppPager = pPager; return SQLITE_OK; @@ -6107,17 +6091,6 @@ int sqlite3PagerIsMemdb(Pager *pPager){ return MEMDB; } -#ifndef SQLITE_OMIT_MERGE_SORT -/* -** Return true if the pager has seen a pagerStress callback. -*/ -int sqlite3PagerUnderStress(Pager *pPager){ - assert( pPager->isSorter ); - assert( pPager->doNotSpill ); - return pPager->hasSeenStress; -} -#endif - /* ** Check that there are at least nSavepoint savepoints open. If there are ** currently less than nSavepoints open, then open one or more savepoints diff --git a/src/pager.h b/src/pager.h index ccd7467d6a..2a02eff75c 100644 --- a/src/pager.h +++ b/src/pager.h @@ -156,9 +156,6 @@ const char *sqlite3PagerJournalname(Pager*); int sqlite3PagerNosync(Pager*); void *sqlite3PagerTempSpace(Pager*); int sqlite3PagerIsMemdb(Pager*); -#ifndef SQLITE_OMIT_MERGE_SORT -int sqlite3PagerUnderStress(Pager*); -#endif /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); diff --git a/src/select.c b/src/select.c index bd5e964e7c..a8ea08a781 100644 --- a/src/select.c +++ b/src/select.c @@ -419,12 +419,18 @@ static void pushOntoSorter( int nExpr = pOrderBy->nExpr; int regBase = sqlite3GetTempRange(pParse, nExpr+2); int regRecord = sqlite3GetTempReg(pParse); + int op; sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pOrderBy->iECursor, regRecord); + if( pSelect->selFlags & SF_UseSorter ){ + op = OP_SorterInsert; + }else{ + op = OP_IdxInsert; + } + sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); if( pSelect->iLimit ){ @@ -893,9 +899,20 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr + 1, regRow); + if( p->selFlags & SF_UseSorter ){ + int regSortOut = ++pParse->nMem; + int ptab2 = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + }else{ + addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); + codeOffset(v, p, addrContinue); + sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + } switch( eDest ){ case SRT_Table: case SRT_EphemTab: { @@ -948,7 +965,11 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + if( p->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); + } sqlite3VdbeResolveLabel(v, addrBreak); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); @@ -3914,6 +3935,10 @@ int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = (double)LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); + if( p->iLimit==0 && addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; + p->selFlags |= SF_UseSorter; + } /* Open a virtual index to use for the distinct set. */ @@ -3949,7 +3974,7 @@ int sqlite3Select( if( pWInfo->eDistinct ){ VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - assert( addrDistinctIndex>0 ); + assert( addrDistinctIndex>=0 ); pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); assert( isDistinct ); @@ -4008,6 +4033,8 @@ int sqlite3Select( int iAbortFlag; /* Mem address which causes query abort if positive */ int groupBySort; /* Rows come from source in GROUP BY order */ int addrEnd; /* End of processing for this SELECT */ + int sortPTab = 0; /* Pseudotable used to decode sorting results */ + int sortOut = 0; /* Output register from the sorter */ /* Remove any and all aliases between the result set and the ** GROUP BY clause. @@ -4069,12 +4096,12 @@ int sqlite3Select( /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out - ** that we do not need it after all, the OpenEphemeral instruction + ** that we do not need it after all, the OP_SorterOpen instruction ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO_HANDOFF); @@ -4155,11 +4182,14 @@ int sqlite3Select( } regRecord = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_IdxInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); sqlite3WhereEnd(pWInfo); - sqlite3VdbeAddOp2(v, OP_Sort, sAggInfo.sortingIdx, addrEnd); + sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; + sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); + sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); sAggInfo.useSortingIdx = 1; sqlite3ExprCacheClear(pParse); @@ -4172,9 +4202,13 @@ int sqlite3Select( */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); sqlite3ExprCacheClear(pParse); + if( groupBySort ){ + sqlite3VdbeAddOp2(v, OP_SorterData, sAggInfo.sortingIdx, sortOut); + } for(j=0; jnExpr; j++){ if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_Column, sAggInfo.sortingIdx, j, iBMem+j); + sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); + if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); @@ -4213,7 +4247,7 @@ int sqlite3Select( /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_Next, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); }else{ sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx, 1); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8c2fd2254b..3dc41d966a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -372,14 +372,6 @@ # define SQLITE_TEMP_STORE 1 #endif -/* -** If all temporary storage is in-memory, then omit the external merge-sort -** logic since it is superfluous. -*/ -#if SQLITE_TEMP_STORE==3 && !defined(SQLITE_OMIT_MERGE_SORT) -# define SQLITE_OMIT_MERGE_SORT -#endif - /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -1570,6 +1562,7 @@ struct AggInfo { u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ int sortingIdx; /* Cursor number of the sorting index */ + int sortingIdxPTab; /* Cursor number of pseudo-table */ ExprList *pGroupBy; /* The group by clause */ int nSortingColumn; /* Number of columns in the sorting index */ struct AggInfo_col { /* For each column used in source tables */ @@ -2102,6 +2095,7 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ +#define SF_UseSorter 0x0040 /* Sort using a sorter */ /* diff --git a/src/test1.c b/src/test1.c index 59b570c28a..26342522c5 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4395,7 +4395,7 @@ static u8 *sqlite3_stack_baseline = 0; static void prepStack(void){ int i; u32 bigBuf[65536]; - for(i=0; i=4 ){ + const char *zKey; + int nKey; + zKey = Tcl_GetStringFromObj(objv[3], &nKey); + rc = sqlite3_key(db, zKey, nKey); + if( rc!=SQLITE_OK ){ + char *zErrMsg = sqlite3_mprintf("error %d: %s", rc, sqlite3_errmsg(db)); + sqlite3_close(db); + Tcl_AppendResult(interp, zErrMsg, (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + } +#endif Md5_Register(db); sqlite3_busy_handler(db, xBusy, 0); @@ -349,7 +364,7 @@ static int sqlthread_proc( if( rc!=TCL_OK ) return rc; pSub = &aSub[iIndex]; - if( objc!=(pSub->nArg+2) ){ + if( objc<(pSub->nArg+2) ){ Tcl_WrongNumArgs(interp, 2, objv, pSub->zUsage); return TCL_ERROR; } diff --git a/src/vdbe.c b/src/vdbe.c index 8491306507..906058c1f8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -673,7 +673,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=p->nMem ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - sqlite3VdbeMemReleaseExternal(pOut); + MemReleaseExt(pOut); pOut->flags = MEM_Int; } @@ -2118,6 +2118,7 @@ case OP_Column: { u32 szField; /* Number of bytes in the content of a field */ int szHdr; /* Size of the header size field at start of record */ int avail; /* Number of bytes of available data */ + u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ @@ -2129,7 +2130,6 @@ case OP_Column: { assert( pOp->p3>0 && pOp->p3<=p->nMem ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); - MemSetTypeFlag(pDest, MEM_Null); zRec = 0; /* This block sets the variable payloadSize to be the total number of @@ -2173,7 +2173,7 @@ case OP_Column: { rc = sqlite3BtreeDataSize(pCrsr, &payloadSize); assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ } - }else if( pC->pseudoTableReg>0 ){ + }else if( ALWAYS(pC->pseudoTableReg>0) ){ pReg = &aMem[pC->pseudoTableReg]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); @@ -2186,9 +2186,10 @@ case OP_Column: { payloadSize = 0; } - /* If payloadSize is 0, then just store a NULL */ + /* If payloadSize is 0, then just store a NULL. This can happen because of + ** nullRow or because of a corrupt database. */ if( payloadSize==0 ){ - assert( pDest->flags&MEM_Null ); + MemSetTypeFlag(pDest, MEM_Null); goto op_column_out; } assert( db->aLimit[SQLITE_LIMIT_LENGTH]>=0 ); @@ -2295,8 +2296,14 @@ case OP_Column: { for(i=0; ip4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - assert( pDest->flags&MEM_Null ); + MemSetTypeFlag(pDest, MEM_Null); } } @@ -2550,7 +2557,7 @@ case OP_Count: { /* out2-prerelease */ BtCursor *pCrsr; pCrsr = p->apCsr[pOp->p1]->pCursor; - if( pCrsr ){ + if( ALWAYS(pCrsr) ){ rc = sqlite3BtreeCount(pCrsr, &nEntry); }else{ nEntry = 0; @@ -3112,15 +3119,9 @@ case OP_OpenWrite: { rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; - /* Since it performs no memory allocation or IO, the only values that - ** sqlite3BtreeCursor() may return are SQLITE_EMPTY and SQLITE_OK. - ** SQLITE_EMPTY is only returned when attempting to open the table - ** rooted at page 1 of a zero-byte database. */ - assert( rc==SQLITE_EMPTY || rc==SQLITE_OK ); - if( rc==SQLITE_EMPTY ){ - pCur->pCursor = 0; - rc = SQLITE_OK; - } + /* Since it performs no memory allocation or IO, the only value that + ** sqlite3BtreeCursor() may return is SQLITE_OK. */ + assert( rc==SQLITE_OK ); /* Set the VdbeCursor.isTable and isIndex variables. Previous versions of ** SQLite used to check if the root-page flags were sane at this point @@ -3161,13 +3162,6 @@ case OP_OpenWrite: { ** by this opcode will be used for automatically created transient ** indices in joins. */ -/* Opcode: OpenSorter P1 P2 * P4 * -** -** This opcode works like OP_OpenEphemeral except that it opens -** a transient index that is specifically designed to sort large -** tables using an external merge-sort algorithm. -*/ -case OP_OpenSorter: case OP_OpenAutoindex: case OP_OpenEphemeral: { VdbeCursor *pCx; @@ -3179,7 +3173,6 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); - assert( (pOp->opcode==OP_OpenSorter)==((pOp->p5 & BTREE_SORTER)!=0) ); pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; @@ -3213,10 +3206,27 @@ case OP_OpenEphemeral: { } pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); pCx->isIndex = !pCx->isTable; + break; +} + +/* Opcode: OpenSorter P1 P2 * P4 * +** +** This opcode works like OP_OpenEphemeral except that it opens +** a transient index that is specifically designed to sort large +** tables using an external merge-sort algorithm. +*/ +case OP_SorterOpen: { + VdbeCursor *pCx; #ifndef SQLITE_OMIT_MERGE_SORT - if( rc==SQLITE_OK && pOp->opcode==OP_OpenSorter ){ - rc = sqlite3VdbeSorterInit(db, pCx); - } + pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->pKeyInfo = pOp->p4.pKeyInfo; + pCx->pKeyInfo->enc = ENC(p->db); + pCx->isSorter = 1; + rc = sqlite3VdbeSorterInit(db, pCx); +#else + pOp->opcode = OP_OpenEphemeral; + pc--; #endif break; } @@ -3333,7 +3343,7 @@ case OP_SeekGt: { /* jump, in3 */ assert( OP_SeekGe == OP_SeekLt+2 ); assert( OP_SeekGt == OP_SeekLt+3 ); assert( pC->isOrdered ); - if( pC->pCursor!=0 ){ + if( ALWAYS(pC->pCursor!=0) ){ oc = pOp->opcode; pC->nullRow = 0; if( pC->isTable ){ @@ -3521,6 +3531,7 @@ case OP_Found: { /* jump, in3 */ int alreadyExists; VdbeCursor *pC; int res; + char *pFree; UnpackedRecord *pIdxKey; UnpackedRecord r; char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*3 + 7]; @@ -3548,18 +3559,18 @@ case OP_Found: { /* jump, in3 */ r.flags = UNPACKED_PREFIX_MATCH; pIdxKey = &r; }else{ + pIdxKey = sqlite3VdbeAllocUnpackedRecord( + pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree + ); + if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ - pIdxKey = sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, - aTempRec, sizeof(aTempRec)); - if( pIdxKey==0 ){ - goto no_mem; - } + sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); pIdxKey->flags |= UNPACKED_PREFIX_MATCH; } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, pIdxKey, 0, 0, &res); if( pOp->p4.i==0 ){ - sqlite3VdbeDeleteUnpackedRecord(pIdxKey); + sqlite3DbFree(db, pFree); } if( rc!=SQLITE_OK ){ break; @@ -3691,7 +3702,7 @@ case OP_NotExists: { /* jump, in3 */ assert( pC->isTable ); assert( pC->pseudoTableReg==0 ); pCrsr = pC->pCursor; - if( pCrsr!=0 ){ + if( ALWAYS(pCrsr!=0) ){ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); @@ -4069,6 +4080,45 @@ case OP_ResetCount: { break; } +/* Opcode: SorterCompare P1 P2 P3 +** +** P1 is a sorter cursor. This instruction compares the record blob in +** register P3 with the entry that the sorter cursor currently points to. +** If, excluding the rowid fields at the end, the two records are a match, +** fall through to the next instruction. Otherwise, jump to instruction P2. +*/ +case OP_SorterCompare: { + VdbeCursor *pC; + int res; + + pC = p->apCsr[pOp->p1]; + assert( isSorter(pC) ); + pIn3 = &aMem[pOp->p3]; + rc = sqlite3VdbeSorterCompare(pC, pIn3, &res); + if( res ){ + pc = pOp->p2-1; + } + break; +}; + +/* Opcode: SorterData P1 P2 * * * +** +** Write into register P2 the current sorter data for sorter cursor P1. +*/ +case OP_SorterData: { + VdbeCursor *pC; +#ifndef SQLITE_OMIT_MERGE_SORT + pOut = &aMem[pOp->p2]; + pC = p->apCsr[pOp->p1]; + assert( pC->isSorter ); + rc = sqlite3VdbeSorterRowkey(pC, pOut); +#else + pOp->opcode = OP_RowKey; + pc--; +#endif + break; +} + /* Opcode: RowData P1 P2 * * * ** ** Write into register P2 the complete row data for cursor P1. @@ -4102,18 +4152,13 @@ case OP_RowData: { /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC->isTable || pOp->opcode==OP_RowKey ); + assert( pC->isSorter==0 ); + assert( pC->isTable || pOp->opcode!=OP_RowData ); assert( pC->isIndex || pOp->opcode==OP_RowData ); assert( pC!=0 ); assert( pC->nullRow==0 ); assert( pC->pseudoTableReg==0 ); - - if( isSorter(pC) ){ - assert( pOp->opcode==OP_RowKey ); - rc = sqlite3VdbeSorterRowkey(pC, pOut); - break; - } - + assert( !pC->isSorter ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; assert( sqlite3BtreeCursorIsValid(pCrsr) ); @@ -4218,6 +4263,7 @@ case OP_NullRow: { assert( pC!=0 ); pC->nullRow = 1; pC->rowidIsValid = 0; + assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } @@ -4241,7 +4287,7 @@ case OP_Last: { /* jump */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pCrsr = pC->pCursor; - if( pCrsr==0 ){ + if( NEVER(pCrsr==0) ){ res = 1; }else{ rc = sqlite3BtreeLast(pCrsr, &res); @@ -4269,6 +4315,10 @@ case OP_Last: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ +case OP_SorterSort: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Sort; +#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4293,10 +4343,13 @@ case OP_Rewind: { /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(db, pC, &res); - }else if( (pCrsr = pC->pCursor)!=0 ){ + }else{ + pCrsr = pC->pCursor; + assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); pC->atFirst = res==0 ?1:0; pC->deferredMoveto = 0; @@ -4311,7 +4364,7 @@ case OP_Rewind: { /* jump */ break; } -/* Opcode: Next P1 P2 * * P5 +/* Opcode: Next P1 P2 * P4 P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through @@ -4320,6 +4373,9 @@ case OP_Rewind: { /* jump */ ** ** The P1 cursor must be for a real table, not a pseudo-table. ** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreeNext(). +** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** @@ -4334,13 +4390,19 @@ case OP_Rewind: { /* jump */ ** ** The P1 cursor must be for a real table, not a pseudo-table. ** +** P4 is always of type P4_ADVANCE. The function pointer points to +** sqlite3BtreePrevious(). +** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ +case OP_SorterNext: /* jump */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_Next; +#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; - BtCursor *pCrsr; int res; CHECK_FOR_INTERRUPT; @@ -4350,19 +4412,17 @@ case OP_Next: { /* jump */ if( pC==0 ){ break; /* See ticket #2273 */ } + assert( pC->isSorter==(pOp->opcode==OP_SorterNext) ); if( isSorter(pC) ){ - assert( pOp->opcode==OP_Next ); + assert( pOp->opcode==OP_SorterNext ); rc = sqlite3VdbeSorterNext(db, pC, &res); }else{ - pCrsr = pC->pCursor; - if( pCrsr==0 ){ - pC->nullRow = 1; - break; - } res = 1; assert( pC->deferredMoveto==0 ); - rc = pOp->opcode==OP_Next ? sqlite3BtreeNext(pCrsr, &res) : - sqlite3BtreePrevious(pCrsr, &res); + assert( pC->pCursor ); + assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); + assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); + rc = pOp->p4.xAdvance(pC->pCursor, &res); } pC->nullRow = (u8)res; pC->cacheStatus = CACHE_STALE; @@ -4389,6 +4449,10 @@ case OP_Next: { /* jump */ ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ +case OP_SorterInsert: /* in2 */ +#ifdef SQLITE_OMIT_MERGE_SORT + pOp->opcode = OP_IdxInsert; +#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; @@ -4398,6 +4462,7 @@ case OP_IdxInsert: { /* in2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isSorter==(pOp->opcode==OP_SorterInsert) ); pIn2 = &aMem[pOp->p2]; assert( pIn2->flags & MEM_Blob ); pCrsr = pC->pCursor; @@ -4405,16 +4470,17 @@ case OP_IdxInsert: { /* in2 */ assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3VdbeSorterWrite(db, pC, nKey); - if( rc==SQLITE_OK ){ + if( isSorter(pC) ){ + rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + }else{ + nKey = pIn2->n; + zKey = pIn2->z; rc = sqlite3BtreeInsert(pCrsr, zKey, nKey, "", 0, 0, pOp->p3, ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); + ); assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; } - pC->cacheStatus = CACHE_STALE; } } break; diff --git a/src/vdbe.h b/src/vdbe.h index fe758d2dbd..f4691c0e8f 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -61,6 +61,7 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_DEBUG char *zComment; /* Comment to improve readability */ @@ -116,6 +117,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ +#define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ /* When adding a P4 argument using P4_KEYINFO, a copy of the KeyInfo structure ** is made. That copy is freed when the Vdbe is finalized. But if the @@ -210,9 +212,9 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif -UnpackedRecord *sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,char*,int); -void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord*); +void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); #ifndef SQLITE_OMIT_TRIGGER void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 11a22bc972..7fbd5693fb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -59,12 +59,13 @@ struct VdbeCursor { Bool isTable; /* True if a table requiring integer keys */ Bool isIndex; /* True if an index containing keys only - no data */ Bool isOrdered; /* True if the underlying table is BTREE_UNORDERED */ + Bool isSorter; /* True if a new-style sorter */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ const sqlite3_module *pModule; /* Module for cursor pVtabCursor */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ i64 lastRowid; /* Last rowid from a Next or NextIdx operation */ - VdbeSorter *pSorter; /* Sorter object for OP_OpenSorter cursors */ + VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Result of last sqlite3BtreeMoveto() done by an OP_NotExists or ** OP_IsUnique opcode on this cursor. */ @@ -384,6 +385,9 @@ int sqlite3VdbeMemNumerify(Mem*); int sqlite3VdbeMemFromBtree(BtCursor*,int,int,int,Mem*); void sqlite3VdbeMemRelease(Mem *p); void sqlite3VdbeMemReleaseExternal(Mem *p); +#define MemReleaseExt(X) \ + if((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame)) \ + sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); @@ -399,13 +403,15 @@ void sqlite3VdbeMemStoreType(Mem *pMem); # define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK # define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK # define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK +# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK #else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); -int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, int); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(VdbeCursor *, Mem *); -int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); int sqlite3VdbeSorterNext(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterRewind(sqlite3 *, VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(sqlite3 *, VdbeCursor *, Mem *); +int sqlite3VdbeSorterCompare(VdbeCursor *, Mem *, int *); #endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 548a3dca1b..dc81f3aed7 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -433,6 +433,12 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; #endif + }else if( opcode==OP_Next || opcode==OP_SorterNext ){ + pOp->p4.xAdvance = sqlite3BtreeNext; + pOp->p4type = P4_ADVANCE; + }else if( opcode==OP_Prev ){ + pOp->p4.xAdvance = sqlite3BtreePrevious; + pOp->p4type = P4_ADVANCE; } if( (pOp->opflags & OPFLG_JUMP)!=0 && pOp->p2<0 ){ @@ -939,6 +945,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ sqlite3_snprintf(nTemp, zTemp, "program"); break; } + case P4_ADVANCE: { + zTemp[0] = 0; + break; + } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -2820,57 +2830,70 @@ u32 sqlite3VdbeSerialGet( return 0; } - /* -** Given the nKey-byte encoding of a record in pKey[], parse the -** record into a UnpackedRecord structure. Return a pointer to -** that structure. +** This routine is used to allocate sufficient space for an UnpackedRecord +** structure large enough to be used with sqlite3VdbeRecordUnpack() if +** the first argument is a pointer to KeyInfo structure pKeyInfo. ** -** The calling function might provide szSpace bytes of memory -** space at pSpace. This space can be used to hold the returned -** VDbeParsedRecord structure if it is large enough. If it is -** not big enough, space is obtained from sqlite3_malloc(). +** The space is either allocated using sqlite3DbMallocRaw() or from within +** the unaligned buffer passed via the second and third arguments (presumably +** stack space). If the former, then *ppFree is set to a pointer that should +** be eventually freed by the caller using sqlite3DbFree(). Or, if the +** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL +** before returning. ** -** The returned structure should be closed by a call to -** sqlite3VdbeDeleteUnpackedRecord(). -*/ -UnpackedRecord *sqlite3VdbeRecordUnpack( - KeyInfo *pKeyInfo, /* Information about the record format */ - int nKey, /* Size of the binary record */ - const void *pKey, /* The binary record */ - char *pSpace, /* Unaligned space available to hold the object */ - int szSpace /* Size of pSpace[] in bytes */ +** If an OOM error occurs, NULL is returned. +*/ +UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( + KeyInfo *pKeyInfo, /* Description of the record */ + char *pSpace, /* Unaligned space available */ + int szSpace, /* Size of pSpace[] in bytes */ + char **ppFree /* OUT: Caller should free this pointer */ ){ - const unsigned char *aKey = (const unsigned char *)pKey; - UnpackedRecord *p; /* The unpacked record that we will return */ - int nByte; /* Memory space needed to hold p, in bytes */ - int d; - u32 idx; - u16 u; /* Unsigned loop counter */ - u32 szHdr; - Mem *pMem; - int nOff; /* Increase pSpace by this much to 8-byte align it */ - - /* - ** We want to shift the pointer pSpace up such that it is 8-byte aligned. + UnpackedRecord *p; /* Unpacked record to return */ + int nOff; /* Increment pSpace by nOff to align it */ + int nByte; /* Number of bytes required for *p */ + + /* We want to shift the pointer pSpace up such that it is 8-byte aligned. ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift ** it by. If pSpace is already 8-byte aligned, nOff should be zero. */ nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; - pSpace += nOff; - szSpace -= nOff; nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); - if( nByte>szSpace ){ - p = sqlite3DbMallocRaw(pKeyInfo->db, nByte); - if( p==0 ) return 0; - p->flags = UNPACKED_NEED_FREE | UNPACKED_NEED_DESTROY; + if( nByte>szSpace+nOff ){ + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + *ppFree = (char *)p; + if( !p ) return 0; }else{ - p = (UnpackedRecord*)pSpace; - p->flags = UNPACKED_NEED_DESTROY; + p = (UnpackedRecord*)&pSpace[nOff]; + *ppFree = 0; } + + p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; p->pKeyInfo = pKeyInfo; p->nField = pKeyInfo->nField + 1; - p->aMem = pMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; + return p; +} + +/* +** Given the nKey-byte encoding of a record in pKey[], populate the +** UnpackedRecord structure indicated by the fourth argument with the +** contents of the decoded record. +*/ +void sqlite3VdbeRecordUnpack( + KeyInfo *pKeyInfo, /* Information about the record format */ + int nKey, /* Size of the binary record */ + const void *pKey, /* The binary record */ + UnpackedRecord *p /* Populate this structure before returning. */ +){ + const unsigned char *aKey = (const unsigned char *)pKey; + int d; + u32 idx; /* Offset in aKey[] to read from */ + u16 u; /* Unsigned loop counter */ + u32 szHdr; + Mem *pMem = p->aMem; + + p->flags = 0; assert( EIGHT_BYTE_ALIGNMENT(pMem) ); idx = getVarint32(aKey, szHdr); d = szHdr; @@ -2889,31 +2912,6 @@ UnpackedRecord *sqlite3VdbeRecordUnpack( } assert( u<=pKeyInfo->nField + 1 ); p->nField = u; - return (void*)p; -} - -/* -** This routine destroys a UnpackedRecord object. -*/ -void sqlite3VdbeDeleteUnpackedRecord(UnpackedRecord *p){ -#ifdef SQLITE_DEBUG - int i; - Mem *pMem; - - assert( p!=0 ); - assert( p->flags & UNPACKED_NEED_DESTROY ); - for(i=0, pMem=p->aMem; inField; i++, pMem++){ - /* The unpacked record is always constructed by the - ** sqlite3VdbeUnpackRecord() function above, which makes all - ** strings and blobs static. And none of the elements are - ** ever transformed, so there is never anything to delete. - */ - if( NEVER(pMem->zMalloc) ) sqlite3VdbeMemRelease(pMem); - } -#endif - if( p->flags & UNPACKED_NEED_FREE ){ - sqlite3DbFree(p->pKeyInfo->db, p); - } } /* diff --git a/src/vdbemem.c b/src/vdbemem.c index 9c964a2581..e6e9156272 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -271,24 +271,18 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ */ void sqlite3VdbeMemReleaseExternal(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); - testcase( p->flags & MEM_Agg ); - testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_RowSet ); - testcase( p->flags & MEM_Frame ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame) ){ - if( p->flags&MEM_Agg ){ - sqlite3VdbeMemFinalize(p, p->u.pDef); - assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn && p->xDel ){ - assert( (p->flags&MEM_RowSet)==0 ); - p->xDel((void *)p->z); - p->xDel = 0; - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); - } + if( p->flags&MEM_Agg ){ + sqlite3VdbeMemFinalize(p, p->u.pDef); + assert( (p->flags & MEM_Agg)==0 ); + sqlite3VdbeMemRelease(p); + }else if( p->flags&MEM_Dyn && p->xDel ){ + assert( (p->flags&MEM_RowSet)==0 ); + p->xDel((void *)p->z); + p->xDel = 0; + }else if( p->flags&MEM_RowSet ){ + sqlite3RowSetClear(p->u.pRowSet); + }else if( p->flags&MEM_Frame ){ + sqlite3VdbeMemSetNull(p); } } @@ -298,7 +292,7 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** (Mem.type==SQLITE_TEXT). */ void sqlite3VdbeMemRelease(Mem *p){ - sqlite3VdbeMemReleaseExternal(p); + MemReleaseExt(p); sqlite3DbFree(p->db, p->zMalloc); p->z = 0; p->zMalloc = 0; @@ -620,7 +614,7 @@ void sqlite3VdbeMemPrepareToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); + MemReleaseExt(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ @@ -638,7 +632,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; assert( (pFrom->flags & MEM_RowSet)==0 ); - sqlite3VdbeMemReleaseExternal(pTo); + MemReleaseExt(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; diff --git a/src/vdbesort.c b/src/vdbesort.c index be99d397d3..ead2f453e4 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -21,6 +21,7 @@ #ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterRecord SorterRecord; /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: @@ -92,8 +93,7 @@ typedef struct VdbeSorterIter VdbeSorterIter; ** being merged (rounded up to the next power of 2). */ struct VdbeSorter { - int nWorking; /* Start a new b-tree after this many pages */ - int nBtree; /* Current size of b-tree contents as PMA */ + int nInMemory; /* Current size of pRecord list as PMA */ int nTree; /* Used size of aTree/aIter (power of 2) */ VdbeSorterIter *aIter; /* Array of iterators to merge */ int *aTree; /* Current state of incremental merge */ @@ -101,6 +101,10 @@ struct VdbeSorter { i64 iReadOff; /* Current read offset within file pTemp1 */ sqlite3_file *pTemp1; /* PMA file 1 */ int nPMA; /* Number of PMAs stored in pTemp1 */ + SorterRecord *pRecord; /* Head of in-memory record list */ + int mnPmaSize; /* Minimum PMA size, in bytes */ + int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ + UnpackedRecord *pUnpacked; /* Used to unpack keys */ }; /* @@ -117,6 +121,17 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ }; +/* +** A structure to store a single record. All in-memory records are connected +** together into a linked list headed at VdbeSorter.pRecord using the +** SorterRecord.pNext pointer. +*/ +struct SorterRecord { + void *pVal; + int nVal; + SorterRecord *pNext; +}; + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -142,8 +157,8 @@ static int vdbeSorterIterNext( ){ int rc; /* Return Code */ int nRead; /* Number of bytes read */ - int nRec; /* Size of record in bytes */ - int iOff; /* Size of serialized size varint in bytes */ + int nRec = 0; /* Size of record in bytes */ + int iOff = 0; /* Size of serialized size varint in bytes */ nRead = pIter->iEof - pIter->iReadOff; if( nRead>5 ) nRead = 5; @@ -154,25 +169,26 @@ static int vdbeSorterIterNext( } rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff); - iOff = getVarint32(pIter->aAlloc, nRec); - - if( rc==SQLITE_OK && (iOff+nRec)>nRead ){ - int nRead2; /* Number of extra bytes to read */ - if( (iOff+nRec)>pIter->nAlloc ){ - int nNew = pIter->nAlloc*2; - while( (iOff+nRec)>nNew ) nNew = nNew*2; - pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); - if( !pIter->aAlloc ) return SQLITE_NOMEM; - pIter->nAlloc = nNew; + if( rc==SQLITE_OK ){ + iOff = getVarint32(pIter->aAlloc, nRec); + if( (iOff+nRec)>nRead ){ + int nRead2; /* Number of extra bytes to read */ + if( (iOff+nRec)>pIter->nAlloc ){ + int nNew = pIter->nAlloc*2; + while( (iOff+nRec)>nNew ) nNew = nNew*2; + pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew); + if( !pIter->aAlloc ) return SQLITE_NOMEM; + pIter->nAlloc = nNew; + } + + nRead2 = iOff + nRec - nRead; + rc = sqlite3OsRead( + pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead + ); } - - nRead2 = iOff + nRec - nRead; - rc = sqlite3OsRead( - pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead - ); } - assert( nRec>0 || rc!=SQLITE_OK ); + assert( rc!=SQLITE_OK || nRec>0 ); pIter->iReadOff += iOff+nRec; pIter->nKey = nRec; pIter->aKey = &pIter->aAlloc[iOff]; @@ -216,21 +232,14 @@ static int vdbeSorterWriteVarint( */ static int vdbeSorterReadVarint( sqlite3_file *pFile, /* File to read from */ - i64 iEof, /* Total number of bytes in file */ i64 *piOffset, /* IN/OUT: Read offset in pFile */ i64 *piVal /* OUT: Value read from file */ ){ u8 aVarint[9]; /* Buffer large enough for a varint */ i64 iOff = *piOffset; /* Offset in file to read from */ - int nRead = 9; /* Number of bytes to read from file */ int rc; /* Return code */ - assert( iEof>iOff ); - if( (iEof-iOff)aAlloc ){ rc = SQLITE_NOMEM; }else{ - i64 iEof = pSorter->iWriteOff; /* EOF of file pSorter->pTemp1 */ i64 nByte; /* Total size of PMA in bytes */ - rc = vdbeSorterReadVarint(pSorter->pTemp1, iEof, &pIter->iReadOff, &nByte); + rc = vdbeSorterReadVarint(pSorter->pTemp1, &pIter->iReadOff, &nByte); *pnByte += nByte; pIter->iEof = pIter->iReadOff + nByte; } @@ -274,6 +282,54 @@ static int vdbeSorterIterInit( return rc; } + +/* +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** size nKey2 bytes). Argument pKeyInfo supplies the collation functions +** used by the comparison. If an error occurs, return an SQLite error code. +** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive +** value, depending on whether key1 is smaller, equal to or larger than key2. +** +** If the bOmitRowid argument is non-zero, assume both keys end in a rowid +** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid +** is true and key1 contains even a single NULL value, it is considered to +** be less than key2. Even if key2 also contains NULL values. +** +** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace +** has been allocated and contains an unpacked record that is used as key2. +*/ +static int vdbeSorterCompare( + VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + int bOmitRowid, /* Ignore rowid field at end of keys */ + void *pKey1, int nKey1, /* Left side of comparison */ + void *pKey2, int nKey2, /* Right side of comparison */ + int *pRes /* OUT: Result of comparison */ +){ + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + VdbeSorter *pSorter = pCsr->pSorter; + UnpackedRecord *r2 = pSorter->pUnpacked; + int i; + + if( pKey2 ){ + sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + } + + if( bOmitRowid ){ + r2->nField = pKeyInfo->nField; + assert( r2->nField>0 ); + for(i=0; inField; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + r2->flags |= UNPACKED_PREFIX_MATCH; + } + + *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2); + return SQLITE_OK; +} + /* ** This function is called to compare two iterator keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] @@ -305,20 +361,21 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ }else if( p2->pFile==0 ){ iRes = i1; }else{ - char aSpace[150]; - UnpackedRecord *r1; - - r1 = sqlite3VdbeRecordUnpack( - pCsr->pKeyInfo, p1->nKey, p1->aKey, aSpace, sizeof(aSpace) + int res; + int rc; + assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + rc = vdbeSorterCompare( + pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); - if( r1==0 ) return SQLITE_NOMEM; + /* The vdbeSorterCompare() call cannot fail since pCsr->pSorter->pUnpacked + ** has already been allocated. */ + assert( rc==SQLITE_OK ); - if( sqlite3VdbeRecordCompare(p2->nKey, p2->aKey, r1)>=0 ){ + if( res<=0 ){ iRes = i1; }else{ iRes = i2; } - sqlite3VdbeDeleteUnpackedRecord(r1); } pSorter->aTree[iOut] = iRes; @@ -329,9 +386,42 @@ static int vdbeSorterDoCompare(VdbeCursor *pCsr, int iOut){ ** Initialize the temporary index cursor just opened as a sorter cursor. */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ - assert( pCsr->pKeyInfo && pCsr->pBt ); - pCsr->pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); - return (pCsr->pSorter ? SQLITE_OK : SQLITE_NOMEM); + int pgsz; /* Page size of main database */ + int mxCache; /* Cache size */ + VdbeSorter *pSorter; /* The new sorter */ + char *d; /* Dummy */ + + assert( pCsr->pKeyInfo && pCsr->pBt==0 ); + pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + if( pSorter==0 ){ + return SQLITE_NOMEM; + } + + pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); + if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; + assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + + if( !sqlite3TempInMemory(db) ){ + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + mxCache = db->aDb[0].pSchema->cache_size; + if( mxCachemxPmaSize = mxCache * pgsz; + } + + return SQLITE_OK; +} + +/* +** Free the list of sorted records starting at pRecord. +*/ +static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ + SorterRecord *p; + SorterRecord *pNext; + for(p=pRecord; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } } /* @@ -350,6 +440,8 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter->pTemp1 ){ sqlite3OsCloseFree(pSorter->pTemp1); } + vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3DbFree(db, pSorter->pUnpacked); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -369,10 +461,103 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ); } +/* +** Attemp to merge the two sorted lists p1 and p2 into a single list. If no +** error occurs set *ppOut to the head of the new list and return SQLITE_OK. +*/ +static int vdbeSorterMerge( + sqlite3 *db, /* Database handle */ + VdbeCursor *pCsr, /* For pKeyInfo */ + SorterRecord *p1, /* First list to merge */ + SorterRecord *p2, /* Second list to merge */ + SorterRecord **ppOut /* OUT: Head of merged list */ +){ + int rc = SQLITE_OK; + SorterRecord *pFinal = 0; + SorterRecord **pp = &pFinal; + void *pVal2 = p2 ? p2->pVal : 0; + + while( p1 && p2 ){ + int res; + rc = vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + if( rc!=SQLITE_OK ){ + *pp = 0; + vdbeSorterRecordFree(db, p1); + vdbeSorterRecordFree(db, p2); + vdbeSorterRecordFree(db, pFinal); + *ppOut = 0; + return rc; + } + if( res<=0 ){ + *pp = p1; + pp = &p1->pNext; + p1 = p1->pNext; + pVal2 = 0; + }else{ + *pp = p2; + pp = &p2->pNext; + p2 = p2->pNext; + if( p2==0 ) break; + pVal2 = p2->pVal; + } + } + *pp = p1 ? p1 : p2; + + *ppOut = pFinal; + return SQLITE_OK; +} /* -** Write the current contents of the b-tree to a PMA. Return SQLITE_OK -** if successful, or an SQLite error code otherwise. +** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK +** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error +** occurs. +*/ +static int vdbeSorterSort(sqlite3 *db, VdbeCursor *pCsr){ + int rc = SQLITE_OK; + int i; + SorterRecord **aSlot; + SorterRecord *p; + VdbeSorter *pSorter = pCsr->pSorter; + + aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); + if( !aSlot ){ + return SQLITE_NOMEM; + } + + p = pSorter->pRecord; + while( p ){ + SorterRecord *pNext = p->pNext; + p->pNext = 0; + for(i=0; rc==SQLITE_OK && aSlot[i]; i++){ + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); + aSlot[i] = 0; + } + if( rc!=SQLITE_OK ){ + vdbeSorterRecordFree(db, pNext); + break; + } + aSlot[i] = p; + p = pNext; + } + + p = 0; + for(i=0; i<64; i++){ + if( rc==SQLITE_OK ){ + rc = vdbeSorterMerge(db, pCsr, p, aSlot[i], &p); + }else{ + vdbeSorterRecordFree(db, aSlot[i]); + } + } + pSorter->pRecord = p; + + sqlite3_free(aSlot); + return rc; +} + + +/* +** Write the current contents of the in-memory linked-list to a PMA. Return +** SQLITE_OK if successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -383,19 +568,19 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){ int rc = SQLITE_OK; /* Return code */ VdbeSorter *pSorter = pCsr->pSorter; - int res = 0; - /* sqlite3BtreeFirst() cannot fail because sorter btrees are always held - ** in memory and so an I/O error is not possible. */ - rc = sqlite3BtreeFirst(pCsr->pCursor, &res); - if( NEVER(rc!=SQLITE_OK) || res ) return rc; - assert( pSorter->nBtree>0 ); + if( pSorter->nInMemory==0 ){ + assert( pSorter->pRecord==0 ); + return rc; + } + + rc = vdbeSorterSort(db, pCsr); /* If the first temporary PMA file has not been opened, open it now. */ - if( pSorter->pTemp1==0 ){ + if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); assert( rc!=SQLITE_OK || pSorter->pTemp1 ); assert( pSorter->iWriteOff==0 ); @@ -403,129 +588,87 @@ static int vdbeSorterBtreeToPMA(sqlite3 *db, VdbeCursor *pCsr){ } if( rc==SQLITE_OK ){ - i64 iWriteOff = pSorter->iWriteOff; - void *aMalloc = 0; /* Array used to hold a single record */ - int nMalloc = 0; /* Allocated size of aMalloc[] in bytes */ + i64 iOff = pSorter->iWriteOff; + SorterRecord *p; + SorterRecord *pNext = 0; + static const char eightZeros[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; pSorter->nPMA++; - for( - rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nBtree, &iWriteOff); - rc==SQLITE_OK && res==0; - rc = sqlite3BtreeNext(pCsr->pCursor, &res) - ){ - i64 nKey; /* Size of this key in bytes */ + rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff); + for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){ + pNext = p->pNext; + rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff); - /* Write the size of the record in bytes to the output file */ - (void)sqlite3BtreeKeySize(pCsr->pCursor, &nKey); - rc = vdbeSorterWriteVarint(pSorter->pTemp1, nKey, &iWriteOff); - - /* Make sure the aMalloc[] buffer is large enough for the record */ - if( rc==SQLITE_OK && nKey>nMalloc ){ - aMalloc = sqlite3DbReallocOrFree(db, aMalloc, nKey); - if( !aMalloc ){ - rc = SQLITE_NOMEM; - }else{ - nMalloc = nKey; - } - } - - /* Write the record itself to the output file */ if( rc==SQLITE_OK ){ - /* sqlite3BtreeKey() cannot fail because sorter btrees held in memory */ - rc = sqlite3BtreeKey(pCsr->pCursor, 0, nKey, aMalloc); - if( ALWAYS(rc==SQLITE_OK) ){ - rc = sqlite3OsWrite(pSorter->pTemp1, aMalloc, nKey, iWriteOff); - iWriteOff += nKey; - } + rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff); + iOff += p->nVal; } - if( rc!=SQLITE_OK ) break; + sqlite3DbFree(db, p); } /* This assert verifies that unless an error has occurred, the size of ** the PMA on disk is the same as the expected size stored in - ** pSorter->nBtree. */ - assert( rc!=SQLITE_OK || pSorter->nBtree==( - iWriteOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nBtree) + ** pSorter->nInMemory. */ + assert( rc!=SQLITE_OK || pSorter->nInMemory==( + iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory) )); - pSorter->iWriteOff = iWriteOff; - sqlite3DbFree(db, aMalloc); + pSorter->iWriteOff = iOff; + if( rc==SQLITE_OK ){ + /* Terminate each file with 8 extra bytes so that from any offset + ** in the file we can always read 9 bytes without a SHORT_READ error */ + rc = sqlite3OsWrite(pSorter->pTemp1, eightZeros, 8, iOff); + } + pSorter->pRecord = p; } - pSorter->nBtree = 0; return rc; } /* -** This function is called on a sorter cursor by the VDBE before each row -** is inserted into VdbeCursor.pCsr. Argument nKey is the size of the key, in -** bytes, about to be inserted. -** -** If it is determined that the temporary b-tree accessed via VdbeCursor.pCsr -** is large enough, its contents are written to a sorted PMA on disk and the -** tree emptied. This prevents the b-tree (which must be small enough to -** fit entirely in the cache in order to support efficient inserts) from -** growing too large. -** -** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. +** Add a record to the sorter. */ -int sqlite3VdbeSorterWrite(sqlite3 *db, VdbeCursor *pCsr, int nKey){ - int rc = SQLITE_OK; /* Return code */ +int sqlite3VdbeSorterWrite( + sqlite3 *db, /* Database handle */ + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal /* Memory cell containing record */ +){ VdbeSorter *pSorter = pCsr->pSorter; - if( pSorter ){ - Pager *pPager = sqlite3BtreePager(pCsr->pBt); - int nPage; /* Current size of temporary file in pages */ + int rc = SQLITE_OK; /* Return Code */ + SorterRecord *pNew; /* New list element */ - /* Sorters never spill to disk */ - assert( sqlite3PagerFile(pPager)->pMethods==0 ); + assert( pSorter ); + pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - /* Determine how many pages the temporary b-tree has grown to */ - sqlite3PagerPagecount(pPager, &nPage); - - /* If pSorter->nWorking is still zero, but the temporary file has been - ** created in the file-system, then the most recent insert into the - ** current b-tree segment probably caused the cache to overflow (it is - ** also possible that sqlite3_release_memory() was called). So set the - ** size of the working set to a little less than the current size of the - ** file in pages. */ - if( pSorter->nWorking==0 && sqlite3PagerUnderStress(pPager) ){ - pSorter->nWorking = nPage-5; - if( pSorter->nWorkingnWorking = SORTER_MIN_WORKING; - } - } - - /* If the number of pages used by the current b-tree segment is greater - ** than the size of the working set (VdbeSorter.nWorking), start a new - ** segment b-tree. */ - if( pSorter->nWorking && nPage>=pSorter->nWorking ){ - BtCursor *p = pCsr->pCursor;/* Cursor structure to close and reopen */ - int iRoot; /* Root page of new tree */ - - /* Copy the current contents of the b-tree into a PMA in sorted order. - ** Close the currently open b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(p); - - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeDropTable(pCsr->pBt, 2, 0); -#ifdef SQLITE_DEBUG - sqlite3PagerPagecount(pPager, &nPage); - assert( rc!=SQLITE_OK || nPage==1 ); -#endif - } - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeCreateTable(pCsr->pBt, &iRoot, BTREE_BLOBKEY); - } - if( rc==SQLITE_OK ){ - assert( iRoot==2 ); - rc = sqlite3BtreeCursor(pCsr->pBt, iRoot, 1, pCsr->pKeyInfo, p); - } - } - - pSorter->nBtree += sqlite3VarintLen(nKey) + nKey; + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->pVal = (void *)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; } + + /* See if the contents of the sorter should now be written out. They + ** are written out when either of the following are true: + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * cache-size), or + ** + ** * The total memory allocated for the in-memory list is greater + ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. + */ + if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( + (pSorter->nInMemory>pSorter->mxPmaSize) + || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + )){ + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + } + return rc; } @@ -543,12 +686,12 @@ static int vdbeSorterInitMerge( i64 nByte = 0; /* Total bytes in all opened PMAs */ /* Initialize the iterators. */ - for(i=0; rc==SQLITE_OK && iaIter[i]; rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); pSorter->iReadOff = pIter->iEof; - assert( pSorter->iReadOff<=pSorter->iWriteOff || rc!=SQLITE_OK ); - if( pSorter->iReadOff>=pSorter->iWriteOff ) break; + assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); + if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; } /* Initialize the aTree[] array. */ @@ -575,15 +718,19 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ assert( pSorter ); - /* Write the current b-tree to a PMA. Close the b-tree cursor. */ - rc = vdbeSorterBtreeToPMA(db, pCsr); - sqlite3BtreeCloseCursor(pCsr->pCursor); - if( rc!=SQLITE_OK ) return rc; + /* If no data has been written to disk, then do not do so now. Instead, + ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly + ** from the in-memory list. */ if( pSorter->nPMA==0 ){ - *pbEof = 1; - return SQLITE_OK; + *pbEof = !pSorter->pRecord; + assert( pSorter->aTree==0 ); + return vdbeSorterSort(db, pCsr); } + /* Write the current b-tree to a PMA. Close the b-tree cursor. */ + rc = vdbeSorterListToPMA(db, pCsr); + if( rc!=SQLITE_OK ) return rc; + /* Allocate space for aIter[] and aTree[]. */ nIter = pSorter->nPMA; if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; @@ -670,42 +817,91 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ */ int sqlite3VdbeSorterNext(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; - int iPrev = pSorter->aTree[1]; /* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ int rc; /* Return code */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( pSorter->aTree ){ + int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); + for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pCsr, i); + } + + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + }else{ + SorterRecord *pFree = pSorter->pRecord; + pSorter->pRecord = pFree->pNext; + pFree->pNext = 0; + vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->pRecord; + rc = SQLITE_OK; + } return rc; } +/* +** Return a pointer to a buffer owned by the sorter that contains the +** current key. +*/ +static void *vdbeSorterRowkey( + VdbeSorter *pSorter, /* Sorter object */ + int *pnKey /* OUT: Size of current key in bytes */ +){ + void *pKey; + if( pSorter->aTree ){ + VdbeSorterIter *pIter; + pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + *pnKey = pIter->nKey; + pKey = pIter->aKey; + }else{ + *pnKey = pSorter->pRecord->nVal; + pKey = pSorter->pRecord->pVal; + } + return pKey; +} + /* ** Copy the current sorter key into the memory cell pOut. */ int sqlite3VdbeSorterRowkey(VdbeCursor *pCsr, Mem *pOut){ VdbeSorter *pSorter = pCsr->pSorter; - VdbeSorterIter *pIter; + void *pKey; int nKey; /* Sorter key to copy into pOut */ - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - - /* Coverage testing note: As things are currently, this call will always - ** succeed. This is because the memory cell passed by the VDBE layer - ** happens to be the same one as was used to assemble the keys before they - ** were passed to the sorter - meaning it is always large enough for the - ** largest key. But this could change very easily, so we leave the call - ** to sqlite3VdbeMemGrow() in. */ - if( NEVER(sqlite3VdbeMemGrow(pOut, pIter->nKey, 0)) ){ + pKey = vdbeSorterRowkey(pSorter, &nKey); + if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ return SQLITE_NOMEM; } - pOut->n = pIter->nKey; + pOut->n = nKey; MemSetTypeFlag(pOut, MEM_Blob); - memcpy(pOut->z, pIter->aKey, pIter->nKey); + memcpy(pOut->z, pKey, nKey); return SQLITE_OK; } +/* +** Compare the key in memory cell pVal with the key that the sorter cursor +** passed as the first argument currently points to. For the purposes of +** the comparison, ignore the rowid field at the end of each record. +** +** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). +** Otherwise, set *pRes to a negative, zero or positive value if the +** key in pVal is smaller than, equal to or larger than the current sorter +** key. +*/ +int sqlite3VdbeSorterCompare( + VdbeCursor *pCsr, /* Sorter cursor */ + Mem *pVal, /* Value to compare to current sorter key */ + int *pRes /* OUT: Result of comparison */ +){ + int rc; + VdbeSorter *pSorter = pCsr->pSorter; + void *pKey; int nKey; /* Sorter key to compare pVal with */ + + pKey = vdbeSorterRowkey(pSorter, &nKey); + rc = vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); + assert( rc!=SQLITE_OK || pVal->db->mallocFailed || (*pRes)<=0 ); + return rc; +} + #endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ diff --git a/test/distinct.test b/test/distinct.test index 87456de876..e0a913604d 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -45,7 +45,7 @@ proc do_temptables_test {tn sql temptables} { uplevel [list do_test $tn [subst -novar { set ret "" db eval "EXPLAIN [set sql]" { - if {$opcode == "OpenEphemeral"} { + if {$opcode == "OpenEphemeral" || $opcode == "SorterOpen"} { if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" } if {$p5 == "10"} { lappend ret hash diff --git a/test/index4.test b/test/index4.test index 6400e34e4e..018ed744a1 100644 --- a/test/index4.test +++ b/test/index4.test @@ -108,5 +108,19 @@ do_execsql_test 1.8 { PRAGMA integrity_check } {ok} +do_execsql_test 2.1 { + BEGIN; + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(14); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(15); + INSERT INTO t2 VALUES(35); + INSERT INTO t2 VALUES(16); + COMMIT; +} +do_catchsql_test 2.2 { + CREATE UNIQUE INDEX i3 ON t2(x); +} {1 {indexed columns are not unique}} + finish_test diff --git a/test/misc3.test b/test/misc3.test index 94a43c437e..2aec49af98 100644 --- a/test/misc3.test +++ b/test/misc3.test @@ -270,7 +270,7 @@ ifcapable {explain} { CREATE UNIQUE INDEX ex1i1 ON ex1(a); EXPLAIN REINDEX; }] - regexp { IsUnique \d+ \d+ \d+ \d+ } $x + regexp { SorterCompare \d+ \d+ \d+ } $x } {1} if {[regexp {16} [db one {PRAGMA encoding}]]} { do_test misc3-6.11-utf16 { diff --git a/test/pager1.test b/test/pager1.test index 33e84d4b35..b9ba775add 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2437,6 +2437,7 @@ do_test pager1-32.1 { BEGIN; INSERT INTO t1 VALUES(1, randomblob(10000)); } + file_control_chunksize_test db main 1024 file_control_sizehint_test db main 20971520; # 20MB execsql { PRAGMA cache_size = 10; diff --git a/test/server1.test b/test/server1.test index 134a9f5773..90673ef000 100644 --- a/test/server1.test +++ b/test/server1.test @@ -29,7 +29,7 @@ if {[llength [info command client_step]]==0 || [sqlite3 -has-codec]} { # The sample server implementation does not work right when memory # management is enabled. # -ifcapable memorymanage { +ifcapable (memorymanage||mutex_noop) { finish_test return } diff --git a/test/thread001.test b/test/thread001.test index 9b788aafaf..7e0893f29d 100644 --- a/test/thread001.test +++ b/test/thread001.test @@ -42,7 +42,7 @@ foreach {tn same_db shared_cache} [list \ sqlite3_enable_shared_cache $shared_cache sqlite3_enable_shared_cache $shared_cache } $shared_cache - sqlite3 db test.db -fullmutex 1 + sqlite3 db test.db -fullmutex 1 -key xyzzy set dbconfig "" if {$same_db} { @@ -77,7 +77,7 @@ foreach {tn same_db shared_cache} [list \ #sqlthread parent {puts STARTING..} set needToClose 0 if {![info exists ::DB]} { - set ::DB [sqlthread open test.db] + set ::DB [sqlthread open test.db xyzzy] #sqlthread parent "puts \"OPEN $::DB\"" set needToClose 1 } diff --git a/test/thread002.test b/test/thread002.test index 60071d9243..b39c9ae7cd 100644 --- a/test/thread002.test +++ b/test/thread002.test @@ -16,9 +16,11 @@ set testdir [file dirname $argv0] +set do_not_use_codec 1 source $testdir/tester.tcl if {[run_thread_tests]==0} { finish_test ; return } + db close set ::enable_shared_cache [sqlite3_enable_shared_cache 1] diff --git a/test/thread003.test b/test/thread003.test index 22cd534c88..8c7440c82e 100644 --- a/test/thread003.test +++ b/test/thread003.test @@ -80,7 +80,7 @@ do_test thread003.2 { foreach zFile {test.db test2.db} { set SCRIPT [format { set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open %s] + set ::DB [sqlthread open %s xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } @@ -117,7 +117,7 @@ do_test thread003.3 { set SCRIPT [format { set iStart [clock_seconds] set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open %s] + set ::DB [sqlthread open %s xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } @@ -156,7 +156,7 @@ unset -nocomplain finished(2) do_test thread003.4 { thread_spawn finished(1) $thread_procs [format { set iEnd [expr {[clock_seconds] + %d}] - set ::DB [sqlthread open test.db] + set ::DB [sqlthread open test.db xyzzy] # Set the cache size to 15 pages per cache. 30 available globally. execsql { PRAGMA cache_size = 15 } diff --git a/test/wal5.test b/test/wal5.test index af4056ccd2..ad6bcfc7d8 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,7 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 7 5} - do_test 2.3.$tn.8 { file_page_counts } {2 7 2 7} + do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7} } # Check that checkpoints block on the correct locks. And respond correctly @@ -343,3 +343,4 @@ foreach {testprefix do_wal_checkpoint} { finish_test + diff --git a/tool/lemon.c b/tool/lemon.c index bd2938be25..1fb0308bec 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -2522,6 +2522,7 @@ void Parse(struct lemon *gp) ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.", filesize+1); gp->errorcnt++; + fclose(fp); return; } if( fread(filebuf,1,filesize,fp)!=filesize ){ @@ -2529,6 +2530,7 @@ void Parse(struct lemon *gp) filesize); free(filebuf); gp->errorcnt++; + fclose(fp); return; } fclose(fp); diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index bf6244e8c9..13ab1af5f7 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -41,8 +41,7 @@ set pageSize [db one {PRAGMA page_size}] # queries the in-memory db to produce the space-analysis report. # sqlite3 mem :memory: -set tabledef\ -{CREATE TABLE space_used( +set tabledef {CREATE TABLE space_used( name clob, -- Name of a table or index in the database file tblname clob, -- Name of associated table is_index boolean, -- TRUE if it is an index, false for a table @@ -293,16 +292,16 @@ proc subreport {title where} { statline {Overflow pages used} $ovfl_pages statline {Total pages used} $total_pages if {$int_unused>0} { - set int_unused_percent \ - [percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] + set int_unused_percent [ + percent $int_unused [expr {$int_pages*$pageSize}] {of index space}] statline "Unused bytes on index pages" $int_unused $int_unused_percent } - statline "Unused bytes on primary pages" $leaf_unused \ - [percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] - statline "Unused bytes on overflow pages" $ovfl_unused \ - [percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] - statline "Unused bytes on all pages" $total_unused \ - [percent $total_unused $storage {of all space}] + statline "Unused bytes on primary pages" $leaf_unused [ + percent $leaf_unused [expr {$leaf_pages*$pageSize}] {of primary space}] + statline "Unused bytes on overflow pages" $ovfl_unused [ + percent $ovfl_unused [expr {$ovfl_pages*$pageSize}] {of overflow space}] + statline "Unused bytes on all pages" $total_unused [ + percent $total_unused $storage {of all space}] return 1 } @@ -452,11 +451,9 @@ Page size in bytes Number of pages in the whole file } -puts \ -" The number of $pageSize-byte pages that go into forming the complete +puts " The number of $pageSize-byte pages that go into forming the complete database" -puts \ -{ +puts { Pages that store data The number of pages that store data, either as primary B*Tree pages or diff --git a/tool/tostr.awk b/tool/tostr.awk index 83c6cc1a50..b4f48d3db5 100644 --- a/tool/tostr.awk +++ b/tool/tostr.awk @@ -3,7 +3,6 @@ # Convert input text into a C string # { - gsub(/\\/,"\\\\"); gsub(/\"/,"\\\""); print "\"" $0 "\\n\""; }