From 31999c5cac3b01dc8a17a9148f8129d33109e240 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Nov 2019 17:46:32 +0000 Subject: [PATCH] Fix a bug in the hard_heap_limit pragma so that it returns the new value of the hard_heap_limit, not the soft_heap_limit. Change SQLITE_MAX_MEMORY so that it works by setting the default hard_heap_limit value. FossilOrigin-Name: 33fd0c3abcad5555a150990a22d9c1bab99e79be01143fccb9fafc9b52cf92c8 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/malloc.c | 16 ++++++++-------- src/pragma.c | 2 +- test/fuzzcheck.c | 34 +++++++++++++++++----------------- test/ossfuzz.c | 3 +++ test/tester.tcl | 12 ++++++++++-- 7 files changed, 50 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 89df4bcd09..50042c0275 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\senhancements\sfrom\strunk. -D 2019-11-14T15:21:15.088 +C Fix\sa\sbug\sin\sthe\shard_heap_limit\spragma\sso\sthat\sit\sreturns\sthe\snew\svalue\sof\nthe\shard_heap_limit,\snot\sthe\ssoft_heap_limit.\s\sChange\sSQLITE_MAX_MEMORY\sso\nthat\sit\sworks\sby\ssetting\sthe\sdefault\shard_heap_limit\svalue. +D 2019-11-14T17:46:32.327 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -491,7 +491,7 @@ F src/insert.c 8701d80b3cecd47f0375483c1ef35dfcdc777f3c67e2d3581dd7841b980910a3 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c d74f5e7bd51f3c9d283442473eb65aef359664efd6513591c03f01881c4ae2da F src/main.c 1505735106a694a6a62f28d912de74c5fe3dfbacdd3aa78a63eb97d0754f8b49 -F src/malloc.c baf74e2798722e51a44685abd44067ec9ee3a05440ce5085565f2614d8d277a9 +F src/malloc.c 550021fcae36f0ffe9f8563d83e6385f9df307a854d55d7d0abb7241ee8dbcc6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c12a42539b1ba105e3707d0e628ad70e611040d8f5e38cf942cee30c867083de F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -519,7 +519,7 @@ F src/parse.y 17c1ae265e1b92cb1f3f1661b020e3eb31f2d8b9588322d2b6f2b22f25b674c9 F src/pcache.c 385ff064bca69789d199a98e2169445dc16e4291fa807babd61d4890c3b34177 F src/pcache.h 4f87acd914cef5016fae3030343540d75f5b85a1877eed1a2a19b9f284248586 F src/pcache1.c 62714cbd1b7299a6e6a27a587b66b4fd3a836a84e1181e7f96f5c34a50917848 -F src/pragma.c 115fd36362c336449be3c5f9dd944f2b1992ab1a320501cec5aae712dce3e794 +F src/pragma.c 36c367a62a9d3415fd3d15523e794cccc2cb74e6d806662b0185b8c686f06acb F src/pragma.h ec3b31eac9b1df040f1cc8cb3d89bc06605c3b4cb3d76f833de8d6d6c3f77f04 F src/prepare.c 6049beb71385f017af6fc320d2c75a4e50b75e280c54232442b785fbb83df057 F src/printf.c 9be6945837c839ba57837b4bc3af349eba630920fa5532aa518816defe42a7d4 @@ -1008,7 +1008,7 @@ F test/fuzz3.test 9c813e6613b837cb7a277b0383cd66bfa07042b4cf0317157c35852f30043c F test/fuzz4.test c229bcdb45518a89e1d208a21343e061503460ac69fae1539320a89f572eb634 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test f348276e732e814802e39f042b1f6da6362a610af73a528d8f76898fde6b22f2 -F test/fuzzcheck.c 3ad76298a80cda31d270dc5e4f31194fa38d507d3e9b3f355cf1c283895cd5a5 +F test/fuzzcheck.c e8cf694f71a1ee39a59f7c2a38c0f8660db0656ce47c8a334b6e9a11f1e66c6d F test/fuzzdata1.db 7ee3227bad0e7ccdeb08a9e6822916777073c664 F test/fuzzdata2.db 128b3feeb78918d075c9b14b48610145a0dd4c8d6f1ca7c2870c7e425f5bf31f F test/fuzzdata3.db c6586d3e3cef0fbc18108f9bb649aa77bfc38aba @@ -1207,7 +1207,7 @@ F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 F test/orderbyA.test df608e59efc2ef50c1eddf1a773b272de3252e9401bfec86d04b52fd973866d5 F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 -F test/ossfuzz.c 18af635fa73d12a109b305faca727a734c1fa28a421b161d9d15c5a84a4998a2 +F test/ossfuzz.c 9636dad2092a05a32110df0ca06713038dd0c43dd89a77dabe4b8b0d71096715 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f F test/pager1.test 1e9ee778bdeaf4f7f09997d029cdaca6a42dfc2092edafe4f5e590acbf1eab13 @@ -1405,7 +1405,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f1a5ce6a F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl 64ac253a411db1af7649438f674213a45876ca70609570310a8652edf23e5d77 +F test/tester.tcl abba168acd7f01dbfa3ffdbf402d151eb97e8a824d9208e845ab34c194441483 F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1849,7 +1849,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6153f3aada0cc9c5e773753797892ee78b518b0d76568382f5fbc9ee960f814c 5baffcda7d5a42a99fa905faa834f14b94b1e2a26b5221f15d8ae8f1f5e3236a -R 467f266e2e2f94b4b973342ee6d27bfc +P b8a631fd30d0732505679230684b3362c965438197a2b11518f01a77599c9202 +R 4e80a787b37e9567df232e950e42c798 U drh -Z 732e505323a67cefb8bb3679c634b931 +Z aa6d0feee69df1227a085bd7741cd7c1 diff --git a/manifest.uuid b/manifest.uuid index aa50d118ca..f3cbd8d9ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8a631fd30d0732505679230684b3362c965438197a2b11518f01a77599c9202 \ No newline at end of file +33fd0c3abcad5555a150990a22d9c1bab99e79be01143fccb9fafc9b52cf92c8 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 4dbac95691..131ed6039a 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -32,6 +32,13 @@ int sqlite3_release_memory(int n){ #endif } +/* +** Default value of the hard heap limit. 0 means "no limit". +*/ +#ifndef SQLITE_MAX_MEMORY +# define SQLITE_MAX_MEMORY 0 +#endif + /* ** State information local to the memory allocation subsystem. */ @@ -45,7 +52,7 @@ static SQLITE_WSD struct Mem0Global { ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0 }; +} mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -232,13 +239,6 @@ static void mallocWithAlarm(int n, void **pp){ ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); -#ifdef SQLITE_MAX_MEMORY - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)+nFull>SQLITE_MAX_MEMORY ){ - *pp = 0; - return; - } -#endif - sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); diff --git a/src/pragma.c b/src/pragma.c index eaa5945ad4..8878e7353e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2106,7 +2106,7 @@ void sqlite3Pragma( sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); } - returnSingleInt(v, sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); break; } diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c index 4d52aedea9..f1d2415de2 100644 --- a/test/fuzzcheck.c +++ b/test/fuzzcheck.c @@ -453,6 +453,9 @@ static unsigned int mxProgressCb = 2000; /* Maximum string length in SQLite */ static int lengthLimit = 1000000; +/* Limit on the amount of heap memory that can be used */ +static sqlite3_int64 heapLimit = 1000000000; + /* Maximum byte-code program length in SQLite */ static int vdbeOpLimit = 25000; @@ -777,6 +780,7 @@ int runCombinedDbSqlInput(const uint8_t *aData, size_t nByte){ if( lengthLimit>0 ){ sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, lengthLimit); } + sqlite3_hard_heap_limit64(heapLimit); if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ aDb[18] = aDb[19] = 1; @@ -1341,7 +1345,7 @@ int main(int argc, char **argv){ int cellSzCkFlag = 0; /* --cell-size-check */ int sqlFuzz = 0; /* True for SQL fuzz. False for DB fuzz */ int iTimeout = 120; /* Default 120-second timeout */ - int nMem = 0; /* Memory limit */ + int nMem = 0; /* Memory limit override */ int nMemThisDb = 0; /* Memory limit set by the CONFIG table */ char *zExpDb = 0; /* Write Databases to files in this directory */ char *zExpSql = 0; /* Write SQL to files in this directory */ @@ -1391,13 +1395,8 @@ int main(int argc, char **argv){ infoFlag = 1; }else if( strcmp(z,"limit-mem")==0 ){ -#if !defined(SQLITE_ENABLE_MEMSYS3) && !defined(SQLITE_ENABLE_MEMSYS5) - fatalError("the %s option requires -DSQLITE_ENABLE_MEMSYS5 or _MEMSYS3", - argv[i]); -#else if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); nMem = integerValue(argv[++i]); -#endif }else if( strcmp(z,"limit-vdbe")==0 ){ vdbeLimitFlag = 1; @@ -1586,14 +1585,9 @@ int main(int argc, char **argv){ ossFuzzThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); } - if( strcmp(zName, "limit-mem")==0 && !nativeMalloc ){ -#if !defined(SQLITE_ENABLE_MEMSYS3) && !defined(SQLITE_ENABLE_MEMSYS5) - fatalError("the limit-mem option requires -DSQLITE_ENABLE_MEMSYS5" - " or _MEMSYS3"); -#else + if( strcmp(zName, "limit-mem")==0 ){ nMemThisDb = sqlite3_column_int(pStmt,1); if( verboseFlag ) printf("Config: limit-mem=%d\n", nMemThisDb); -#endif } } sqlite3_finalize(pStmt); @@ -1720,12 +1714,18 @@ int main(int argc, char **argv){ /* Limit available memory, if requested */ sqlite3_shutdown(); - if( nMemThisDb>0 && !nativeMalloc ){ - pHeap = realloc(pHeap, nMemThisDb); - if( pHeap==0 ){ - fatalError("failed to allocate %d bytes of heap memory", nMem); + if( nMemThisDb>0 && nMem==0 ){ + if( !nativeMalloc ){ + pHeap = realloc(pHeap, nMemThisDb); + if( pHeap==0 ){ + fatalError("failed to allocate %d bytes of heap memory", nMem); + } + sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); + }else{ + sqlite3_hard_heap_limit64((sqlite3_int64)nMemThisDb); } - sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); + }else{ + sqlite3_hard_heap_limit64(0); } /* Disable lookaside with the --native-malloc option */ diff --git a/test/ossfuzz.c b/test/ossfuzz.c index 3b1017f726..b0156a640e 100644 --- a/test/ossfuzz.c +++ b/test/ossfuzz.c @@ -155,6 +155,9 @@ int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { /* Set a limit on the maximum size of a prepared statement */ sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, 25000); + /* Limit total memory available to SQLite to 20MB */ + sqlite3_hard_heap_limit64(20000000); + /* Set a limit on the maximum length of a string or BLOB. Without this ** limit, fuzzers will invoke randomblob(N) for a large N, and the process ** will timeout trying to generate the huge blob */ diff --git a/test/tester.tcl b/test/tester.tcl index 6efb59bd6d..05c52343f0 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -388,6 +388,7 @@ proc print_help_and_quit {} { puts {Options: --pause Wait for user input before continuing --soft-heap-limit=N Set the soft-heap-limit to N + --hard-heap-limit=N Set the hard-heap-limit to N --maxerror=N Quit after N errors --verbose=(0|1) Control the amount of output. Default '1' --output=FILE set --verbose=2 and output to FILE. Implies -q @@ -408,6 +409,7 @@ if {[info exists cmdlinearg]==0} { # # --pause # --soft-heap-limit=NN + # --hard-heap-limit=NN # --maxerror=NN # --malloctrace=N # --backtrace=N @@ -424,6 +426,7 @@ if {[info exists cmdlinearg]==0} { # --help # set cmdlinearg(soft-heap-limit) 0 + set cmdlinearg(hard-heap-limit) 0 set cmdlinearg(maxerror) 1000 set cmdlinearg(malloctrace) 0 set cmdlinearg(backtrace) 10 @@ -450,6 +453,9 @@ if {[info exists cmdlinearg]==0} { {^-+soft-heap-limit=.+$} { foreach {dummy cmdlinearg(soft-heap-limit)} [split $a =] break } + {^-+hard-heap-limit=.+$} { + foreach {dummy cmdlinearg(hard-heap-limit)} [split $a =] break + } {^-+maxerror=.+$} { foreach {dummy cmdlinearg(maxerror)} [split $a =] break } @@ -586,7 +592,8 @@ if {[info exists cmdlinearg]==0} { # way if an individual test file changes the soft-heap-limit, it # will be reset at the start of the next test file. # -sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit) +sqlite3_soft_heap_limit64 $cmdlinearg(soft-heap-limit) +sqlite3_hard_heap_limit64 $cmdlinearg(hard-heap-limit) # Create a test database # @@ -1207,7 +1214,8 @@ proc finalize_testing {} { db close sqlite3_reset_auto_extension - sqlite3_soft_heap_limit 0 + sqlite3_soft_heap_limit64 0 + sqlite3_hard_heap_limit64 0 set nTest [incr_ntest] set nErr [set_test_counter errors]