From e455e3acf0c6042885deba23e2d3fb1ff7610a83 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Jan 2020 20:16:32 +0000 Subject: [PATCH] In fts3, avoid making a very large memory allocation if the merge-hint record is corrupt. FossilOrigin-Name: 9add58fe9688d5c1f675736e7f1c6ff7831d2ff6fdeb884a2e62c1bf9536ee50 --- ext/fts3/fts3_write.c | 12 +++++++++--- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fts3corrupt4.test | 9 +++++++++ 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 203b48e2d1..197aebdd49 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -335,7 +335,7 @@ static int fts3SqlStmt( ** returns zero rows. */ /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " " GROUP BY level HAVING cnt>=?" - " ORDER BY (level %% 1024) ASC LIMIT 1", + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment ** created by merging the oldest :2 segments from absolute level :1. See @@ -4931,8 +4931,14 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); bUseHint = 1; bDirtyHint = 1; }else{ @@ -4945,7 +4951,7 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* If nSeg is less that zero, then there is no level with at least ** nMin segments and no hint in the %_stat table. No work to do. ** Exit early in this case. */ - if( nSeg<0 ) break; + if( nSeg<=0 ) break; /* Open a cursor to iterate through the contents of the oldest nSeg ** indexes of absolute level iAbsLevel. If this cursor is opened using diff --git a/manifest b/manifest index 5b6e4d66c3..1e35dea297 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sNEVER\smacros\sto\serror-detection\sbranches\sthat\swere\smade\sunreachable\n(as\sfar\sas\sI\scan\stell)\sby\scheck-in\s[ceacc28b03580334]. -D 2020-01-03T17:40:30.845 +C In\sfts3,\savoid\smaking\sa\svery\slarge\smemory\sallocation\sif\sthe\smerge-hint\srecord\sis\scorrupt. +D 2020-01-03T20:16:32.201 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -99,7 +99,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 4b9af6151c29b35ed09574937083cece7c31e911f69615e168a39677569b684d F ext/fts3/fts3_unicode2.c 416eb7e1e81142703520d284b768ca2751d40e31fa912cae24ba74860532bf0f -F ext/fts3/fts3_write.c 51e0a4e3782ee17b6dd5e89949b7095fc98e36e87725c53de631734535507498 +F ext/fts3/fts3_write.c 6f9dd5d774003ea81b8b32daa7d0819f9aaf01bf2b5f33498a69aab096094ed3 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87fea4ac01b73 @@ -940,7 +940,7 @@ F test/fts3conf.test c84bbaec81281c1788aa545ac6e78a6bd6cde2bdbbce2da261690e3659f F test/fts3corrupt.test ce7f7b5eaeee5f1804584d061b978d85e64abf2af9adaa7577589fac6f7eae01 F test/fts3corrupt2.test bf55c3fa0b0dc8ea1c0fe5543623bd27714585da6a129038fd6999fe3b0d25f3 F test/fts3corrupt3.test 0d5b69a0998b4adf868cc301fc78f3d0707745f1d984ce044c205cdb764b491f -F test/fts3corrupt4.test 50f8e119b688c8b8aa46c1543a022e0e1a931f5820c7da099b34e9acea7ece3b +F test/fts3corrupt4.test e407c7b4f4cd3335080833aff3a8855d520e531b79f84dcc77be4623af2342d4 F test/fts3corrupt5.test 0549f85ec4bd22e992f645f13c59b99d652f2f5e643dac75568bfd23a6db7ed5 F test/fts3cov.test 7eacdbefd756cfa4dc2241974e3db2834e9b372ca215880e00032222f32194cf F test/fts3d.test 2bd8c97bcb9975f2334147173b4872505b6a41359a4f9068960a36afe07a679f @@ -1853,7 +1853,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 523e09bd228750fb7be1d4e68d18f6c1bf9db19293d66fbf30b86409321e9814 -R 9db325677ae78118554163eac3ed2eae -U drh -Z e0ea28a1a8dedd28fad05208e3f0151a +P 7b62555e285f32d354df7746ef8dce16856c6d9d53a02d895b74ba02362d2968 +R 7f27ee6576dc13efabadbee4643d1518 +U dan +Z 162e3132aa65eb8605e2f1b28bf07203 diff --git a/manifest.uuid b/manifest.uuid index e4bdd27cd6..778ba3f4ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b62555e285f32d354df7746ef8dce16856c6d9d53a02d895b74ba02362d2968 \ No newline at end of file +9add58fe9688d5c1f675736e7f1c6ff7831d2ff6fdeb884a2e62c1bf9536ee50 \ No newline at end of file diff --git a/test/fts3corrupt4.test b/test/fts3corrupt4.test index c4cfd60d93..7404ddeb2c 100644 --- a/test/fts3corrupt4.test +++ b/test/fts3corrupt4.test @@ -5812,4 +5812,13 @@ do_catchsql_test 35.1 { INSERT INTO f(f) VALUES ('integrity-check'); } {1 {database disk image is malformed}} +reset_db +do_catchsql_test 36.0 { + CREATE VIRTUAL TABLE f USING fts3(a,tokenize=porter); + CREATE TABLE 'f_stat'(id INTEGER PRIMARY KEY, value BLOB); + INSERT INTO f VALUES (1); + INSERT INTO f_stat VALUES (1,x'00000000000101010119013d00ffff0400fa83717b71a69297979701f63d010101010101010101010101190000000000000000fa83717b71a601f63d01010101010101010101010119013d00ffffff0400fa83717b71a69297979701f63d010101010101010101010101190000000000000000fa83717b71a69201f63d010101f63d01010101010101010101010119013d00ffffff0400fa83717b71a6929797010101010101010101010119013d00ffff01f63d01010101010101010101010119013d00ffffff0400fa83717b71a69297979701f63d00fa03ffffffa69297979701f63d010101000000000101010101197e9797976567656565ffa63535354e'); + INSERT INTO f(f) VALUES ('merge=53,216'); +} {0 {}} + finish_test