From 7a2280fe6579c69235a78c5df83f34a555558049 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 15:01:53 +0000 Subject: [PATCH 1/6] Multiple optimizations that try to preserve or infer the zero-terminated property of TEXT values. Avoid unnecessary copying of text values destined to become function parameters. All changes help improve performance of doing UPDATEs on large JSON values that are indexed multiple ways. FossilOrigin-Name: d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 3 +-- src/json.c | 3 ++- src/vdbe.c | 6 ++++++ src/vdbeapi.c | 9 +++++++++ 6 files changed, 28 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 1eaf96f72f..2bf0e377ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sto\sthe\sJSON\sparser\sand\scode\sgenerator. -D 2023-07-19T23:02:45.962 +C Multiple\soptimizations\sthat\stry\sto\spreserve\sor\sinfer\sthe\szero-terminated\nproperty\sof\sTEXT\svalues.\s\sAvoid\sunnecessary\scopying\sof\stext\svalues\sdestined\nto\sbecome\sfunction\sparameters.\s\sAll\schanges\shelp\simprove\sperformance\sof\ndoing\sUPDATEs\son\slarge\sJSON\svalues\sthat\sare\sindexed\smultiple\sways. +D 2023-07-21T15:01:53.046 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -587,7 +587,7 @@ F src/date.c f73f203b3877cef866c60ab402aec2bf89597219b60635cf50cbe3c5e4533e94 F src/dbpage.c f3eea5f7ec47e09ee7da40f42b25092ecbe961fc59566b8e5f705f34335b2387 F src/dbstat.c ec92074baa61d883de58c945162d9e666c13cd7cf3a23bc38b4d1c4d0b2c2bef F src/delete.c cd5f5cd06ed0b6a882ec1a8c2a0d73b3cecb28479ad19e9931c4706c5e2182be -F src/expr.c 8d1656b65e26af3e34f78e947ac423f0d20c214ed25a67486e433bf16ca6b543 +F src/expr.c ef4a81822da6f767696bd7f4b9983328af061158958138540142285a5b1181b7 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c a7fcbf7e66d14dbb73cf49f31489ebf66d0e6006c62b95246924a3bae9f37b36 F src/func.c 25f2e0204c011be56fc3c9a180534b68ca4866c61ec19806880136450434112d @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e48136fce64e5004b1b8be76624cc9a4ac9ff6ae630a97df187c0ea4b9692d1b +F src/json.c 2bf3223892b4b9be39d701a5d349efcce5f5eebf71baaf91ba5866e10127e839 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -707,10 +707,10 @@ F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 -F src/vdbe.c 74282a947234513872a83b0bab1b8c644ece64b3e27b053ef17677c8ff9c81e0 +F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c de9703f8705afc393cc2864669ce28cf9516983c8331d59aa2b978de01634365 +F src/vdbeapi.c 1b17d1a53c3b88eb8bf2534f4f3557957f59f9431e5283734452b191cd45898d F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee9e403ae47103938aabb9a7a7e120dfa741b464875965e58788a51fa56a8fe 144c8ccf6e5bb2527dd98742f0d67e0a16c627e7c67f754ce8ed4c4fb5b8d8b6 -R af54725f832c177294e9e511cdacd53d +P 0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e +R ad6898efbc6819851ab380be79271f0e U drh -Z 1c679f829efc53f4f55993e30939aa14 +Z ecdcb9c56dec3594bcd0055088d236ad # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 10f9cdd123..000a57e974 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e \ No newline at end of file +d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a8e552f2c7..0c41f6684a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -4690,8 +4690,7 @@ expr_code_doover: } } - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } diff --git a/src/json.c b/src/json.c index 176dcbfdb9..a9425aecda 100644 --- a/src/json.c +++ b/src/json.c @@ -501,7 +501,8 @@ static void jsonAppendValue( */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + if( p->nAlloc>=p->nUsed+1 ) p->zBuf[p->nUsed] = 0; + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); jsonZero(p); diff --git a/src/vdbe.c b/src/vdbe.c index b248a664d3..075a632117 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -556,6 +556,9 @@ void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); } sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + if( f & MEM_Term ){ + sqlite3_str_appendf(pStr, "(0-term)"); + } } } #endif @@ -3085,6 +3088,9 @@ op_column_restart: rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, aOffset[p2], len, pDest); if( rc!=SQLITE_OK ) goto abort_due_to_error; sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->flags |= MEM_Term; + } pDest->flags &= ~MEM_Ephem; } } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 920780a896..7f44d22bc7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -514,6 +514,15 @@ void sqlite3_result_text64( (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); + if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ + Mem *pOut = pCtx->pOut; + if( pOut->z==z + && sqlite3_msize(pOut->z)>=pOut->n+1 + && pOut->z[n]==0 + ){ + pOut->flags |= MEM_Term; + } + } } } #ifndef SQLITE_OMIT_UTF16 From eee8687a9ff50465a720298bd8d50cbd85bb0fe7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 15:07:56 +0000 Subject: [PATCH 2/6] Fix harmless compiler warning about signed/unsigned comparison. FossilOrigin-Name: 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2bf0e377ab..b4f361aba9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Multiple\soptimizations\sthat\stry\sto\spreserve\sor\sinfer\sthe\szero-terminated\nproperty\sof\sTEXT\svalues.\s\sAvoid\sunnecessary\scopying\sof\stext\svalues\sdestined\nto\sbecome\sfunction\sparameters.\s\sAll\schanges\shelp\simprove\sperformance\sof\ndoing\sUPDATEs\son\slarge\sJSON\svalues\sthat\sare\sindexed\smultiple\sways. -D 2023-07-21T15:01:53.046 +C Fix\sharmless\scompiler\swarning\sabout\ssigned/unsigned\scomparison. +D 2023-07-21T15:07:56.630 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -710,7 +710,7 @@ F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c 1b17d1a53c3b88eb8bf2534f4f3557957f59f9431e5283734452b191cd45898d +F src/vdbeapi.c b8866af13401c9779759640457fa37dafe7bc2466c8428ade10ef625d012e52b F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0a745897d66e9f36ea6e787fad54f0c413d52c226dc96fad36ccd33a0aa1930e -R ad6898efbc6819851ab380be79271f0e +P d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e +R e667842b6755e6f486934eef004d9f18 U drh -Z ecdcb9c56dec3594bcd0055088d236ad +Z 993e102a87ed849f759c01f42525f684 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 000a57e974..247f7c86b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e \ No newline at end of file +75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 7f44d22bc7..079fb3976a 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -517,7 +517,7 @@ void sqlite3_result_text64( if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ Mem *pOut = pCtx->pOut; if( pOut->z==z - && sqlite3_msize(pOut->z)>=pOut->n+1 + && sqlite3_msize(pOut->z) >= (sqlite3_uint64)(pOut->n+1) && pOut->z[n]==0 ){ pOut->flags |= MEM_Term; From 569700a72e58a1cf9efc7fb743768a64cfd77d3c Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 18:09:07 +0000 Subject: [PATCH 3/6] Further improvements to large string handling in relation to JSON. FossilOrigin-Name: 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/json.c | 4 ++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 10 +--------- src/vdbemem.c | 31 ++++++++++++++++++++++++++++++- 6 files changed, 44 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index b4f361aba9..0ff7e75f17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sabout\ssigned/unsigned\scomparison. -D 2023-07-21T15:07:56.630 +C Further\simprovements\sto\slarge\sstring\shandling\sin\srelation\sto\sJSON. +D 2023-07-21T18:09:07.505 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -597,7 +597,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 2bf3223892b4b9be39d701a5d349efcce5f5eebf71baaf91ba5866e10127e839 +F src/json.c 46ea5566e1363f4f353db79b0378c2bf8ffdf9d4667daee3df67b14669767bed F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -709,11 +709,11 @@ F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 -F src/vdbeInt.h 7bd49eef8f89c1a271fbf12d80a206bf56c876814c5fc6bee340f4e1907095ae -F src/vdbeapi.c b8866af13401c9779759640457fa37dafe7bc2466c8428ade10ef625d012e52b +F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 +F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c cf4a1556dd5b18c071cf7c243373c29ce752eb516022e3ad49ba72f08b785033 +F src/vdbemem.c d6588e873aec78077c95c95df03920a1bcff1ee59fe58579261c957bf6118988 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d0278cdedfa04fb0b61838ab9622be8a2c462f58d5c3ebc4c5f802a727d0974e -R e667842b6755e6f486934eef004d9f18 +P 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 +R b0bbcf196bde05ae8f599a4f16022c78 U drh -Z 993e102a87ed849f759c01f42525f684 +Z 8dfc9dc2f2e2752cbadaab8d45cf08cb # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 247f7c86b6..9ac6a758a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 \ No newline at end of file +1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c \ No newline at end of file diff --git a/src/json.c b/src/json.c index a9425aecda..05046b5b54 100644 --- a/src/json.c +++ b/src/json.c @@ -501,8 +501,8 @@ static void jsonAppendValue( */ static void jsonResult(JsonString *p){ if( p->bErr==0 ){ - if( p->nAlloc>=p->nUsed+1 ) p->zBuf[p->nUsed] = 0; - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + jsonAppendChar(p, 0); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed-1, p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, SQLITE_UTF8); jsonZero(p); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 4c3394716b..3a5b961a25 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -611,6 +611,7 @@ int sqlite3VdbeMemSetZeroBlob(Mem*,int); int sqlite3VdbeMemIsRowSet(const Mem*); #endif int sqlite3VdbeMemSetRowSet(Mem*); +void sqlite3VdbeMemZeroTerminateIfAble(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); int sqlite3VdbeMemStringify(Mem*, u8, u8); int sqlite3IntFloatCompare(i64,double); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 079fb3976a..92bf38160f 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -514,15 +514,7 @@ void sqlite3_result_text64( (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); - if( xDel==sqlite3_free && enc==SQLITE_UTF8 ){ - Mem *pOut = pCtx->pOut; - if( pOut->z==z - && sqlite3_msize(pOut->z) >= (sqlite3_uint64)(pOut->n+1) - && pOut->z[n]==0 - ){ - pOut->flags |= MEM_Term; - } - } + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); } } #ifndef SQLITE_OMIT_UTF16 diff --git a/src/vdbemem.c b/src/vdbemem.c index 1250eea77d..2baa35e24f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -314,6 +314,32 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ return SQLITE_OK; } +/* +** If pMem is already a string, detect if it is a zero-terminated +** string, or make it into one if possible, and mark it as such. +** +** This is an optimization. Correct operation continues even if +** this routine is a no-op. +*/ +void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ + if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return; + if( pMem->enc!=SQLITE_UTF8 ) return; + if( pMem->z==0 ) return; + if( pMem->flags & MEM_Dyn ){ + if( pMem->xDel==sqlite3_free + && sqlite3_msize(pMem->z) >= pMem->n+1 + && pMem->z[pMem->n]==0 + ){ + pMem->flags |= MEM_Term; + return; + } + }else if( pMem->szMalloc>0 && pMem->szMalloc >= pMem->n+1 ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } +} + /* ** It is already known that pMem contains an unterminated string. ** Add the zero terminator. @@ -803,6 +829,7 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ break; } default: { + int rc; assert( aff==SQLITE_AFF_TEXT ); assert( MEM_Str==(MEM_Blob>>3) ); pMem->flags |= (pMem->flags&MEM_Blob)>>3; @@ -810,7 +837,9 @@ int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; - return sqlite3VdbeChangeEncoding(pMem, encoding); + rc = sqlite3VdbeChangeEncoding(pMem, encoding); + if( rc ) return rc; + sqlite3VdbeMemZeroTerminateIfAble(pMem); } } return SQLITE_OK; From 028acd974f3c03c489dfaa02da618af99613cde7 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 18:38:59 +0000 Subject: [PATCH 4/6] Avoid a potentially large strlen() in sqlite3DecOrHexToI64(). FossilOrigin-Name: 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0ff7e75f17..1fd56ed788 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\simprovements\sto\slarge\sstring\shandling\sin\srelation\sto\sJSON. -D 2023-07-21T18:09:07.505 +C Avoid\sa\spotentially\slarge\sstrlen()\sin\ssqlite3DecOrHexToI64(). +D 2023-07-21T18:38:59.114 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -705,7 +705,7 @@ F src/trigger.c ad6ab9452715fa9a8075442e15196022275b414b9141b566af8cdb7a1605f2b0 F src/update.c 0aa36561167a7c40d01163238c297297962f31a15a8d742216b3c37cdf25f731 F src/upsert.c 5303dc6c518fa7d4b280ec65170f465c7a70b7ac2b22491598f6d0b4875b3145 F src/utf.c ee39565f0843775cc2c81135751ddd93eceb91a673ea2c57f61c76f288b041a0 -F src/util.c fea6fffdee3cdae917a66b70deec59d4f238057cfd6de623d15cf990c196940d +F src/util.c c2aa170f2eb429235b1dddce8952770787ffa5124dc89d405bfbe8ebad8e7ebd F src/vacuum.c 604fcdaebe76f3497c855afcbf91b8fa5046b32de3045bab89cc008d68e40104 F src/vdbe.c 4cda877d413a18fa07346b08d6959b3d18ce982357921e7acb9649fca2534a12 F src/vdbe.h 41485521f68e9437fdb7ec4a90f9d86ab294e9bb8281e33b235915e29122cfc0 @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 75cc3c89ee2dcfefa9421ce60bee77e85d2895918e8c5cfd05c434f8932a99b5 -R b0bbcf196bde05ae8f599a4f16022c78 +P 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c +R 985a846b4f2b586d366fd130e337d17a U drh -Z 8dfc9dc2f2e2752cbadaab8d45cf08cb +Z f082f5b7a15300eec42c5fbe873f6275 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9ac6a758a7..ff10b67548 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c \ No newline at end of file +5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa \ No newline at end of file diff --git a/src/util.c b/src/util.c index 409482d395..1200aef0e3 100644 --- a/src/util.c +++ b/src/util.c @@ -846,7 +846,9 @@ int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); + if( z[n] ) n++; + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); } } From 89f07b12769548185c122bb3cb071cb4748c4f04 Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 19:17:26 +0000 Subject: [PATCH 5/6] Fix harmless compiler warning introduced by the previous check-in. FossilOrigin-Name: 06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1fd56ed788..083967c4a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\spotentially\slarge\sstrlen()\sin\ssqlite3DecOrHexToI64(). -D 2023-07-21T18:38:59.114 +C Fix\sharmless\scompiler\swarning\sintroduced\sby\sthe\sprevious\scheck-in. +D 2023-07-21T19:17:26.591 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -713,7 +713,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c d6588e873aec78077c95c95df03920a1bcff1ee59fe58579261c957bf6118988 +F src/vdbemem.c 490342e7ab9ac561e8e210e1efe107bc290a97bc39046637fffceb84c02cc8e4 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1e5df0aa3dae5cadbf1d07c718ae2a5212543300b68e49d35e8c96855a7f619c -R 985a846b4f2b586d366fd130e337d17a +P 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa +R 99401c28791119c3b9a7ff587d00483e U drh -Z f082f5b7a15300eec42c5fbe873f6275 +Z 82c739c7a09ae5acca56a5e43e87910d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ff10b67548..c901fe6612 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa \ No newline at end of file +06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 2baa35e24f..b007a89cda 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -327,7 +327,7 @@ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( pMem->z==0 ) return; if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free - && sqlite3_msize(pMem->z) >= pMem->n+1 + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) && pMem->z[pMem->n]==0 ){ pMem->flags |= MEM_Term; From 21f22f9088f5e9ce2ae4c1e666ab73cc96bfa99e Mon Sep 17 00:00:00 2001 From: drh <> Date: Fri, 21 Jul 2023 22:22:13 +0000 Subject: [PATCH 6/6] Mark an unreachable branch as NEVER(). FossilOrigin-Name: 414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 083967c4a7..6095497a50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sintroduced\sby\sthe\sprevious\scheck-in. -D 2023-07-21T19:17:26.591 +C Mark\san\sunreachable\sbranch\sas\sNEVER(). +D 2023-07-21T22:22:13.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -713,7 +713,7 @@ F src/vdbeInt.h 401813862f9d75af01bdb2ab99253ad019e9d6ddcc8058e4fa61a43e9a60d1f7 F src/vdbeapi.c dde6c4d0f87486f056b9db4d1ea185bb1d84a6839102b86e76316ba590d07cc7 F src/vdbeaux.c b5e3f7e158518b4eca6f166ac43900640a3fe9735c710e12bfa119af21059339 F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2ce -F src/vdbemem.c 490342e7ab9ac561e8e210e1efe107bc290a97bc39046637fffceb84c02cc8e4 +F src/vdbemem.c b3013dd11696db423bb410a50126d46d700fe3133b936a32195700e9731d2960 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac @@ -2043,8 +2043,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5413b02bb629b9cbb76f7e688e94ebcf24276c01436d3feb73ff57c036e1d2aa -R 99401c28791119c3b9a7ff587d00483e +P 06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 +R f650c894c3b7e9fe01bf709dbdf65d35 U drh -Z 82c739c7a09ae5acca56a5e43e87910d +Z ff67cbf0340dd6d2b6c094e434a4b95a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c901fe6612..58efe19050 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06f829e9e57c5aa495f519ad7bb379be611e8e21ef60d36e63d055e45f7a6117 \ No newline at end of file +414a4d660c3b505640227066b468db1268fcb1bd3fad931644a769617c6c00cd \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index b007a89cda..3f845452c4 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -324,7 +324,7 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ if( (pMem->flags & (MEM_Str|MEM_Term))!=MEM_Str ) return; if( pMem->enc!=SQLITE_UTF8 ) return; - if( pMem->z==0 ) return; + if( NEVER(pMem->z==0) ) return; if( pMem->flags & MEM_Dyn ){ if( pMem->xDel==sqlite3_free && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1)