From aec0aa83a9a4ad1c380707594fea1da191bdcb9b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jun 2023 18:59:10 +0000 Subject: [PATCH 01/14] Add the "nexec" and "ncycle" columns to the bytecode virtual table. For accessing counters collected when SQLITE_DBCONFIG_STMT_SCANSTATUS is enabled. FossilOrigin-Name: f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006 --- manifest | 16 +++++++++------- manifest.uuid | 2 +- src/vdbevtab.c | 29 +++++++++++++++++++++++------ 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 0b84625ebf..b4de521228 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\ssqlite3_stmt_scanstatus_v2()\sprofiling\sof\sGROUP\sBY\sclauses\sthat\suse\sa\stemp\sb-tree,\sand\sfor\ssub-queries\simplemented\sas\sco-routines. -D 2023-06-30T18:31:37.545 +C Add\sthe\s"nexec"\sand\s"ncycle"\scolumns\sto\sthe\sbytecode\svirtual\stable.\sFor\saccessing\scounters\scollected\swhen\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sis\senabled. +D 2023-06-30T18:59:10.790 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -716,7 +716,7 @@ F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2c F src/vdbemem.c aed58a560caab12540f7c14c43ee188636017814e21247a97902f78de2d43117 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 -F src/vdbevtab.c aae4bd769410eb7e1d02c42613eec961d514459b1c3c1c63cfc84e92a137daac +F src/vdbevtab.c c3085edc48b123ca1c30d478f2a7cb326b5e4c2960b65ca8f0f0d6ebf497140a F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c cbfeeb7415baa545efa244dd34bb5af4ae953a206fed720c6fa7f1ef763ec122 @@ -2041,9 +2041,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f936f101406069b29218c89a36581b4497226fb61906782ea368f12d943c901c 4e8718dc35dbbaf75f17265a88d14acd9750dc75efbadf41377f9c97e732009c -R 4da01a7efae23c104b98c2eba70ec2ef -T +closed 4e8718dc35dbbaf75f17265a88d14acd9750dc75efbadf41377f9c97e732009c +P 7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1 +R 4dba3e78e9c48b15e9dce867cfd3b67a +T *branch * scanstatus-exp +T *sym-scanstatus-exp * +T -sym-trunk * U dan -Z 8ca5997f8884b68f398b8d0f077d59ae +Z 262a3b7e30cd989cb3c51edc832f13b0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f482cd32b9..848125017e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1 \ No newline at end of file +f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006 \ No newline at end of file diff --git a/src/vdbevtab.c b/src/vdbevtab.c index 6557d8cb01..b95852fb5d 100644 --- a/src/vdbevtab.c +++ b/src/vdbevtab.c @@ -69,6 +69,8 @@ static int bytecodevtabConnect( "p5 INT," "comment TEXT," "subprog TEXT," + "nexec INT," + "ncycle INT," "stmt HIDDEN" ");", @@ -231,7 +233,7 @@ static int bytecodevtabColumn( } } } - i += 10; + i += 20; } } switch( i ){ @@ -281,16 +283,31 @@ static int bytecodevtabColumn( } break; } - case 10: /* tables_used.type */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + case 9: /* nexec */ + sqlite3_result_int(ctx, pCur->aOp[0].nExec); + break; + case 10: /* ncycle */ + sqlite3_result_int(ctx, pCur->aOp[0].nCycle); + break; +#else + case 9: /* nexec */ + case 10: /* ncycle */ + sqlite3_result_int(ctx, 0); + break; +#endif + + case 20: /* tables_used.type */ sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); break; - case 11: /* tables_used.schema */ + case 21: /* tables_used.schema */ sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); break; - case 12: /* tables_used.name */ + case 22: /* tables_used.name */ sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); break; - case 13: /* tables_used.wr */ + case 23: /* tables_used.wr */ sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); break; } @@ -364,7 +381,7 @@ static int bytecodevtabBestIndex( int rc = SQLITE_CONSTRAINT; struct sqlite3_index_constraint *p; bytecodevtab *pVTab = (bytecodevtab*)tab; - int iBaseCol = pVTab->bTablesUsed ? 4 : 8; + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; pIdxInfo->estimatedCost = (double)100; pIdxInfo->estimatedRows = 100; pIdxInfo->idxNum = 0; From e6d7ae24e3682a8f98d1a34bdfdab3caf0f85371 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jun 2023 19:13:27 +0000 Subject: [PATCH 02/14] Fix an error in the previous commit on this branch. FossilOrigin-Name: 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/vdbevtab.c | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index b4de521228..af6823742c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"nexec"\sand\s"ncycle"\scolumns\sto\sthe\sbytecode\svirtual\stable.\sFor\saccessing\scounters\scollected\swhen\sSQLITE_DBCONFIG_STMT_SCANSTATUS\sis\senabled. -D 2023-06-30T18:59:10.790 +C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. +D 2023-06-30T19:13:27.228 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -716,7 +716,7 @@ F src/vdbeblob.c 2516697b3ee8154eb8915f29466fb5d4f1ae39ee8b755ea909cefaf57ec5e2c F src/vdbemem.c aed58a560caab12540f7c14c43ee188636017814e21247a97902f78de2d43117 F src/vdbesort.c 0d40dca073c94e158ead752ef4225f4fee22dee84145e8c00ca2309afb489015 F src/vdbetrace.c fe0bc29ebd4e02c8bc5c1945f1d2e6be5927ec12c06d89b03ef2a4def34bf823 -F src/vdbevtab.c c3085edc48b123ca1c30d478f2a7cb326b5e4c2960b65ca8f0f0d6ebf497140a +F src/vdbevtab.c 57fa8f56478e5b5cb558cb425e7878515e0a105c54f96f1d1bbf4b9433529254 F src/vtab.c 1ecf8c3745d29275688d583e12822fa984d421e0286b5ef50c137bc3bf6d7a64 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c cbfeeb7415baa545efa244dd34bb5af4ae953a206fed720c6fa7f1ef763ec122 @@ -2041,11 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7afad1f759f7ceda873c6d869422fd56fe4399c2d24d47ad9bc3b84b06d830d1 -R 4dba3e78e9c48b15e9dce867cfd3b67a -T *branch * scanstatus-exp -T *sym-scanstatus-exp * -T -sym-trunk * +P f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006 +R 24e32cd4f6a370917cd11359ca963cf5 U dan -Z 262a3b7e30cd989cb3c51edc832f13b0 +Z dd437f2b1d92b46f41dc47026405c30b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 848125017e..48abfa6e16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006 \ No newline at end of file +47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2 \ No newline at end of file diff --git a/src/vdbevtab.c b/src/vdbevtab.c index b95852fb5d..59030e0e12 100644 --- a/src/vdbevtab.c +++ b/src/vdbevtab.c @@ -286,10 +286,10 @@ static int bytecodevtabColumn( #ifdef SQLITE_ENABLE_STMT_SCANSTATUS case 9: /* nexec */ - sqlite3_result_int(ctx, pCur->aOp[0].nExec); + sqlite3_result_int(ctx, pOp->nExec); break; case 10: /* ncycle */ - sqlite3_result_int(ctx, pCur->aOp[0].nCycle); + sqlite3_result_int(ctx, pOp->nCycle); break; #else case 9: /* nexec */ From f8be243fefc5459355f4febd82f9b75004c4746f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Jun 2023 19:14:35 +0000 Subject: [PATCH 03/14] Add experimental ".scanstats vm" command to the shell tool. FossilOrigin-Name: e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 44 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 43 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index af6823742c..f50f10ae04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\sprevious\scommit\son\sthis\sbranch. -D 2023-06-30T19:13:27.228 +C Add\sexperimental\s".scanstats\svm"\scommand\sto\sthe\sshell\stool. +D 2023-06-30T19:14:35.552 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -638,7 +638,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88 -F src/shell.c.in 2c02c819349de410d63fcc0217763dfe5a42dbe58f2d68046d4ea8a376d12c26 +F src/shell.c.in 6266eac6a51f1b931d72b3ea95b9f688f390cad7b43bbdb465f216b39eaaa0d3 F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2041,8 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f7b163a319bee9e935a4abf0bd87e16c9974cba5ae75b0cbba63c9da168f2006 -R 24e32cd4f6a370917cd11359ca963cf5 +P 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2 +R 1fc4f436f0d8b18e387b07b97d0465bf U dan -Z dd437f2b1d92b46f41dc47026405c30b +Z 39b6ffde042c1822fbacb7543ed0038f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 48abfa6e16..df83ac764a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2 \ No newline at end of file +e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2b0884e1c0..04407e54ae 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3332,17 +3332,11 @@ static int scanStatsHeight(sqlite3_stmt *p, int iEntry){ } #endif -/* -** Display scan stats. -*/ -static void display_scanstats( +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void display_explain_scanstats( sqlite3 *db, /* Database to query */ ShellState *pArg /* Pointer to ShellState */ ){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else static const int f = SQLITE_SCANSTAT_COMPLEX; sqlite3_stmt *p = pArg->pStmt; int ii = 0; @@ -3414,6 +3408,37 @@ static void display_scanstats( } eqp_render(pArg, nTotal); +} +#endif + +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, "SELECT * FROM bytecode(?)", -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + exec_prepared_stmt(pArg, pStmt); + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } #endif } @@ -9933,6 +9958,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='s' && cli_strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ + if( cli_strcmp(azArg[1], "vm")==0 ){ + p->scanstatsOn = 3; + }else if( cli_strcmp(azArg[1], "est")==0 ){ p->scanstatsOn = 2; }else{ From 3ba691412109e192b5e052e5006337f9ba907fbe Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 26 Jul 2023 07:57:55 +0000 Subject: [PATCH 04/14] Reformulate [907dfc4a7aa1] using awk instead of sed for better cross-platform portability. FossilOrigin-Name: 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da --- ext/wasm/GNUmakefile | 6 +++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 2346773337..45790bd701 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -678,13 +678,13 @@ sqlite3-wasmfs.cfiles := $(sqlite3-wasm.cfiles) # Upstream RFE: # https://github.com/emscripten-core/emscripten/issues/18237 # -# Maintenance reminder: Mac sed -i works differently than GNU sed, so -# don't use that flag here. +# Maintenance reminder: Mac sed works differently than GNU sed, so +# don't use sed for this. define SQLITE3.xJS.ESM-EXPORT-DEFAULT if [ x1 = x$(1) ]; then \ echo "Fragile workaround for emscripten/issues/18237. See SQLITE3.xJS.RECIPE."; \ {\ - sed -e '0,/^export default/{/^export default/d;}' $@ > $@.tmp && mv $@.tmp $@; \ + awk '/^export default/ && !f{f=1; next} 1' $@ > $@.tmp && mv $@.tmp $@; \ } || exit $$?; \ if [ x != x$(2) ]; then \ if ! grep -q '^export default' $@; then \ diff --git a/manifest b/manifest index f9ef9ad44c..5f921c0e86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\s"set-but-not-used"\scompiler\swarning\sin\ssqlite3session.c. -D 2023-07-25T17:54:25.000 +C Reformulate\s[907dfc4a7aa1]\susing\sawk\sinstead\sof\ssed\sfor\sbetter\scross-platform\sportability. +D 2023-07-26T07:57:55.835 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile a3e316bf51d8915a49d4728e0d41f19ec4c3ead3409d1171989a4fe163ec2214 +F ext/wasm/GNUmakefile f1aab41e2afaad68907ff8e8eea32ccca97ab8c216e33dbea817f9e263d4b502 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 907dfc4a7aa129cdcedeb3ba2d75e1b68a8f22c2545ee1c8cf7d705041644e5c -R 262b3aa3dc9bbec239621d23dc374114 -U dan -Z f47917699640df15bd88b843742372bf +P 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1 +R bd64b4635e08e21fec1b91b485560c00 +U stephan +Z 9b053abd49b0547974c7dd9e0baf8df0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7561758d66..9d9c089cd8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1 \ No newline at end of file +82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da \ No newline at end of file From aef4fbf2032894a8b3096953b9aed54763b12e7f Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 26 Jul 2023 11:11:39 +0000 Subject: [PATCH 05/14] Add SAHPoolUtil.getFileNames() method, and tests for it, per [forum:a3da1e34d8|forum feedback]. Add a test to demonstrate that two SAH pools can coexist so long as they have different names. FossilOrigin-Name: 72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e --- ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js | 15 +++++++++++++++ ext/wasm/tester1.c-pp.js | 18 +++++++++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js index 9bf1421a9f..ee8d1a5d68 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js @@ -501,6 +501,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ /* Current number of in-use files from pool. */ getFileCount(){return this.#mapFilenameToSAH.size} + /* Returns an array of the names of all + currently-opened client-specified filenames. */ + getFileNames(){ + const rc = []; + const iter = this.#mapFilenameToSAH.keys(); + for(const n of iter) rc.push(n); + return rc; + } + // #createFileObject(sah,clientName,opaqueName){ // const f = Object.assign(Object.create(null),{ // clientName, opaqueName @@ -901,6 +910,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ getCapacity(){ return this.#p.getCapacity(this.#p) } getFileCount(){ return this.#p.getFileCount() } + getFileNames(){ return this.#p.getFileNames() } async reserveMinimumCapacity(min){ const c = this.#p.getCapacity(); @@ -1059,6 +1069,11 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Returns the number of files from the pool currently allocated to slots. This is not the same as the files being "opened". + - array getFileNames() + + Returns an array of the names of the files currently allocated to + slots. This list is the same length as getFileCount(). + - void importDb(name, byteArray) Imports the contents of an SQLite database, provided as a byte diff --git a/ext/wasm/tester1.c-pp.js b/ext/wasm/tester1.c-pp.js index 905340b234..db4b50f4b6 100644 --- a/ext/wasm/tester1.c-pp.js +++ b/ext/wasm/tester1.c-pp.js @@ -3072,10 +3072,23 @@ globalThis.sqlite3InitModule = sqlite3InitModule; db = new u2.OpfsSAHPoolDb(dbName); T.assert(1 === u1.getFileCount()); db.close(); - T.assert(1 === u1.getFileCount()) + const fileNames = u1.getFileNames(); + T.assert(1 === fileNames.length) + .assert(dbName === fileNames[0]) + .assert(1 === u1.getFileCount()) .assert(true === u1.unlink(dbName)) .assert(false === u1.unlink(dbName)) - .assert(0 === u1.getFileCount()); + .assert(0 === u1.getFileCount()) + .assert(0 === u1.getFileNames().length); + + // Demonstrate that two SAH pools can coexist so long as + // they have different names. + const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); + conf2.name += '-test2'; + const POther = await inst(conf2); + log("Installed second SAH instance as",conf2.name); + T.assert(0 === POther.getFileCount()) + .assert(true === await POther.removeVfs()); if(0){ /* Enable this block to inspect vfs's contents via the dev console or OPFS Explorer browser extension. The @@ -3087,7 +3100,6 @@ globalThis.sqlite3InitModule = sqlite3InitModule; .assert(!sqlite3.capi.sqlite3_vfs_find(sahPoolConfig.name)); let cErr, u3; - const conf2 = JSON.parse(JSON.stringify(sahPoolConfig)); conf2.$testThrowInInit = new Error("Testing throwing during init."); conf2.name = sahPoolConfig.name+'-err'; const P3 = await inst(conf2).then(u=>u3 = u).catch((e)=>cErr=e); diff --git a/manifest b/manifest index 5f921c0e86..5f4f8afbf0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reformulate\s[907dfc4a7aa1]\susing\sawk\sinstead\sof\ssed\sfor\sbetter\scross-platform\sportability. -D 2023-07-26T07:57:55.835 +C Add\sSAHPoolUtil.getFileNames()\smethod,\sand\stests\sfor\sit,\sper\s[forum:a3da1e34d8|forum\sfeedback].\sAdd\sa\stest\sto\sdemonstrate\sthat\stwo\sSAH\spools\scan\scoexist\sso\slong\sas\sthey\shave\sdifferent\snames. +D 2023-07-26T11:11:39.057 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 7931b50b63246a3d6b46a4c703c28820aa10c5b1ae7c0718e1f58dae2cf6db85 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 44740935f6eeab3a0c6dc9f0ca93aebb6339f8d8365f03ea6d7808c134dbfe46 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e7a690e0e78ff4d563f2eca468f91db69f001ff4b79c6d2304cbb6f62dca437d F ext/wasm/api/sqlite3-wasm.c 8867f1d41c112fb4a2cfe22ff224eccaf309fcdea266cee0ec554f85db72ef0f F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555 F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2 -F ext/wasm/tester1.c-pp.js f835c9f703b562142f23a3607fa4a34cb6aece5fb5d674ea5bd7d37b0e47e104 +F ext/wasm/tester1.c-pp.js b9a493a764f2113f6a8205d4e5573a25731d6f965338005460920389ac0693e0 F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1 F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54b3c43fdfdaca6b129a5f0ee93c34eb001663775d33c087066650f5e164d1c1 -R bd64b4635e08e21fec1b91b485560c00 +P 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da +R 49d65d8d7213fb16653e94d68282a21d U stephan -Z 9b053abd49b0547974c7dd9e0baf8df0 +Z ce157938736081078b3928b891966201 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9d9c089cd8..f9d3b51fcc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da \ No newline at end of file +72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e \ No newline at end of file From 8c7ee455ffc3d84dde5e970857931ddf1b8ee0b8 Mon Sep 17 00:00:00 2001 From: stephan Date: Wed, 26 Jul 2023 11:41:41 +0000 Subject: [PATCH 06/14] Remove the batch SQL runner from the JS build's 'all' target, as it's long-since unused and adds noticable build time. Add makefile comments about JSPI. FossilOrigin-Name: b8f708e35d4fa027d12089ac7c5589c36da5f68b98cf9792bb11276bf233a860 --- ext/wasm/GNUmakefile | 9 ++++++++- ext/wasm/index.html | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/ext/wasm/GNUmakefile b/ext/wasm/GNUmakefile index 45790bd701..3cd1d930af 100644 --- a/ext/wasm/GNUmakefile +++ b/ext/wasm/GNUmakefile @@ -451,6 +451,13 @@ emcc.exportedRuntimeMethods := \ emcc.jsflags += $(emcc.exportedRuntimeMethods) emcc.jsflags += -sUSE_CLOSURE_COMPILER=0 emcc.jsflags += -sIMPORTED_MEMORY +#emcc.jsflags += -sASYNCIFY=2 +# ^^^ ASYNCIFY=2 is for experimental JSPI support +# (https://v8.dev/blog/jspi), but enabling it causes the lib-level +# init code to throw inexplicable complaints about C-level function +# signatures not matching what we expect them to be. JSPI requires, as of +# this writing, requires an experimental Chrome flag: +# chrome://flags/#enable-experimental-webassembly-stack-switching emcc.jsflags += -sSTRICT_JS=0 # STRICT_JS disabled due to: # https://github.com/emscripten-core/emscripten/issues/18610 @@ -863,7 +870,7 @@ clean-batch: # a regular basis with different -Ox flags and rebuilding the batch # pieces each time is an unnecessary time sink. batch: batch-runner.list -all: batch +#all: batch # end batch-runner.js ######################################################################## # Wasmified speedtest1 is our primary benchmarking tool. diff --git a/ext/wasm/index.html b/ext/wasm/index.html index d91dca6cb4..70ce0441e0 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -107,7 +107,8 @@
  • module-symbols gives a high-level overview of the symbols exposed by the JS module.
  • -
  • batch-runner: runs batches of SQL exported from speedtest1.
  • +
  • test-opfs-vfs (same with verbose output and sanity-checking tests) is an diff --git a/manifest b/manifest index 5f4f8afbf0..9e1a1f3698 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSAHPoolUtil.getFileNames()\smethod,\sand\stests\sfor\sit,\sper\s[forum:a3da1e34d8|forum\sfeedback].\sAdd\sa\stest\sto\sdemonstrate\sthat\stwo\sSAH\spools\scan\scoexist\sso\slong\sas\sthey\shave\sdifferent\snames. -D 2023-07-26T11:11:39.057 +C Remove\sthe\sbatch\sSQL\srunner\sfrom\sthe\sJS\sbuild's\s'all'\starget,\sas\sit's\slong-since\sunused\sand\sadds\snoticable\sbuild\stime.\sAdd\smakefile\scomments\sabout\sJSPI. +D 2023-07-26T11:41:41.032 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c -F ext/wasm/GNUmakefile f1aab41e2afaad68907ff8e8eea32ccca97ab8c216e33dbea817f9e263d4b502 +F ext/wasm/GNUmakefile ddf1aede4275e404c7eda782462c33b6406fcd2dd327241f6b22c0f7b80938e4 F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576 F ext/wasm/README.md 0895244c0539ae68cf8c70d59c2de512532fd47cfba313268e2b672e6359112e F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab @@ -531,7 +531,7 @@ F ext/wasm/fiddle/fiddle-worker.js 163d6139a93fab4bcb72064923df050d4e7c0ff0d8aa0 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html 22379774f0ad4edcaaa8cf9c674c82e794cc557719a8addabed74eb8069d412e -F ext/wasm/index.html b4e55de741be9fb7656445ea55085f703a784aebde620e1c4852fa21c1ac1c5b +F ext/wasm/index.html 4e7847b909f4ae0da8c829b150b79454050e53b3658431f138636257729cd42b F ext/wasm/jaccwabyt/jaccwabyt.js 1264710db3cfbcb6887d95665b7aeba60c1126eaef789ca4cf1a4a17d5bc7f54 F ext/wasm/jaccwabyt/jaccwabyt.md 37911f00db12cbcca73aa1ed72594430365f30aafae2fa9c886961de74e5e0eb F ext/wasm/module-symbols.html 841de62fc198988b8330e238c260e70ec93028b096e1a1234db31b187a899d10 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 82ff7cc6a4b0331677be87bc069da414a56fd531bae402d0f0808b5d2b0d45da -R 49d65d8d7213fb16653e94d68282a21d +P 72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e +R 9fd280b1c194f8e5991f01cbec9ff41b U stephan -Z ce157938736081078b3928b891966201 +Z 790924414d34e6bddbbcb85cdb1bbe6d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f9d3b51fcc..70d28d9e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72dc3f8c3255186ec412412b685b0b51ddcd08240f2353ac742fc7da8c23568e \ No newline at end of file +b8f708e35d4fa027d12089ac7c5589c36da5f68b98cf9792bb11276bf233a860 \ No newline at end of file From a1016b4275a473586647e9cd545409f709297f25 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Jul 2023 16:41:23 +0000 Subject: [PATCH 07/14] Improve the output when ".scanstats vm" is enabled. FossilOrigin-Name: 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22 --- manifest | 12 ++-- manifest.uuid | 2 +- src/shell.c.in | 181 ++++++++++++++++++++++++++----------------------- 3 files changed, 104 insertions(+), 91 deletions(-) diff --git a/manifest b/manifest index f50f10ae04..963fde8d6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\s".scanstats\svm"\scommand\sto\sthe\sshell\stool. -D 2023-06-30T19:14:35.552 +C Improve\sthe\soutput\swhen\s".scanstats\svm"\sis\senabled. +D 2023-07-26T16:41:23.325 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -638,7 +638,7 @@ F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c F src/resolve.c 37953a5f36c60bea413c3c04efcd433b6177009f508ef2ace0494728912fe2e9 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97 F src/select.c 3ab1186290a311a8ceed1286c0e286209f7fe97b2d02c7593258004ce295dd88 -F src/shell.c.in 6266eac6a51f1b931d72b3ea95b9f688f390cad7b43bbdb465f216b39eaaa0d3 +F src/shell.c.in 921831fe4acca388f66472f6e4076c4531edf17b1b0a64d85843606f339d00d3 F src/sqlite.h.in 3076d78836b6dac53b3ab0875fc8fd15bca8077aad4d33c85336e05af6aef8c7 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h da473ce2b3d0ae407a6300c4a164589b9a6bfdbec9462688a8593ff16f3bb6e4 @@ -2041,8 +2041,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 47c11ca90f98ffc4d91f07e2ab35826a604a2c903008eeddf8b814bb984971f2 -R 1fc4f436f0d8b18e387b07b97d0465bf +P e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d +R cd10daa978b5e0a84d84b4bb41300995 U dan -Z 39b6ffde042c1822fbacb7543ed0038f +Z 2a26d9e4014ba821912c8a2a17121583 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index df83ac764a..88f96d904b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e727640fb5c17d23b8e27972065b4acbf169c9240298d3ff7aed092b727d052d \ No newline at end of file +7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 04407e54ae..57d16e8736 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1607,6 +1607,7 @@ static ShellState shellState; #define MODE_Box 16 /* Unicode box-drawing characters */ #define MODE_Count 17 /* Output only a count of the rows of output */ #define MODE_Off 18 /* No query output shown */ +#define MODE_ScanExp 19 /* Like MODE_Explain, but for ".scanstats vm" */ static const char *modeDescr[] = { "line", @@ -2520,38 +2521,58 @@ static int shell_callback( } break; } + case MODE_ScanExp: case MODE_Explain: { - static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; - if( nArg>ArraySize(aExplainWidth) ){ - nArg = ArraySize(aExplainWidth); + static const int aExplainWidth[] = {4, 13, 4, 4, 4, 13, 2, 13}; + static const int aExplainMap[] = {0, 1, 2, 3, 4, 5, 6, 7 }; + static const int aScanExpWidth[] = {4, 6, 6, 13, 4, 4, 4, 13, 2, 13}; + static const int aScanExpMap[] = {0, 9, 8, 1, 2, 3, 4, 5, 6, 7 }; + + const int *aWidth = aExplainWidth; + const int *aMap = aExplainMap; + int nWidth = ArraySize(aExplainWidth); + int iIndent = 1; + + if( p->cMode==MODE_ScanExp ){ + aWidth = aScanExpWidth; + aMap = aScanExpMap; + nWidth = ArraySize(aScanExpWidth); + iIndent = 3; } + if( nArg>nWidth ) nArg = nWidth; + + /* If this is the first row seen, print out the headers */ if( p->cnt++==0 ){ for(i=0; iout, w, azCol[i]); + utf8_width_print(p->out, aWidth[i], azCol[ aMap[i] ]); fputs(i==nArg-1 ? "\n" : " ", p->out); } for(i=0; iout, w); + print_dashes(p->out, aWidth[i]); fputs(i==nArg-1 ? "\n" : " ", p->out); } } + + /* If there is no data, exit early. */ if( azArg==0 ) break; + for(i=0; iw ){ - w = strlenChar(azArg[i]); + if( zVal && strlenChar(zVal)>w ){ + w = strlenChar(zVal); + zSep = " "; } - if( i==1 && p->aiIndent && p->pStmt ){ + if( i==iIndent && p->aiIndent && p->pStmt ){ if( p->iIndentnIndent ){ utf8_printf(p->out, "%*.s", p->aiIndent[p->iIndent], ""); } p->iIndent++; } - utf8_width_print(p->out, w, azArg[i] ? azArg[i] : p->nullValue); - fputs(i==nArg-1 ? "\n" : " ", p->out); + utf8_width_print(p->out, w, zVal ? zVal : p->nullValue); + fputs(i==nArg-1 ? "\n" : zSep, p->out); } break; } @@ -3411,36 +3432,6 @@ static void display_explain_scanstats( } #endif -static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); - -/* -** Display scan stats. -*/ -static void display_scanstats( - sqlite3 *db, /* Database to query */ - ShellState *pArg /* Pointer to ShellState */ -){ -#ifndef SQLITE_ENABLE_STMT_SCANSTATUS - UNUSED_PARAMETER(db); - UNUSED_PARAMETER(pArg); -#else - if( pArg->scanstatsOn==3 ){ - int rc = SQLITE_OK; - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(db, "SELECT * FROM bytecode(?)", -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - sqlite3_stmt *pSave = pArg->pStmt; - pArg->pStmt = pStmt; - sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); - exec_prepared_stmt(pArg, pStmt); - sqlite3_finalize(pStmt); - pArg->pStmt = pSave; - } - }else{ - display_explain_scanstats(db, pArg); - } -#endif -} /* ** Parameter azArray points to a zero-terminated array of strings. zStr @@ -3478,8 +3469,6 @@ static int str_in_array(const char *zStr, const char **azArray){ ** and "Goto" by 2 spaces. */ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ - const char *zSql; /* The text of the SQL statement */ - const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ @@ -3490,65 +3479,45 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; - /* Try to figure out if this is really an EXPLAIN statement. If this - ** cannot be verified, return early. */ - if( sqlite3_column_count(pSql)!=8 ){ - p->cMode = p->mode; - return; - } - zSql = sqlite3_sql(pSql); - if( zSql==0 ) return; - for(z=zSql; *z==' ' || *z=='\t' || *z=='\n' || *z=='\f' || *z=='\r'; z++); - if( sqlite3_strnicmp(z, "explain", 7) ){ - p->cMode = p->mode; - return; - } + /* The caller guarantees that the leftmost 4 columns of the statement + ** passed to this function are equivalent to the leftmost 4 columns + ** of EXPLAIN statement output. In practice the statement may be + ** an EXPLAIN, or it may be a query on the bytecode() virtual table. */ + assert( sqlite3_column_count(pSql)>=4 ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 0), "addr" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 1), "opcode" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 2), "p1" ) ); + assert( 0==sqlite3_stricmp( sqlite3_column_name(pSql, 3), "p2" ) ); for(iOp=0; SQLITE_ROW==sqlite3_step(pSql); iOp++){ int i; int iAddr = sqlite3_column_int(pSql, 0); const char *zOp = (const char*)sqlite3_column_text(pSql, 1); - - /* Set p2 to the P2 field of the current opcode. Then, assuming that - ** p2 is an instruction address, set variable p2op to the index of that - ** instruction in the aiIndent[] array. p2 and p2op may be different if - ** the current instruction is part of a sub-program generated by an - ** SQL trigger or foreign key. */ + int p1 = sqlite3_column_int(pSql, 2); int p2 = sqlite3_column_int(pSql, 3); + + /* Assuming that p2 is an instruction address, set variable p2op to the + ** index of that instruction in the aiIndent[] array. p2 and p2op may be + ** different if the current instruction is part of a sub-program generated + ** by an SQL trigger or foreign key. */ int p2op = (p2 + (iOp-iAddr)); /* Grow the p->aiIndent array as required */ if( iOp>=nAlloc ){ - if( iOp==0 ){ - /* Do further verification that this is explain output. Abort if - ** it is not */ - static const char *explainCols[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment" }; - int jj; - for(jj=0; jjcMode = p->mode; - sqlite3_reset(pSql); - return; - } - } - } nAlloc += 100; p->aiIndent = (int*)sqlite3_realloc64(p->aiIndent, nAlloc*sizeof(int)); shell_check_oom(p->aiIndent); abYield = (int*)sqlite3_realloc64(abYield, nAlloc*sizeof(int)); shell_check_oom(abYield); } + abYield[iOp] = str_in_array(zOp, azYield); p->aiIndent[iOp] = 0; p->nIndent = iOp+1; - if( str_in_array(zOp, azNext) && p2op>0 ){ for(i=p2op; iaiIndent[i] += 2; } - if( str_in_array(zOp, azGoto) && p2opnIndent - && (abYield[p2op] || sqlite3_column_int(pSql, 2)) - ){ + if( str_in_array(zOp, azGoto) && p2opaiIndent[i] += 2; } } @@ -3568,6 +3537,48 @@ static void explain_data_delete(ShellState *p){ p->iIndent = 0; } +static void exec_prepared_stmt(ShellState*, sqlite3_stmt*); + +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + UNUSED_PARAMETER(db); + UNUSED_PARAMETER(pArg); +#else + if( pArg->scanstatsOn==3 ){ + const char *zSql = + " SELECT addr, opcode, p1, p2, p3, p4, p5, comment, nexec," + " round(ncycle*100.0 / (sum(ncycle) OVER ()), 2)||'%' AS cycles" + " FROM bytecode(?)"; + + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pSave = pArg->pStmt; + pArg->pStmt = pStmt; + sqlite3_bind_pointer(pStmt, 1, pSave, "stmt-pointer", 0); + + pArg->cnt = 0; + pArg->cMode = MODE_ScanExp; + explain_data_prepare(pArg, pStmt); + exec_prepared_stmt(pArg, pStmt); + explain_data_delete(pArg); + + sqlite3_finalize(pStmt); + pArg->pStmt = pSave; + } + }else{ + display_explain_scanstats(db, pArg); + } +#endif +} + /* ** Disable and restore .wheretrace and .treetrace/.selecttrace settings. */ @@ -4387,6 +4398,7 @@ static int shell_exec( rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ pArg->cMode = MODE_Explain; + assert( sqlite3_stmt_isexplain(pExplain)==1 ); explain_data_prepare(pArg, pExplain); exec_prepared_stmt(pArg, pExplain); explain_data_delete(pArg); @@ -4405,9 +4417,10 @@ static int shell_exec( } if( pArg ){ + int bIsExplain = (sqlite3_stmt_isexplain(pStmt)==1); pArg->cMode = pArg->mode; if( pArg->autoExplain ){ - if( sqlite3_stmt_isexplain(pStmt)==1 ){ + if( bIsExplain ){ pArg->cMode = MODE_Explain; } if( sqlite3_stmt_isexplain(pStmt)==2 ){ @@ -4417,7 +4430,7 @@ static int shell_exec( /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ - if( pArg->cMode==MODE_Explain ){ + if( pArg->cMode==MODE_Explain && bIsExplain ){ explain_data_prepare(pArg, pStmt); } } From beb3fb690469c90bb77b00b041ff474cc6e76cf2 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 19:11:47 +0000 Subject: [PATCH 08/14] Make sure jsonReplaceNode() always leaves the JsonParse in a consistent state even if an error is encountered. FossilOrigin-Name: 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/json.c | 1 + 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 96c91cdd84..5c7bcfa676 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".scanstats\svm"\scommand\sto\sthe\sshell\stool.\sFor\sprofiling\sVM\scode\sin\sSQLITE_ENABLE_STMT_SCANSTATUS\sbuilds. -D 2023-07-26T17:51:05.557 +C Make\ssure\sjsonReplaceNode()\salways\sleaves\sthe\sJsonParse\sin\sa\sconsistent\sstate\seven\sif\san\serror\sis\sencountered. +D 2023-07-26T19:11:47.116 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 512bc389b42c68b34571ca532afb0f4bb235b1fc11ea5c9a4f6850c64fa12ab4 +F src/json.c 00ee51e8365de51d76473900c181edb3265cde636badc779068bbb425e401211 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,9 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P df099ad713011b67b09cb1e5f0fa2e6b45f9cee0ce9d3c118c5dbca3563d20a5 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22 -R 41e3bcc8b2de77a8519996fee810e7fd -T +closed 7df08fd35e9d4bc471aa9fbc4c81d2ebcfd2be6c4c38143342b3d9d727c9df22 -U dan -Z 4c8141df90dc53dee045e3726a5dbff7 +P 0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d +R f441bd92e9087b2cb305790bb3fcf419 +U drh +Z 5cac453cc99ba9487a886bdc07348b7a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a215484c0d..4831e7a0a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d \ No newline at end of file +01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b \ No newline at end of file diff --git a/src/json.c b/src/json.c index 9eb2302684..7fb72aecb3 100644 --- a/src/json.c +++ b/src/json.c @@ -2853,6 +2853,7 @@ static void jsonReplaceNode( break; } case SQLITE_BLOB: { + jsonParseAddNode(p, JSON_NULL, 0, 0); sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); p->nErr++; break; From 3f3dc2f5d8c8fb5fad4d94f5d1dc05926cb34794 Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 19:22:43 +0000 Subject: [PATCH 09/14] Change a switch() case to default for coverage. FossilOrigin-Name: 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5c7bcfa676..8b99a3700b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sjsonReplaceNode()\salways\sleaves\sthe\sJsonParse\sin\sa\sconsistent\sstate\seven\sif\san\serror\sis\sencountered. -D 2023-07-26T19:11:47.116 +C Change\sa\sswitch()\scase\sto\sdefault\sfor\scoverage. +D 2023-07-26T19:22:43.180 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 00ee51e8365de51d76473900c181edb3265cde636badc779068bbb425e401211 +F src/json.c e3675df7edfd0874bed04915de9a4d91300e6745a1359457e617b50049435b0d F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0cbec3990d4101142bfb831f8e6527b73baabebbd30fa7f59275b217dbce6a8d -R f441bd92e9087b2cb305790bb3fcf419 +P 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b +R 54cc47c29344f6b15ce5cc5de43ab981 U drh -Z 5cac453cc99ba9487a886bdc07348b7a +Z b1ee9ef830f4f85e2fbcc03f50cf7801 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 4831e7a0a7..6845fa3cbe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b \ No newline at end of file +04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7fb72aecb3..26ce90c516 100644 --- a/src/json.c +++ b/src/json.c @@ -2852,7 +2852,7 @@ static void jsonReplaceNode( } break; } - case SQLITE_BLOB: { + default: { jsonParseAddNode(p, JSON_NULL, 0, 0); sqlite3_result_error(pCtx, "JSON cannot hold BLOB values", -1); p->nErr++; From f7a164f34518825ae0351fd1c4453df8f30c3afe Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 21:53:09 +0000 Subject: [PATCH 10/14] Minor changes to make coverage testing easier. FossilOrigin-Name: ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8b99a3700b..a2ffc6aed4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\sswitch()\scase\sto\sdefault\sfor\scoverage. -D 2023-07-26T19:22:43.180 +C Minor\schanges\sto\smake\scoverage\stesting\seasier. +D 2023-07-26T21:53:09.569 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c e3675df7edfd0874bed04915de9a4d91300e6745a1359457e617b50049435b0d +F src/json.c 8c21f66368929975ed41e6f5aff141b5ff2706bc0f817b0a99dfea9cb6b1b7cd F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 01d52232dd6fbd253e77419a17df3df83d49434792d288ef96e14739a89cef3b -R 54cc47c29344f6b15ce5cc5de43ab981 +P 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7 +R b26ff6d575d69d55cdae9f68086f2a13 U drh -Z b1ee9ef830f4f85e2fbcc03f50cf7801 +Z a6ad53b5bf3028b66640cc0d3b3e396d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 6845fa3cbe..53efddff64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7 \ No newline at end of file +ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 26ce90c516..7fb860a6ef 100644 --- a/src/json.c +++ b/src/json.c @@ -2828,14 +2828,17 @@ static void jsonReplaceNode( break; } if( sqlite3_value_subtype(pValue)!=JSON_SUBTYPE ){ - int k = jsonParseAddNode(p, JSON_STRING, n, z); char *zCopy = sqlite3DbStrDup(0, z); - if( k>0 ) p->aNode[k].jnFlags |= JNODE_RAW; + int k; if( zCopy ){ jsonParseAddCleanup(p, sqlite3_free, zCopy); - }else{ + }else{ + p->oom = 1; sqlite3_result_error_nomem(pCtx); } + k = jsonParseAddNode(p, JSON_STRING, n, zCopy); + assert( k>0 || p->oom ); + if( p->oom==0 ) p->aNode[k].jnFlags |= JNODE_RAW; }else{ JsonParse *pPatch = jsonParseCached(pCtx, pValue, pCtx, 1); if( pPatch==0 ){ From e94e132994e5144c39bbc68aa4a6180a3e60fd7a Mon Sep 17 00:00:00 2001 From: drh <> Date: Wed, 26 Jul 2023 23:22:32 +0000 Subject: [PATCH 11/14] Reduce the number of memory allocations when parsing JSON. FossilOrigin-Name: 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 31 ++++++++----------------------- 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index a2ffc6aed4..0f8c113090 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\smake\scoverage\stesting\seasier. -D 2023-07-26T21:53:09.569 +C Reduce\sthe\snumber\sof\smemory\sallocations\swhen\sparsing\sJSON. +D 2023-07-26T23:22:32.813 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 8c21f66368929975ed41e6f5aff141b5ff2706bc0f817b0a99dfea9cb6b1b7cd +F src/json.c bea467f5bb75311759f2a5f8a7bbf517384f6489bcbe85a41b7c39e8faeea698 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 04f497074b9210326030f36107a43d6490a2a59c8a574e2c5429cd9bde681bf7 -R b26ff6d575d69d55cdae9f68086f2a13 +P ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764 +R 60fa67ed7b82b4d2c0db2475f139f2f9 U drh -Z a6ad53b5bf3028b66640cc0d3b3e396d +Z 0d5cf9ec935b3fe18206d9fca33e2d88 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 53efddff64..46453a3ccb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764 \ No newline at end of file +9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 7fb860a6ef..24b3613e14 100644 --- a/src/json.c +++ b/src/json.c @@ -144,8 +144,7 @@ struct JsonNode { /* A parsed and possibly edited JSON string. Lifecycle: ** ** 1. JSON comes in and is parsed into an array aNode[]. The original -** JSON text is stored in zJson. This object may or may not be the -** owner of the input JSON - the bOwnsJson variables determines which. +** JSON text is stored in zJson. ** ** 2. Zero or more changes are made (via json_remove() or json_replace() ** or similar) to the aNode[] array. @@ -176,7 +175,6 @@ struct JsonParse { u8 nErr; /* Number of errors seen */ u8 oom; /* Set to true if out of memory */ u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ - u8 bOwnsJson; /* This object owns zJson and response for freeing it */ u8 useMod; /* Actually use the edits contain inside aNode */ u8 hasMod; /* aNode contains edits from the original zJson */ u32 nJPRef; /* Number of references to this object */ @@ -619,11 +617,6 @@ static void jsonParseReset(JsonParse *pParse){ sqlite3RCStrUnref(pParse->zAlt); pParse->zAlt = 0; } - if( pParse->bOwnsJson ){ - pParse->bOwnsJson = 0; - sqlite3RCStrUnref(pParse->zJson); - pParse->zJson = 0; - } } /* @@ -1759,14 +1752,12 @@ json_parse_restart: static int jsonParse( JsonParse *pParse, /* Initialize and fill this JsonParse object */ sqlite3_context *pCtx, /* Report errors here */ - char *zJson, /* Input JSON text to be parsed */ - int bTakeJson /* Assume ownership of zJson if true */ + char *zJson /* Input JSON text to be parsed */ ){ int i; memset(pParse, 0, sizeof(*pParse)); if( zJson==0 ) return 1; pParse->zJson = zJson; - pParse->bOwnsJson = bTakeJson; pParse->nJPRef = 1; i = jsonParseValue(pParse, 0); if( pParse->oom ) i = -1; @@ -1933,21 +1924,16 @@ static JsonParse *jsonParseCached( /* The input JSON was not found anywhere in the cache. We will need ** to parse it ourselves and generate a new JsonParse object. */ - p = sqlite3_malloc64( sizeof(*p) ); + p = sqlite3_malloc64( sizeof(*p) + nJson + 1 ); if( p==0 ){ sqlite3_result_error_nomem(pCtx); return 0; } memset(p, 0, sizeof(*p)); - p->zJson = sqlite3RCStrNew( nJson ); - if( p->zJson==0 ){ - sqlite3_free(p); - sqlite3_result_error_nomem(pCtx); - return 0; - } + p->zJson = (char*)&p[1]; memcpy(p->zJson, zJson, nJson); p->zJson[nJson] = 0; - if( jsonParse(p, pErrCtx, p->zJson, 1) ){ + if( jsonParse(p, pErrCtx, p->zJson) ){ if( pErrCtx==0 ){ p->nErr = 1; assert( p->nJPRef==1 ); /* Caller will own the new JsonParse object p */ @@ -2365,7 +2351,6 @@ static void jsonParseFunc( printf("nErr = %u\n", p->nErr); printf("oom = %u\n", p->oom); printf("hasNonstd = %u\n", p->hasNonstd); - printf("bOwnsJson = %u\n", p->bOwnsJson); printf("useMod = %u\n", p->useMod); printf("hasMod = %u\n", p->hasMod); printf("nJPRef = %u\n", p->nJPRef); @@ -2677,8 +2662,8 @@ static void jsonPatchFunc( JsonNode *pResult; /* The result of the merge */ UNUSED_PARAMETER(argc); - if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0]), 0) ) return; - if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1]), 0) ){ + if( jsonParse(&x, ctx, (char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (char*)sqlite3_value_text(argv[1])) ){ jsonParseReset(&x); return; } @@ -3645,7 +3630,7 @@ static int jsonEachFilter( p->zJson = sqlite3_malloc64( n+1 ); if( p->zJson==0 ) return SQLITE_NOMEM; memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson, 0) ){ + if( jsonParse(&p->sParse, 0, p->zJson) ){ int rc = SQLITE_NOMEM; if( p->sParse.oom==0 ){ sqlite3_free(cur->pVtab->zErrMsg); From 93853a48465e5eb14ab347eecc53f2b58a45ee02 Mon Sep 17 00:00:00 2001 From: drh <> Date: Thu, 27 Jul 2023 00:21:59 +0000 Subject: [PATCH 12/14] Fix a performance regression in JSON associated with generating small snippets of JSON from a larger JSON string. FossilOrigin-Name: 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/json.c | 18 ++++++++++++------ 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 0f8c113090..b9a6b41406 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\smemory\sallocations\swhen\sparsing\sJSON. -D 2023-07-26T23:22:32.813 +C Fix\sa\sperformance\sregression\sin\sJSON\sassociated\swith\sgenerating\ssmall\nsnippets\sof\sJSON\sfrom\sa\slarger\sJSON\sstring. +D 2023-07-27T00:21:59.567 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -598,7 +598,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c bea467f5bb75311759f2a5f8a7bbf517384f6489bcbe85a41b7c39e8faeea698 +F src/json.c c992d2a87a6888bf16a5c7270ef523eb838cf49e6643420c3cf917124f41f97f F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 176d6b2cb18a6ad73b133db17f6fc351c4d9a2d510deebdb76c22bde9cfd1465 F src/main.c 512b1d45bc556edf4471a845afb7ba79e64bd5b832ab222dc195c469534cd002 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ec8b43382e5402e15d9f2dda3cf21ac8be8c1589ddbe6c9433c33eef0036f764 -R 60fa67ed7b82b4d2c0db2475f139f2f9 +P 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3 +R 2d6be27c61998a8da52821b68136a1e9 U drh -Z 0d5cf9ec935b3fe18206d9fca33e2d88 +Z 453c25d31204436e7cf8e11e8d84051c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 46453a3ccb..d9fba749f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3 \ No newline at end of file +837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4 \ No newline at end of file diff --git a/src/json.c b/src/json.c index 24b3613e14..493a1b99c4 100644 --- a/src/json.c +++ b/src/json.c @@ -563,12 +563,18 @@ static void jsonAppendValue( ** The JSON string is reset. */ static void jsonResult(JsonString *p){ - if( p->bErr==0 && jsonForceRCStr(p) ){ - sqlite3RCStrRef(p->zBuf); - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - (void(*)(void*))sqlite3RCStrUnref, - SQLITE_UTF8); - }else if( p->bErr==1 ){ + if( p->bErr==0 ){ + if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonForceRCStr(p) ){ + sqlite3RCStrRef(p->zBuf); + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + (void(*)(void*))sqlite3RCStrUnref, + SQLITE_UTF8); + } + } + if( p->bErr==1 ){ sqlite3_result_error_nomem(p->pCtx); } jsonReset(p); From 4cfd54256dca63f95717428416033237e3bad607 Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 27 Jul 2023 01:38:19 +0000 Subject: [PATCH 13/14] Accommodate a breaking change in emcc 3.1.44. FossilOrigin-Name: 2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d --- ext/wasm/api/sqlite3-api-cleanup.js | 4 +++- ext/wasm/common/whwasmutil.js | 3 ++- manifest | 17 +++++++++-------- manifest.uuid | 2 +- 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js index 3661464e1b..65dbb4eb64 100644 --- a/ext/wasm/api/sqlite3-api-cleanup.js +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -22,7 +22,9 @@ if('undefined' !== typeof Module){ // presumably an Emscripten build */ const SABC = Object.assign( Object.create(null), { - exports: Module['asm'], + exports: ('undefined'===typeof wasmExports) + ? Module['asm']/* emscripten <=3.1.43 */ + : wasmExports /* emscripten >=3.1.44 */, memory: Module.wasmMemory /* gets set if built with -sIMPORTED_MEMORY */ }, globalThis.sqlite3ApiConfig || {} diff --git a/ext/wasm/common/whwasmutil.js b/ext/wasm/common/whwasmutil.js index f48e8a7d21..ee7ea20c34 100644 --- a/ext/wasm/common/whwasmutil.js +++ b/ext/wasm/common/whwasmutil.js @@ -111,7 +111,8 @@ of `target.instance` (a WebAssembly.Module instance) and it must contain the symbols exported by the WASM module associated with this code. In an Enscripten environment it must be set to - `Module['asm']`. The exports object must contain a minimum of the + `Module['asm']` (versions <=3.1.43) or `wasmExports` (versions + >=3.1.44). The exports object must contain a minimum of the following symbols: - `memory`: a WebAssembly.Memory object representing the WASM diff --git a/manifest b/manifest index b9a6b41406..f38df6f97b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sperformance\sregression\sin\sJSON\sassociated\swith\sgenerating\ssmall\nsnippets\sof\sJSON\sfrom\sa\slarger\sJSON\sstring. -D 2023-07-27T00:21:59.567 +C Accommodate\sa\sbreaking\schange\sin\semcc\s3.1.44. +D 2023-07-27T01:38:19.863 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -494,7 +494,7 @@ F ext/wasm/api/extern-pre-js.js cc61c09c7a24a07dbecb4c352453c3985170cec12b4e7e7e F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08902f15c34720ee4a1 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 -F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf +F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a @@ -513,7 +513,7 @@ F ext/wasm/c-pp.c 6d80d8569d85713effe8b0818a3cf51dc779e3f0bf8dc88771b8998552ee25 F ext/wasm/common/SqliteTestUtil.js 7adaeffef757d8708418dc9190f72df22367b531831775804b31598b44f6aa51 F ext/wasm/common/emscripten.css 11bd104b6c0d597c67d40cc8ecc0a60dae2b965151e3b6a37fa5708bac3acd15 F ext/wasm/common/testing.css e97549bab24126c24e0daabfe2de9bb478fb0a69fdb2ddd0a73a992c091aad6f -F ext/wasm/common/whwasmutil.js ae263dec9d7384f4c530f324b99d00516a4d6f26424372daee65031e00eb49b3 +F ext/wasm/common/whwasmutil.js db6368ee57af90ee6691b6fb3ca97ee8064d12482e06a29113127c67b08f956e F ext/wasm/demo-123-worker.html a0b58d9caef098a626a1a1db567076fca4245e8d60ba94557ede8684350a81ed F ext/wasm/demo-123.html 8c70a412ce386bd3796534257935eb1e3ea5c581e5d5aea0490b8232e570a508 F ext/wasm/demo-123.js 38aa8faec4d0ace1c973bc8a7a1533584463ebeecd4c420daa7d9687beeb9cb5 @@ -2044,8 +2044,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9edd67162113df57dae21d4683f9495611e2cf4717c6d12f5b7b8e44156d5fe3 -R 2d6be27c61998a8da52821b68136a1e9 -U drh -Z 453c25d31204436e7cf8e11e8d84051c +P 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4 +Q +4ce386030092b8bafe860350c66ef5516b6aea75bd1b4467ac184875109d66e7 +R d8cc9478d62bd3b1eb88b1d0f9b38e0e +U stephan +Z 31e0335b3afe627b111db5001ad6debf # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9fba749f0..81aebb401a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4 \ No newline at end of file +2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d \ No newline at end of file From 9fcca3d27c18693eddf65a0edc0f4e609a3a24fc Mon Sep 17 00:00:00 2001 From: stephan Date: Thu, 27 Jul 2023 17:50:10 +0000 Subject: [PATCH 14/14] Dynamically determine whether the wasm.xWrap() argc check can be applied, depending on how the wasm environment exposes its exports. FossilOrigin-Name: fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c --- ext/wasm/api/sqlite3-api-glue.js | 9 +++++++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index f444ec975c..572efeed5c 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -728,6 +728,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ Populate api object with sqlite3_...() by binding the "raw" wasm exports into type-converting proxies using wasm.xWrap(). */ + if(0 === wasm.exports.sqlite3_step.length){ + /* This environment wraps exports in nullary functions, which means + we must disable the arg-count validation we otherwise perform + on the wrappers. */ + wasm.xWrap.doArgcCheck = false; + sqlite3.config.warn( + "Disabling sqlite3.wasm.xWrap.doArgcCheck due to environmental quirks." + ); + } for(const e of wasm.bindingSignatures){ capi[e[0]] = wasm.xWrap.apply(null, e); } diff --git a/manifest b/manifest index f38df6f97b..87e6fa0b52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Accommodate\sa\sbreaking\schange\sin\semcc\s3.1.44. -D 2023-07-27T01:38:19.863 +C Dynamically\sdetermine\swhether\sthe\swasm.xWrap()\sargc\scheck\scan\sbe\sapplied,\sdepending\son\show\sthe\swasm\senvironment\sexposes\sits\sexports. +D 2023-07-27T17:50:10.521 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -495,7 +495,7 @@ F ext/wasm/api/post-js-footer.js cd0a8ec768501d9bd45d325ab0442037fb0e33d1f3b4f08 F ext/wasm/api/post-js-header.js 47b6b281f39ad59fa6e8b658308cd98ea292c286a68407b35ff3ed9cfd281a62 F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219 F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e -F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 +F ext/wasm/api/sqlite3-api-glue.js cc6b0bb093bdb6279d4af259200b7b9e150e3796a8a3a4cd09a4928c43d25e56 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 F ext/wasm/api/sqlite3-api-prologue.js cbd7d6ba185f3a844a8b0020e954b49bbc2ca78b305d117bec2ceca21431795a F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec @@ -2044,9 +2044,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 837f2907e10b026f6db1ca2d44b4bf60a6f069bf534bf369ad9b5c513cb0c6e4 -Q +4ce386030092b8bafe860350c66ef5516b6aea75bd1b4467ac184875109d66e7 -R d8cc9478d62bd3b1eb88b1d0f9b38e0e +P 2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d +Q +86bb464f31b77ac7a89ee9a50c3d988a186f84fa0bbee16b15fcbbac33523fa1 +R 26c73044c10199d992b00944e358a20b U stephan -Z 31e0335b3afe627b111db5001ad6debf +Z 7df747b854c1185863732d1df816e30a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 81aebb401a..7ece24bb69 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c5dd34199f5bcf729be814b8b46d9997821fe3a39ab12779c93df1bb2fd108d \ No newline at end of file +fd59226b34fffb1479fb2d7bd7c0aff982aa4a1a73e6c0d81de6eaf9c075998c \ No newline at end of file