From 4f5bbedb3aad0095caef387dc66e884bfc1f1675 Mon Sep 17 00:00:00 2001 From: stephan Date: Mon, 3 Oct 2022 13:03:41 +0000 Subject: [PATCH] Export sqlite3_trace_v2() to wasm and use it to ensure that the new per-VFS post-open SQL support in the DB ctor works. Default opfs vfs to journal_mode=truncate, as it's faster in that mode. Add 't' DB open-mode flag to enable SQL tracing to console.log(). FossilOrigin-Name: 508f7f6d63e52f61fae5abe817579a4e130fa7fbd18733d741d521a5bdabb7ce --- ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 1 + ext/wasm/api/sqlite3-api-glue.js | 2 +- ext/wasm/api/sqlite3-api-oo1.js | 88 +++++++++++++++------ ext/wasm/api/sqlite3-api-opfs.js | 9 ++- ext/wasm/api/sqlite3-api-prologue.js | 1 + ext/wasm/api/sqlite3-wasm.c | 29 +++++++ ext/wasm/sqlite3-opfs-async-proxy.js | 6 +- ext/wasm/test-opfs-vfs.js | 2 +- manifest | 26 +++--- manifest.uuid | 2 +- 10 files changed, 122 insertions(+), 44 deletions(-) diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api index 014812946d..506775d2fe 100644 --- a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -74,6 +74,7 @@ _sqlite3_strglob _sqlite3_strlike _sqlite3_total_changes _sqlite3_total_changes64 +_sqlite3_trace_v2 _sqlite3_uri_boolean _sqlite3_uri_int64 _sqlite3_uri_key diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js index e31618d5eb..a4c9627bdd 100644 --- a/ext/wasm/api/sqlite3-api-glue.js +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -621,7 +621,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ for(const t of ['access', 'blobFinalizers', 'dataTypes', 'encodings', 'fcntl', 'flock', 'ioCap', 'openFlags', 'prepareFlags', 'resultCodes', - 'serialize', 'syncFlags', 'udfFlags', + 'serialize', 'syncFlags', 'trace', 'udfFlags', 'version' ]){ for(const e of Object.entries(wasm.ctype[t])){ diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js index e6685d35a7..37c2928406 100644 --- a/ext/wasm/api/sqlite3-api-oo1.js +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -61,6 +61,25 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } }; + /** + sqlite3_trace_v2() callback which gets installed by the DB ctor + if its open-flags contain "t". + */ + const __dbTraceToConsole = + wasm.installFunction('i(ippp)', function(t,c,p,x){ + if(capi.SQLITE_TRACE_STMT===t){ + // x == SQL, p == sqlite3_stmt* + console.log("SQL TRACE #"+(++this.counter), + wasm.cstringToJs(x)); + } + }.bind({counter: 0})); + + /** + A map of sqlite3_vfs pointers to SQL code to run when the DB + constructor opens a database with the given VFS. + */ + const __vfsPostOpenSql = Object.create(null); + /** A proxy for DB class constructors. It must be called with the being-construct DB object as its "this". See the DB constructor @@ -101,12 +120,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") : false; ctor._name2vfs[':localStorage:'] = { - vfs: 'kvvfs', - filename: isWorkerThread || (()=>'local') + vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') }; ctor._name2vfs[':sessionStorage:'] = { - vfs: 'kvvfs', - filename: isWorkerThread || (()=>'session') + vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') }; } const opt = ctor.normalizeArgs(...args); @@ -123,7 +140,7 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ vfsName = vfsCheck.vfs; fn = fnJs = vfsCheck.filename(fnJs); } - let ptr, oflags = 0; + let pDb, oflags = 0; if( flagsStr.indexOf('c')>=0 ){ oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; } @@ -132,24 +149,48 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ oflags |= capi.SQLITE_OPEN_EXRESCODE; const scope = wasm.scopedAllocPush(); try { - const ppDb = wasm.allocPtr() /* output (sqlite3**) arg */; + const pPtr = wasm.allocPtr() /* output (sqlite3**) arg */; const pVfsName = vfsName ? ( ('number'===typeof vfsName ? vfsName : wasm.scopedAllocCString(vfsName)) ): 0; - const rc = capi.sqlite3_open_v2(fn, ppDb, oflags, pVfsName); - ptr = wasm.getPtrValue(ppDb); - checkSqlite3Rc(ptr, rc); + let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, pVfsName); + pDb = wasm.getPtrValue(pPtr); + checkSqlite3Rc(pDb, rc); + if(flagsStr.indexOf('t')>=0){ + capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, + __dbTraceToConsole, 0); + } + // Check for per-VFS post-open SQL... + wasm.setPtrValue(pPtr, 0); + if(0===capi.sqlite3_file_control( + pDb, "main", capi.SQLITE_FCNTL_VFS_POINTER, pPtr + )){ + const postInitSql = __vfsPostOpenSql[wasm.getPtrValue(pPtr)]; + if(postInitSql){ + rc = capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0); + checkSqlite3Rc(pDb, rc); + } + } }catch( e ){ - if( ptr ) capi.sqlite3_close_v2(ptr); + if( pDb ) capi.sqlite3_close_v2(pDb); throw e; }finally{ wasm.scopedAllocPop(scope); } this.filename = fnJs; - __ptrMap.set(this, ptr); + __ptrMap.set(this, pDb); __stmtMap.set(this, Object.create(null)); }; + /** + Sets SQL which should be exec()'d on a DB instance after it is + opened with the given VFS pointer. This is intended only for use + by DB subclasses or sqlite3_vfs implementations. + */ + dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ + __vfsPostOpenSql[pVfs] = sql; + }; + /** A helper for DB constructors. It accepts either a single config-style object or up to 3 arguments (filename, dbOpenFlags, @@ -175,7 +216,6 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ } return arg; }; - /** The DB class provides a high-level OO wrapper around an sqlite3 db handle. @@ -193,14 +233,18 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ database. It must be string containing a sequence of letters (in any order, but case sensitive) specifying the mode: - - "c" => create if it does not exist, else fail if it does not + - "c": create if it does not exist, else fail if it does not exist. Implies the "w" flag. - - "w" => write. Implies "r": a db cannot be write-only. + - "w": write. Implies "r": a db cannot be write-only. - - "r" => read-only if neither "w" nor "c" are provided, else it + - "r": read-only if neither "w" nor "c" are provided, else it is ignored. + - "t": enable tracing of SQL executed on this database handle, + sending it to `console.log()`. Once enabled, it cannot + currently be easily switched off (TODO). + If "w" is not provided, the db is implicitly read-only, noting that "rc" is meaningless @@ -229,16 +273,10 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ `sqlite3*` pointer value. That property can also be used to check whether this DB instance is still open. - - EXPERIMENTAL: in the main window thread, the filenames - ":localStorage:" and ":sessionStorage:" are special: they cause - the db to use either localStorage or sessionStorage for storing - the database. In this mode, only a single database is permitted - in each storage object. This feature is experimental and subject - to any number of changes (including outright removal). This - support requires the kvvfs sqlite3 VFS, the existence of which - can be determined at runtime by checking for a non-0 return value - from sqlite3.capi.sqlite3_vfs_find("kvvfs"). + In the main window thread, the filenames ":localStorage:" and + ":sessionStorage:" are special: they cause the db to use either + localStorage or sessionStorage for storing the database using + the kvvfs. */ const DB = function(...args){ dbCtorHelper.apply(this, args); diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js index 9588fc8a20..42c0e11662 100644 --- a/ext/wasm/api/sqlite3-api-opfs.js +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -898,8 +898,14 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ sqlite3.oo1.dbCtorHelper.call(this, opt); }; opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + /* Truncate journal mode is faster than delete or wal for + OPFS, per speedtest1. */ + "pragma journal_mode=truncate" + ); } - + /** Potential TODOs: @@ -907,7 +913,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){ publish an interface for proxying the higher-level OPFS features like getting a directory listing. */ - const sanityCheck = function(){ const scope = wasm.scopedAllocPush(); const sq3File = new sqlite3_file(); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index 83e47cc93c..377f671fce 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -815,6 +815,7 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( ["sqlite3_step", "int", "sqlite3_stmt*"], ["sqlite3_strglob", "int", "string","string"], ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], ["sqlite3_total_changes", "int", "sqlite3*"], ["sqlite3_uri_boolean", "int", "string", "string", "int"], ["sqlite3_uri_key", "string", "string", "int"], diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c index b9454155d7..99196db159 100644 --- a/ext/wasm/api/sqlite3-wasm.c +++ b/ext/wasm/api/sqlite3-wasm.c @@ -527,6 +527,13 @@ const char * sqlite3_wasm_enum_json(void){ DefInt(SQLITE_SYNC_DATAONLY); } _DefGroup; + DefGroup(trace) { + DefInt(SQLITE_TRACE_STMT); + DefInt(SQLITE_TRACE_PROFILE); + DefInt(SQLITE_TRACE_ROW); + DefInt(SQLITE_TRACE_CLOSE); + } _DefGroup; + DefGroup(udfFlags) { DefInt(SQLITE_DETERMINISTIC); DefInt(SQLITE_DIRECTONLY); @@ -680,6 +687,28 @@ int sqlite3_wasm_vfs_unlink(const char * zName){ return rc; } +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function resets the given db pointer's database as described at +** +** https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase +** +** Returns 0 on success, an SQLITE_xxx code on error. Returns +** SQLITE_MISUSE if pDb is NULL. +*/ +WASM_KEEP +int sqlite3_wasm_db_reset(sqlite3*pDb){ + int rc = SQLITE_MISUSE; + if( pDb ){ + rc = sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } + return rc; +} + /* ** Uses the current database's VFS xRead to stream the db file's ** contents out to the given callback. The callback gets a single diff --git a/ext/wasm/sqlite3-opfs-async-proxy.js b/ext/wasm/sqlite3-opfs-async-proxy.js index b951873ea5..0abc3ee233 100644 --- a/ext/wasm/sqlite3-opfs-async-proxy.js +++ b/ext/wasm/sqlite3-opfs-async-proxy.js @@ -609,7 +609,11 @@ const waitLoop = async function f(){ now - relinquishTime >= fh.syncHandleTime )){ //warn("Relinquishing for timeout:",fh.filenameAbs); - closeSyncHandle(fh)/*async!*/; + await closeSyncHandle(fh) + /* Testing shows that we have to wait on this async + op to finish, else we might try to re-open it + before the close has run. The FS layer does not + retain the order those operations, apparently. */; } } } diff --git a/ext/wasm/test-opfs-vfs.js b/ext/wasm/test-opfs-vfs.js index 755b73a97c..2b22a7e96c 100644 --- a/ext/wasm/test-opfs-vfs.js +++ b/ext/wasm/test-opfs-vfs.js @@ -43,7 +43,7 @@ const tryOpfsVfs = async function(sqlite3){ const dbFile = "my-persistent.db"; if(urlArgs.has('delete')) sqlite3.opfs.deleteEntry(dbFile); - const db = new opfs.OpfsDb(dbFile); + const db = new opfs.OpfsDb(dbFile,'ct'); log("db file:",db.filename); await waitForRelinquish(); try{ diff --git a/manifest b/manifest index 11baf122fc..5a7dbf25a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reimplement\sfiddle_reset_db()\sso\sthat\sit\sworks\swith\sall\sVFSes. -D 2022-10-03T11:42:45.345 +C Export\ssqlite3_trace_v2()\sto\swasm\sand\suse\sit\sto\sensure\sthat\sthe\snew\sper-VFS\spost-open\sSQL\ssupport\sin\sthe\sDB\sctor\sworks.\sDefault\sopfs\svfs\sto\sjournal_mode=truncate,\sas\sit's\sfaster\sin\sthat\smode.\sAdd\s't'\sDB\sopen-mode\sflag\sto\senable\sSQL\stracing\sto\sconsole.log(). +D 2022-10-03T13:03:41.207 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -476,7 +476,7 @@ F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34ce F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle 0e88c8cfc3719e4b7e74980d9da664c709e68acf863e48386cda376edfd3bfb0 F ext/wasm/GNUmakefile b313a82060c733c990b91afa981e10f5e21a0b33a483f33b739ce932ed6bc725 F ext/wasm/README.md 1e5b28158b74ab3ffc9d54fcbc020f0bbeb82c2ff8bbd904214c86c70e8a3066 -F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 9b16040f37805ee7c30f922a970a57d3f2a822d0675a8f5d70f15061e300c4ce +F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 89983a8d122c35a90c65ec667844b95a78bcd04f3198a99c1e0c8368c1a0b03a F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287 F ext/wasm/api/README.md 1e350b611465566cfa2e5eccf7c9b29a34f48ee38bbf6d5fb086dd06ce32b3ff F ext/wasm/api/extern-post-js.js dc68cbf552d8ea085181400a6963907c32e0b088b03ffd8969b1869fea246629 @@ -485,13 +485,13 @@ F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba814 F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1ecc5ba9e64ef90a8b F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8de2d3f9b0b F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99 -F ext/wasm/api/sqlite3-api-glue.js 5391338550ef9fafb3a7a6d060b8c19a190c47c01fa14e032c6e30dd96742c02 -F ext/wasm/api/sqlite3-api-oo1.js 484f9ea5c7140d07745f4b534a1f6dd67120c65ef34abcf7cdb3a388d73f5ef4 -F ext/wasm/api/sqlite3-api-opfs.js 1b097808b7b081b0f0700cf97d49ef19760e401706168edff9cd45cf9169f541 -F ext/wasm/api/sqlite3-api-prologue.js b827e2353799b54fffaa9577f51ebf08b8dedc58dcabe344c73be977235da227 +F ext/wasm/api/sqlite3-api-glue.js b6e7ae7bfe272a8623a8cae4a84830697314c743cadd14e4c167650455f0fff9 +F ext/wasm/api/sqlite3-api-oo1.js 1cf37b2b6e1fb35f9c425380aad198b66a36cdcc65e399e68085509096eca5b0 +F ext/wasm/api/sqlite3-api-opfs.js 3d17da752181d67847bd2e3d1f4662df31c5023a1bed5424ebcdeb89770046af +F ext/wasm/api/sqlite3-api-prologue.js 915e3119d2c3cd53afb9575d1681646f3cbebbfc62cdae8bc199d08e7fd566f1 F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 -F ext/wasm/api/sqlite3-wasm.c 2a0f9e4bf1b141a787918951360601128d6a0a190a31a8e5cfe237c99fa640c6 +F ext/wasm/api/sqlite3-wasm.c 3bb9fe3de3bb536fdae01934760e651c7c20e7066b9415895042555d62ecd7c0 F ext/wasm/batch-runner.html c363032aba7a525920f61f8be112a29459f73f07e46f0ba3b7730081a617826e F ext/wasm/batch-runner.js ce92650a6681586c89bef26ceae96674a55ca5a9727815202ca62e1a00ff5015 F ext/wasm/common/SqliteTestUtil.js 647bf014bd30bdd870a7e9001e251d12fc1c9ec9ce176a1004b838a4b33c5c05 @@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95 F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 -F ext/wasm/sqlite3-opfs-async-proxy.js d571a40230f54b99863f469e050cc4d4317e90239e64026d8d574689956ff76c +F ext/wasm/sqlite3-opfs-async-proxy.js 5c6748361f2eb2ee0f2941890917fd9f2d9271de80ec1b65695fb023435b8375 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 -F ext/wasm/test-opfs-vfs.js 623de926913fa8cb7bac170f105a0c397f1951aba52d9f54792aadce7892a3c2 +F ext/wasm/test-opfs-vfs.js 0115d56f3f2a6475040dffe75a6851a353b1690708795944cf0fe1890bc7ff54 F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893 F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae @@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2703ac9842335962e488e597168d70b1389b95a6ad39edf70a211b95979b4708 -R 0bcb919ac67ff690a464f41cfdc2aecd +P 18462052903bfeb037dd9af3a1fe1898d141eff5d6f527721858120553de808d +R 362ba5dc75c0e8e6f26510dab6893d04 U stephan -Z 78ecb20703c7cf55fa5a9c67c82bb309 +Z 8b3cb31b700bd695aa51a5be567b8bc8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index ccfe9b233b..909d73bfff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18462052903bfeb037dd9af3a1fe1898d141eff5d6f527721858120553de808d \ No newline at end of file +508f7f6d63e52f61fae5abe817579a4e130fa7fbd18733d741d521a5bdabb7ce \ No newline at end of file