From da6a42a92143723e0c6c4554e7fec19c9a15cb03 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 16 Jul 2023 16:52:09 +0000 Subject: [PATCH] Move SAH pool configuration options from the library-level config to a config passed to the VFS install routine. Extend and document the PoolUtil object. FossilOrigin-Name: d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 --- ext/wasm/api/sqlite3-api-prologue.js | 18 +- ext/wasm/api/sqlite3-vfs-opfs-sahpool.js | 207 ++++++++++++++++++----- ext/wasm/speedtest1-worker.js | 9 +- manifest | 16 +- manifest.uuid | 2 +- 5 files changed, 184 insertions(+), 68 deletions(-) diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js index fb085e299c..ac3253670f 100644 --- a/ext/wasm/api/sqlite3-api-prologue.js +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -91,21 +91,6 @@ - `wasmfsOpfsDir`[^1]: Specifies the "mount point" of the OPFS-backed filesystem in WASMFS-capable builds. - - `opfs-sahpool.dir`[^1]: Specifies the OPFS directory name in - which to store metadata for the `"opfs-sahpool"` sqlite3_vfs. - Changing this name will effectively orphan any databases stored - under previous names. The default is unspecified but descriptive. - This option may contain multiple path elements, - e.g. "foo/bar/baz", and they are created automatically. In - practice there should be no driving need to change this. - - - `opfs-sahpool.defaultCapacity`[^1]: Specifies the default - capacity of the `"opfs-sahpool"` VFS. This should not be set - unduly high because the VFS has to open (and keep open) a file - for each entry in the pool. This setting only has an effect when - the pool is initially empty. It does not have any effect if a - pool already exists. - [^1] = This property may optionally be a function, in which case this function calls that function to fetch the value, @@ -158,8 +143,7 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( [ // If any of these config options are functions, replace them with // the result of calling that function... - 'exports', 'memory', 'wasmfsOpfsDir', - 'opfs-sahpool.dir', 'opfs-sahpool.defaultCapacity' + 'exports', 'memory', 'wasmfsOpfsDir' ].forEach((k)=>{ if('function' === typeof config[k]){ config[k] = config[k](); diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js index d40581aba3..a19589aac9 100644 --- a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -55,11 +55,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ const toss = sqlite3.util.toss; let vfsRegisterResult = undefined; +/** The PoolUtil object will be the result of the + resolved Promise. */ +const PoolUtil = Object.create(null); +let isPromiseReady; + /** installOpfsSAHPoolVfs() asynchronously initializes the OPFS - SyncAccessHandle Pool VFS. It returns a Promise which either - resolves to a utility object described below or rejects with an - Error value. + SyncAccessHandle (a.k.a. SAH) Pool VFS. It returns a Promise which + either resolves to a utility object described below or rejects with + an Error value. Initialization of this VFS is not automatic because its registration requires that it lock all resources it @@ -72,18 +77,113 @@ let vfsRegisterResult = undefined; due to OPFS locking errors. On calls after the first this function immediately returns a - resolved or rejected Promise. If called while the first call is - still pending resolution, a rejected promise with a descriptive - error is returned. + pending, resolved, or rejected Promise, depending on the state + of the first call's Promise. On success, the resulting Promise resolves to a utility object - which can be used to query and manipulate the pool. Its API is... + which can be used to query and manipulate the pool. Its API is + described at the end of these docs. - TODO + This function accepts an options object to configure certain + parts but it is only acknowledged for the very first call and + ignored for all subsequent calls. + + The options, in alphabetical order: + + - `clearOnInit`: if truthy, as each SAH is acquired during + initalization of the VFS, its contents and filename name mapping + are removed, leaving the VFS's storage in a pristine state. + + - `defaultCapacity`: Specifies the default capacity of the + VFS. This should not be set unduly high because the VFS has to + open (and keep open) a file for each entry in the pool. This + setting only has an effect when the pool is initially empty. It + does not have any effect if a pool already exists. + + - `directory`: Specifies the OPFS directory name in which to store + metadata for the `"opfs-sahpool"` sqlite3_vfs. Only 1 instance + of this VFS can be installed per JavaScript engine, and any two + engines with the same storage directory name will collide with + each other, leading to locking errors and the inability to + register the VFS in the second and subsequent engine. Using a + different directory name for each application enables different + engines in the same HTTP origin to co-exist, but their data are + invisible to each other. Changing this name will effectively + orphan any databases stored under previous names. The default is + unspecified but descriptive. This option may contain multiple + path elements, e.g. "foo/bar/baz", and they are created + automatically. In practice there should be no driving need to + change this. + + + API for the utility object passed on by this function's Promise, in + alphabetical order... + +- [async] addCapacity(n) + + Adds `n` entries to the current pool. This change is persistent + across sessions so should not be called automatically at each app + startup (but see `reserveMinimumCapacity()`). Its returned Promise + resolves to the new capacity. Because this operation is necessarily + asynchronous, the C-level VFS API cannot call this on its own as + needed. + +- byteArray exportFile(name) + + Synchronously reads the contents of the given file into a Uint8Array + and returns it. This will throw if the given name is not currently + in active use or on I/O error. + +- number getCapacity() + + Returns the number of files currently contained + in the SAH pool. The default capacity is only large enough for one + or two databases and their associated temp files. + +- number getActiveFileCount() + + Returns the number of files from the pool currently in use. + +- importDb(name, byteArray) + + Imports the contents of an SQLite database, provided as a byte + array, under the given name, overwriting any existing + content. Throws if the pool has no available file slots, on I/O + error, or if the input does not appear to be a database. In the + latter case, only a cursory examination is made. Note that this + routine is _only_ for importing database files, not arbitrary files, + the reason being that this VFS will automatically clean up any + non-database files so importing them is pointless. + +- [async] number reduceCapacity(n) + + Removes up to `n` entries from the pool, with the caveat that it can + only remove currently-unused entries. It returns a Promise which + resolves to the number of entries actually removed. + +- [async] number reserveMinimumCapacity(min) + + If the current capacity is less than `min`, the capacity is + increased to `min`, else this returns with no side effects. The + resulting Promise resolves to the new capacity. + +- boolean unlink(filename) + + If a virtual file exists with the given name, disassociates it from + the pool and returns true, else returns false without side + effects. Results are undefined if the file is currently in active + use. + +- [async] wipeFiles() + + Clears all client-defined state of all SAHs and makes all of them + available for re-use by the pool. Results are undefined if any such + handles are currently in use, e.g. by an sqlite3 db. */ -sqlite3.installOpfsSAHPoolVfs = async function(){ - if(sqlite3===vfsRegisterResult) return Promise.resolve(sqlite3); +sqlite3.installOpfsSAHPoolVfs = async function(options=Object.create(null)){ + if(PoolUtil===vfsRegisterResult) return Promise.resolve(PoolUtil); + else if(isPromiseReady) return isPromiseReady; else if(undefined!==vfsRegisterResult){ return Promise.reject(vfsRegisterResult); } @@ -94,7 +194,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ !navigator?.storage?.getDirectory){ return Promise.reject(vfsRegisterResult = new Error("Missing required OPFS APIs.")); } - vfsRegisterResult = new Error("VFS initialization still underway."); + vfsRegisterResult = new Error("opfs-sahpool initialization still underway."); const verbosity = 2 /*3+ == everything*/; const loggers = [ sqlite3.config.error, @@ -118,9 +218,6 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ vfsRegisterResult = err; return Promise.reject(err); }; - /** The PoolUtil object will be the result of the - resolved Promise. */ - const PoolUtil = Object.create(null); const promiseResolve = ()=>Promise.resolve(vfsRegisterResult = PoolUtil); // Config opts for the VFS... @@ -133,7 +230,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; const HEADER_OFFSET_DATA = SECTOR_SIZE; const DEFAULT_CAPACITY = - sqlite3.config['opfs-sahpool.defaultCapacity'] || 6; + options.defaultCapacity || 6; /* Bitmask of file types which may persist across sessions. SQLITE_OPEN_xyz types not listed here may be inadvertently left in OPFS but are treated as transient by this VFS and @@ -171,14 +268,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ */ const SAHPool = Object.assign(Object.create(null),{ /* OPFS dir in which VFS metadata is stored. */ - vfsDir: sqlite3.config['opfs-sahpool.dir'] - || ".sqlite3-opfs-sahpool", + vfsDir: options.directory || ".sqlite3-opfs-sahpool", /* Directory handle to this.vfsDir. */ dirHandle: undefined, /* Maps SAHs to their opaque file names. */ mapSAHToName: new Map(), /* Maps client-side file names to SAHs. */ - mapPathToSAH: new Map(), + mapFilenameToSAH: new Map(), /* Set of currently-unused SAHs. */ availableSAH: new Set(), /* Maps (sqlite3_file*) to xOpen's file objects. */ @@ -186,7 +282,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ /* Current pool capacity. */ getCapacity: function(){return this.mapSAHToName.size}, /* Current number of in-use files from pool. */ - getFileCount: function(){return this.mapPathToSAH.size}, + getFileCount: function(){return this.mapFilenameToSAH.size}, /** Adds n files to the pool's capacity. This change is persistent across settings. Returns a Promise which resolves @@ -229,7 +325,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ releaseAccessHandles: function(){ for(const ah of this.mapSAHToName.keys()) ah.close(); this.mapSAHToName.clear(); - this.mapPathToSAH.clear(); + this.mapFilenameToSAH.clear(); this.availableSAH.clear(); }, /** @@ -238,8 +334,13 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ but completes once all SAHs are acquired. If acquiring an SAH throws, SAHPool.$error will contain the corresponding exception. + + + If clearFiles is true, the client-stored state of each file is + cleared when its handle is acquired, including its name, flags, + and any data stored after the metadata block. */ - acquireAccessHandles: async function(){ + acquireAccessHandles: async function(clearFiles){ const files = []; for await (const [name,h] of this.dirHandle){ if('file'===h.kind){ @@ -250,11 +351,16 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ try{ const ah = await h.createSyncAccessHandle() this.mapSAHToName.set(ah, name); - const path = this.getAssociatedPath(ah); - if(path){ - this.mapPathToSAH.set(path, ah); + if(clearFiles){ + ah.truncate(HEADER_OFFSET_DATA); + this.setAssociatedPath(ah, '', 0); }else{ - this.availableSAH.add(ah); + const path = this.getAssociatedPath(ah); + if(path){ + this.mapFilenameToSAH.set(path, ah); + }else{ + this.availableSAH.add(ah); + } } }catch(e){ SAHPool.storeErr(e); @@ -327,12 +433,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ sah.flush(); if(path){ - this.mapPathToSAH.set(path, sah); + this.mapFilenameToSAH.set(path, sah); this.availableSAH.delete(sah); }else{ // This is not a persistent file, so eliminate the contents. sah.truncate(HEADER_OFFSET_DATA); - this.mapPathToSAH.delete(path); + this.mapFilenameToSAH.delete(path); this.availableSAH.add(sah); } }, @@ -352,9 +458,12 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ /** Re-initializes the state of the SAH pool, releasing and re-acquiring all handles. + + See acquireAccessHandles() for the specifics of the clearFiles + argument. */ - reset: async function(){ - await this.isReady; + reset: async function(clearFiles){ + await isPromiseReady; let h = await navigator.storage.getDirectory(); for(const d of this.vfsDir.split('/')){ if(d){ @@ -363,7 +472,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ } this.dirHandle = h; this.releaseAccessHandles(); - await this.acquireAccessHandles(); + await this.acquireAccessHandles(clearFiles); }, /** Returns the pathname part of the given argument, @@ -381,14 +490,17 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ }, /** Removes the association of the given client-specified file - name (JS string) from the pool. + name (JS string) from the pool. Returns true if a mapping + is found, else false. */ deletePath: function(path) { - const sah = this.mapPathToSAH.get(path); + const sah = this.mapFilenameToSAH.get(path); if(sah) { - // Un-associate the SQLite path from the OPFS file. + // Un-associate the name from the SAH. + this.mapFilenameToSAH.delete(path); this.setAssociatedPath(sah, '', 0); } + return !!sah; }, /** Sets e as this object's current error. Pass a falsy @@ -549,7 +661,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ SAHPool.storeErr(); try{ const name = this.getPath(zName); - wasm.poke32(pOut, SAHPool.mapPathToSAH.has(name) ? 1 : 0); + wasm.poke32(pOut, SAHPool.mapFilenameToSAH.has(name) ? 1 : 0); }catch(e){ /*ignored*/; } @@ -606,7 +718,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ const path = (zName && wasm.peek8(zName)) ? SAHPool.getPath(zName) : getRandomName(); - let sah = SAHPool.mapPathToSAH.get(path); + let sah = SAHPool.mapFilenameToSAH.get(path); if(!sah && (flags & capi.SQLITE_OPEN_CREATE)) { // File not found so try to create it. if(SAHPool.getFileCount() < SAHPool.getCapacity()) { @@ -701,7 +813,7 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ not currently in active use or on I/O error. */ PoolUtil.exportFile = function(name){ - const sah = SAHPool.mapPathToSAH.get(name) || toss("File not found:",name); + const sah = SAHPool.mapFilenameToSAH.get(name) || toss("File not found:",name); const n = sah.getSize() - HEADER_OFFSET_DATA; const b = new Uint8Array(n>=0 ? n : 0); if(n>0) sah.read(b, {at: HEADER_OFFSET_DATA}); @@ -734,14 +846,33 @@ sqlite3.installOpfsSAHPoolVfs = async function(){ toss("Input does not contain an SQLite database header."); } } - const sah = SAHPool.mapPathToSAH.get(name) + const sah = SAHPool.mapFilenameToSAH.get(name) || SAHPool.nextAvailableSAH() || toss("No available handles to import to."); sah.write(bytes, {at: HEADER_OFFSET_DATA}); SAHPool.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB); }; + /** + Clears all client-defined state of all SAHs and makes all of them + available for re-use by the pool. Results are undefined if any + such handles are currently in use, e.g. by an sqlite3 db. + */ + PoolUtil.wipeFiles = async ()=>SAHPool.reset(true); - return SAHPool.isReady = SAHPool.reset().then(async ()=>{ + /** + If a virtual file exists with the given name, disassociates it + from the pool and returns true, else returns false without side + effects. + */ + PoolUtil.unlink = (filename)=>SAHPool.deletePath(filename); + + /** + PoolUtil TODOs: + + - function to wipe out all traces of the VFS from storage. + */ + + return isPromiseReady = SAHPool.reset(!!options.clearOnInit).then(async ()=>{ if(SAHPool.$error){ throw SAHPool.$error; } diff --git a/ext/wasm/speedtest1-worker.js b/ext/wasm/speedtest1-worker.js index c2bf37b831..61af26b23e 100644 --- a/ext/wasm/speedtest1-worker.js +++ b/ext/wasm/speedtest1-worker.js @@ -61,7 +61,11 @@ && !App.sqlite3.$SAHPoolUtil && cliFlagsArray.indexOf('opfs-sahpool')>=0){ log("Installing opfs-sahpool..."); - await App.sqlite3.installOpfsSAHPoolVfs().then(PoolUtil=>{ + await App.sqlite3.installOpfsSAHPoolVfs({ + directory: '.speedtest1-sahpool', + defaultCapacity: 3, + clearOnInit: true + }).then(PoolUtil=>{ log("opfs-sahpool successfully installed."); App.sqlite3.$SAHPoolUtil = PoolUtil; }); @@ -122,9 +126,6 @@ //else log("Using transient storage."); mPost('ready',true); log("Registered VFSes:", ...S.capi.sqlite3_js_vfs_list()); - if(0 && S.installOpfsSAHPoolVfs){ - sahpSanityChecks(S); - } }).catch(e=>{ logErr(e); }); diff --git a/manifest b/manifest index d2e88ef2ac..1e2c3ff671 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C speedtest1.js:\sonly\sinstall\sopfs-sahpool\sif\sit's\sprovided\svia\s--vfs\sflag,\sto\savoid\slocking\serrors\sin\sconcurrent\sspeedtest1\stabs\swith\sother\sVFSes.\sAdd\sopfs-sahpool\sreserveMinimumCapacity(). -D 2023-07-16T14:07:59.930 +C Move\sSAH\spool\sconfiguration\soptions\sfrom\sthe\slibrary-level\sconfig\sto\sa\sconfig\spassed\sto\sthe\sVFS\sinstall\sroutine.\sExtend\sand\sdocument\sthe\sPoolUtil\sobject. +D 2023-07-16T16:52:09.106 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -497,12 +497,12 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af F ext/wasm/api/sqlite3-api-cleanup.js 23ceec5ef74a0e649b19694ca985fd89e335771e21f24f50df352a626a8c81bf F ext/wasm/api/sqlite3-api-glue.js f1b2dcb944de5138bb5bd9a1559d2e76a4f3ec25260963d709e8237476688803 F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8 -F ext/wasm/api/sqlite3-api-prologue.js 5dcb5d2d74269545073eec197614b86bd28950132b5fe4de67c10a8a0d5524b2 +F ext/wasm/api/sqlite3-api-prologue.js f68e87edc049793c4ed46b0ec8f3a3d8013eeb3fd56481029dda916d4d5fa3a3 F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 -F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 5ffed44d7bac1b4038e1505ffc7ab63e82726a97a64193ddbd5b414722f0808b +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js c19ccfc2995c0dcae00f13fe1be6fa436a39a3d629b6bf4208965ea78a50cab3 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 842d55b35a871ee5483cc5e0cf067a968362b4d61321f08c71aab5505c72f556 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -540,7 +540,7 @@ F ext/wasm/scratchpad-wasmfs.mjs 66034b9256b218de59248aad796760a1584c1dd84223150 F ext/wasm/speedtest1-wasmfs.html 0e9d335a9b5b5fafe6e1bc8dc0f0ca7e22e6eb916682a2d7c36218bb7d67379d F ext/wasm/speedtest1-wasmfs.mjs ac5cadbf4ffe69e9eaac8b45e8523f030521e02bb67d654c6eb5236d9c456cbe F ext/wasm/speedtest1-worker.html e33e2064bda572c0c3ebaec7306c35aa758d9d27e245d67e807f8cc4a9351cc5 -F ext/wasm/speedtest1-worker.js cda2f6cf0a6b864d82e51b9e4dfd1dfb0c4024987c5d94a81cc587e07acc9be4 +F ext/wasm/speedtest1-worker.js 41fdc91878d3481b198bba771f073aad8837063ea2a23a0e9a278a54634f8ffe F ext/wasm/speedtest1.html ff048b4a623aa192e83e143e48f1ce2a899846dd42c023fdedc8772b6e3f07da F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0 @@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29905b7a75b73e32125bf9116033cae7235a135b668a3b783a3d8dcb0bc80374 -R 04e7987eb127f55eddff193be36455e6 +P aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d +R 76883f55e00ff0c4af4f43f15f164d03 U stephan -Z 53c8cb4a4900e0bba3854b3011d70f79 +Z fd9b47fd6c1916432a0f6dc613a90b88 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c9034b92fd..9dc2139fb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aa94c8abfbdfc4c7b36554c4b3ea90a5065e7e3f4294c64c8cbf688b4688300d \ No newline at end of file +d2ed99556fa1f40994c1c6bd90d1d5733bebc824b1ebfabe978fae9e18948437 \ No newline at end of file