diff --git a/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js new file mode 100644 index 0000000000..0832b32bbf --- /dev/null +++ b/ext/wasm/api/sqlite3-vfs-opfs-sahpool.js @@ -0,0 +1,310 @@ +/* + 2023-07-14 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + INCOMPLETE! WORK IN PROGRESS! + + This file holds an experimental sqlite3_vfs backed by OPFS storage + which uses a different implementation strategy than the "opfs" + VFS. This one is a port of Roy Hashimoto's OPFS SyncAccessHandle + pool: + + https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js + + As described at: + + https://github.com/rhashimoto/wa-sqlite/discussions/67 + + with Roy's explicit permission to permit us to port his to our + infrastructure rather than having to clean-room reverse-engineer it: + + https://sqlite.org/forum/forumpost/e140d84e71 + + Primary differences from the original "opfs" VFS include: + + - This one avoids the need for a sub-worker to synchronize + communication between the synchronous C API and the only-partly + synchronous OPFS API. + + - It does so by opening a fixed number of OPFS files at + library-level initialization time, obtaining SyncAccessHandles to + each, and manipulating those handles via the synchronous sqlite3_vfs + interface. + + - Because of that, this one lacks all library-level concurrency + support. + + - Also because of that, it does not require the SharedArrayBuffer, + so can function without the COOP/COEP HTTP response headers. + + - It can hypothetically support Safari 16.4+, whereas the "opfs" + VFS requires v17 due to a bug in 16.x which makes it incompatible + with that VFS. + + - This VFS requires the "semi-fully-sync" FileSystemSyncAccessHandle + (hereafter "SAH") APIs released with Chrome v108. There is + unfortunately no known programmatic way to determine whether a given + API is from that release or newer without actually calling it and + checking whether one of the "fully-sync" functions returns a Promise + (in which case it's the older version). (Reminder to self: when + opening up the initial pool of files, we can close() the first one + we open and see if close() returns a Promise. If it does, it's the + older version so fail VFS initialization. If it doesn't, re-open it.) + +*/ +'use strict'; +globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +const installOpfsVfs = async function(sqlite3){ + const pToss = (...args)=>Promise.reject(new Error(args.join(' '))); + if(!globalThis.FileSystemHandle || + !globalThis.FileSystemDirectoryHandle || + !globalThis.FileSystemFileHandle || + !globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator?.storage?.getDirectory){ + return pToss("Missing required OPFS APIs."); + } + const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const verbosity = 3; + const loggers = [ + sqlite3.config.error, + sqlite3.config.warn, + sqlite3.config.log + ]; + const logImpl = (level,...args)=>{ + if(verbosity>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = sqlite3.util.toss; + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const opfsIoMethods = new capi.sqlite3_io_methods(); + const opfsVfs = new capi.sqlite3_vfs() + .addOnDispose(()=>opfsIoMethods.dispose()); + const promiseReject = (err)=>{ + opfsVfs.dispose(); + return promiseReject_(err); + }; + + // Config opts for the VFS... + const SECTOR_SIZE = 4096; + const HEADER_MAX_PATH_SIZE = 512; + const HEADER_FLAGS_SIZE = 4; + const HEADER_DIGEST_SIZE = 8; + const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE; + const HEADER_OFFSET_FLAGS = HEADER_MAX_PATH_SIZE; + const HEADER_OFFSET_DIGEST = HEADER_CORPUS_SIZE; + const HEADER_OFFSET_DATA = SECTOR_SIZE; + const DEFAULT_CAPACITY = 6; + /* Bitmask of file types which may persist across sessions. + SQLITE_OPEN_xyz types not listed here may be inadvertently + left in OPFS but are treated as transient by this VFS and + they will be cleaned up during VFS init. */ + const PERSISTENT_FILE_TYPES = + capi.SQLITE_OPEN_MAIN_DB | + capi.SQLITE_OPEN_MAIN_JOURNAL | + capi.SQLITE_OPEN_SUPER_JOURNAL | + capi.SQLITE_OPEN_WAL /* noting that WAL support is + unavailable in the WASM build.*/; + const pDVfs = capi.sqlite3_vfs_find(null)/*default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. */; + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = HEADER_MAX_PATH_SIZE; + opfsVfs.$zName = wasm.allocCString("opfs-sahpool"); + opfsVfs.addOnDispose( + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null) + ); + + const VState = Object.assign(Object.create(null),{ + /* OPFS dir in which VFS metadata is stored. */ + vfsDir: ".sqlite3-sahpool", + dirHandle: undefined, + /* Maps OPFS access handles to their opaque file names. */ + mapAH2Name: new Map(), + mapPath2AH: new Map(), + availableAH: new Set(), + mapId2File: new Map(), + getCapacity: function(){return this.mapAH2Name.size}, + getFileCount: function(){return this.mapPath2AH.size}, + addCapacity: async function(n){ + for(let i = 0; i < n; ++i){ + const name = Math.random().toString(36).replace('0.',''); + const h = await this.dirHandle.getFileName(name, {create:true}); + const ah = await h.createSyncAccessHandle(); + this.mapAH2Name(ah,name); + this.setAssociatedPath(ah, '', 0); + } + }, + setAssociatedPath: function(accessHandle, path, flags){ + // TODO + }, + releaseAccessHandles: function(){ + for(const ah of this.mapAH2Name.keys()) ah.close(); + this.mapAH2Name.clear(); + this.mapPath2AH.clear(); + this.availableAH.clear(); + }, + acquireAccessHandles: async function(){ + // TODO + }, + reset: async function(){ + await this.isReady; + let h = await navigator.storage.getDirectory(); + for(const d of this.vfsDir.split('/')){ + if(d){ + h = await h.getDirectoryHandle(d,{create:true}); + } + } + this.dirHandle = h; + this.releaseAccessHandles(); + await this.acquireAccessHandles(); + } + // much more TODO + })/*VState*/; + + // Much, much more TODO... + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + return 0; + }, + xClose: function(pFile){ + let rc = 0; + return rc; + }, + xDeviceCharacteristics: function(pFile){ + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + return capi.SQLITE_NOTFOUND; + }, + xFileSize: function(pFile,pSz64){ + let rc = 0; + return rc; + }, + xLock: function(pFile,lockType){ + let rc = capi.SQLITE_IOERR_LOCK; + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + let rc = capi.SQLITE_IOERR_READ; + return rc; + }, + xSync: function(pFile,flags){ + let rc = capi.SQLITE_IOERR_FSYNC; + return rc; + }, + xTruncate: function(pFile,sz64){ + let rc = capi.SQLITE_IOERR_TRUNCATE; + return rc; + }, + xUnlock: function(pFile,lockType){ + let rc = capi.SQLITE_IOERR_UNLOCK; + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + let rc = capi.SQLITE_IOERR_WRITE; + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + const rc = capi.SQLITE_ERROR; + return rc; + }, + xCurrentTime: function(pVfs,pOut){ + wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + wasm.poke(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + const rc = capi.SQLITE_ERROR; + return rc; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i{ + if(0===VState.getCapacity())[ + await VState.addCapacity(DEFAULT_CAPACITY); + } + promiseResolve(sqlite3); + }).catch(promiseReject); + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; + +globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + return installOpfsVfs(sqlite3).catch((e)=>{ + sqlite3.config.warn("Ignoring inability to install opfs-sahpool sqlite3_vfs:", + e.message); + }); +}/*sqlite3ApiBootstrap.initializersAsync*/); +}/*sqlite3ApiBootstrap.initializers*/); diff --git a/manifest b/manifest index 918e28c08b..28f5f9852d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sinternal\scleanups\sin\sthe\sOPFS\sVFS. -D 2023-07-14T21:06:00.870 +C Initial\ssketches\sfor\san\salternate\sOPFS\sVFS\swhich\suses\sa\spool\sof\spre-opened\sSyncAccessHandles\sto\sbypass\sthe\sneed\sfor\sa\sdedicated\sI/O\sworker\sand\sthe\sCOOP/COEP\sHTTP\sresponse\sheaders.\sCurrently\scompletely\snon-functional. +D 2023-07-14T21:17:29.056 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,6 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386 F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89 F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2 F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487 +F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js d0bc04c29983e967e37a91ca4e849beae6db6c883a6612da9717ca22a24119d1 F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87 F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f @@ -2043,8 +2044,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 816b503f093c4e6d92d0eb2f9fbd841acd01cc9bc89ee58d961b56c64f71406a -R 6597235c929ad6ed0b90d6716177ff1b +P 984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e +R 81377ab74a003e9bec4077c3a186be8e +T *branch * opfs-sahpool +T *sym-opfs-sahpool * +T -sym-trunk * Cancelled\sby\sbranch. U stephan -Z b7b5463364c5290e11704a96855173c7 +Z bc905b63d521aded8c5f64b8ee091932 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9fb0905d5..a6b6be909d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -984d491eb3fe06f714bf07d6873321f3992a072812b46508e599bfefd39dff3e \ No newline at end of file +a93de9f2a553a3a4edd1b361dd6f465a1b0b5b51f7bb8ede432067aedcfefda4 \ No newline at end of file