From 3e2823cbb6e46549229ac177152d048fca8df958 Mon Sep 17 00:00:00 2001 From: stephan Date: Sun, 18 Sep 2022 00:16:12 +0000 Subject: [PATCH] Plug OPFS methods in to their sqlite3_vfs/io_methods counterparts. Add URL args to control debug output and running of sanity-checks in the OPFS init code. FossilOrigin-Name: a0e93ed20b2463606a63b03ce8ca41ec1fb22886db5c5c898ace86ba24636f70 --- ext/wasm/index.html | 2 +- ext/wasm/x-sync-async.html | 2 +- ext/wasm/x-sync-async.js | 104 ++++++++++++++++++++++++++++++------- manifest | 16 +++--- manifest.uuid | 2 +- 5 files changed, 95 insertions(+), 31 deletions(-) diff --git a/ext/wasm/index.html b/ext/wasm/index.html index a1cc194854..562bb9b5ef 100644 --- a/ext/wasm/index.html +++ b/ext/wasm/index.html @@ -50,7 +50,7 @@ worker due to an Emscripten limitation.
  • scratchpad-opfs-worker: experimenting with OPFS from a Worker thread (without WASMFS).
    • -
  • x-sync-async is an +
  • x-sync-async is an experiment in implementing a syncronous sqlite3 VFS proxy for a fully synchronous backend interface (namely OPFS), using SharedArrayBuffer and the Atomics APIs to regulate communication between the synchronous diff --git a/ext/wasm/x-sync-async.html b/ext/wasm/x-sync-async.html index ec0a6353b5..b83d93378c 100644 --- a/ext/wasm/x-sync-async.html +++ b/ext/wasm/x-sync-async.html @@ -17,6 +17,6 @@
    - + diff --git a/ext/wasm/x-sync-async.js b/ext/wasm/x-sync-async.js index d36dc1700f..203c8eece4 100644 --- a/ext/wasm/x-sync-async.js +++ b/ext/wasm/x-sync-async.js @@ -44,7 +44,6 @@ const initOpfsVfs = function(sqlite3){ const error = (...args)=>{ console.error(logPrefix,...args); }; - warn("This file is very much experimental and under construction.",self.location.pathname); if(self.window===self || !self.SharedArrayBuffer || @@ -56,7 +55,7 @@ const initOpfsVfs = function(sqlite3){ warn("This environment does not have OPFS support."); return; } - + warn("This file is very much experimental and under construction.",self.location.pathname); const capi = sqlite3.capi; const wasm = capi.wasm; const sqlite3_vfs = capi.sqlite3_vfs @@ -65,6 +64,8 @@ const initOpfsVfs = function(sqlite3){ || toss("Missing sqlite3.capi.sqlite3_file object."); const sqlite3_io_methods = capi.sqlite3_io_methods || toss("Missing sqlite3.capi.sqlite3_io_methods object."); + const StructBinder = sqlite3.StructBinder || toss("Missing sqlite3.StructBinder."); + const thisUrl = new URL(self.location.href); const W = new Worker("sqlite3-opfs-async-proxy.js"); const wMsg = (type,payload)=>W.postMessage({type,payload}); @@ -76,7 +77,7 @@ const initOpfsVfs = function(sqlite3){ of data may be added to it. */ const state = Object.create(null); - state.verbose = 2; + state.verbose = thisUrl.searchParams.has('opfs-verbose') ? 3 : 2; state.fileBufferSize = 1024 * 64 + 8 /* size of fileHandle.sab. 64k = max sqlite3 page size */; state.fbInt64Offset = state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64*/; state.opIds = Object.create(null); @@ -110,7 +111,6 @@ const initOpfsVfs = function(sqlite3){ }); const isWorkerErrCode = (n)=>!!state.sq3Codes._reverse[n]; - const opStore = (op,val=-1)=>Atomics.store(state.opSABView, state.opIds[op], val); const opWait = (op,val=-1)=>Atomics.wait(state.opSABView, state.opIds[op], val); @@ -128,12 +128,6 @@ const initOpfsVfs = function(sqlite3){ return Atomics.load(state.opSABView, state.opIds[op]); }; - const wait = (ms,value)=>{ - return new Promise((resolve)=>{ - setTimeout(()=>resolve(value), ms); - }); - }; - /** Generates a random ASCII string len characters long, intended for use as a temporary file name. @@ -194,6 +188,75 @@ const initOpfsVfs = function(sqlite3){ environment or the other when sqlite3_os_end() is called (_if_ it gets called at all in a wasm build, which is undefined). */ + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + //log("installMethod",tgt, name, sigN); + const fProxy = 1 + // We can remove this proxy middle-man once the VFS is working + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; /** Impls for the sqlite3_io_methods methods. Maintenance reminder: @@ -390,10 +453,11 @@ const initOpfsVfs = function(sqlite3){ vfsSyncWrappers.xSleep = (pVfs,ms)=>opRun('xSleep',ms); } - /* - TODO: plug in the above functions in to opfsVfs and opfsIoMethods. - Code for doing so is in api/sqlite3-api-opfs.js. - */ + /* Install the vfs/io_methods into their C-level shared instances... */ + let inst = installMethod(opfsIoMethods); + for(let k of Object.keys(ioSyncWrappers)) inst(k, ioSyncWrappers[k]); + inst = installMethod(opfsVfs); + for(let k of Object.keys(vfsSyncWrappers)) inst(k, vfsSyncWrappers[k]); const sanityCheck = async function(){ //state.ioBuf = new Uint8Array(state.sabIo); @@ -430,10 +494,10 @@ const initOpfsVfs = function(sqlite3){ rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); if(rc) toss('xFileSize failed w/ rc',rc); log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); - rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 0); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); if(rc) toss("xWrite() failed!"); const readBuf = wasm.scopedAlloc(16); - rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 1); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); wasm.setMemValue(readBuf+6,0); let jRead = wasm.cstringToJs(readBuf); log("xRead() got:",jRead); @@ -453,9 +517,8 @@ const initOpfsVfs = function(sqlite3){ } }; - W.onmessage = function({data}){ - log("Worker.onmessage:",data); + //log("Worker.onmessage:",data); switch(data.type){ case 'loaded': /*Pass our config and shared state on to the async worker.*/ @@ -476,8 +539,9 @@ const initOpfsVfs = function(sqlite3){ } capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); state.opSABView = new Int32Array(state.opSAB); - if(self.location && +self.location.port > 1024){ - log("Running sanity check for dev mode..."); + + if(thisUrl.searchParams.has('opfs-sanity-check')){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); sanityCheck(); } warn("End of (very incomplete) OPFS setup.", opfsVfs); diff --git a/manifest b/manifest index b288f12d6c..561605cbab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\sOPFS\sxRead()\sand\sxWrite()\simpls\sand\sadd\sa\svery\sbasic\ssanity\stest\sfor\sthem. -D 2022-09-17T23:47:20.619 +C Plug\sOPFS\smethods\sin\sto\stheir\ssqlite3_vfs/io_methods\scounterparts.\sAdd\sURL\sargs\sto\scontrol\sdebug\soutput\sand\srunning\sof\ssanity-checks\sin\sthe\sOPFS\sinit\scode. +D 2022-09-18T00:16:12.445 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -502,7 +502,7 @@ F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d695 F ext/wasm/fiddle/fiddle-worker.js bccf46045be8824752876f3eec01c223be0616ccac184bffd0024cfe7a3262b8 F ext/wasm/fiddle/fiddle.html 550c5aafce40bd218de9bf26192749f69f9b10bc379423ecd2e162bcef885c08 F ext/wasm/fiddle/fiddle.js 4ffcfc9a235beebaddec689a549e9e0dfad6dca5c1f0b41f03468d7e76480686 -F ext/wasm/index.html 8365e47e2aff1829923f0481292948487b27a0755fde9c0d3ad15f7ea0118992 +F ext/wasm/index.html b8a47afa96d0c7ac3081c6d469e1fd7be6b87c64b94e4fb62e26984a1564cac4 F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106 F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f @@ -533,8 +533,8 @@ F ext/wasm/testing1.js 7cd8ab255c238b030d928755ae8e91e7d90a12f2ae601b1b8f7827aaa F ext/wasm/testing2.html a66951c38137ff1d687df79466351f3c734fa9c6d9cce71d3cf97c291b2167e3 F ext/wasm/testing2.js 25584bcc30f19673ce13a6f301f89f8820a59dfe044e0c4f2913941f4097fe3c F ext/wasm/wasmfs.make 21a5cf297954a689e0dc2a95299ae158f681cae5e90c10b99d986097815fd42d -F ext/wasm/x-sync-async.html 717b0d3bee96e49cbd36731bead497ab27a8bf3a3b23dd11e40e61d4ac9e8b80 -F ext/wasm/x-sync-async.js 961eb6c268086babd20d7b5e07cad72dc3118f9c46954e9a73b8b826deb4f771 +F ext/wasm/x-sync-async.html d85cb9b1ab398ef5a20ce64a689ce4bf9a347ffe69edd46c2d3dc57b869a8925 +F ext/wasm/x-sync-async.js 2cd04d73ddc515479cc2e4b9246d6da21b3f494020261d47470f4b710c84c0da F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -2030,8 +2030,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c342b5d745f301104c59851c753287ebbbe95a69a56cb522d376d0f3e352c30f -R 70b6b0622f3dd6730d580a81783f068d +P cd06cc670029763955cf60ffcf944b36d41cb005b859d9b9fd0eea1b6741d0e9 +R af48df928e44c11df9f0c9eb2cf8376e U stephan -Z 5c4a37b98dc31ffc6cd63d317f5738d7 +Z 07b2e0eb2359d701cc905c5402ed9c24 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 39390f941c..274fb7f0c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cd06cc670029763955cf60ffcf944b36d41cb005b859d9b9fd0eea1b6741d0e9 \ No newline at end of file +a0e93ed20b2463606a63b03ce8ca41ec1fb22886db5c5c898ace86ba24636f70 \ No newline at end of file