Get fiddle db export working for OPFS VFS. Add root dir handle to the main OPFS VFS worker to enable creation of certain utility functions without delegating to the async worker. Add sqlite3.capi.sqlite3_wasm_rc_str() to map integer result codes back to their SQLITE_xxx counterparts. Minor doc touchups.

FossilOrigin-Name: 9b2244e1c8a40efe6547094a1b57acc8f2173145a8731abb0e36268ce0a8ef41
This commit is contained in:
stephan 2022-09-26 11:38:58 +00:00
parent 6a643e4bcd
commit 1f095d482d
9 changed files with 197 additions and 67 deletions

View File

@ -5,4 +5,5 @@ _fiddle_experiment
_fiddle_interrupt
_fiddle_main
_fiddle_reset_db
_fiddle_the_db
_fiddle_db_handle
_fiddle_db_is_opfs

View File

@ -185,9 +185,20 @@ self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
capi[e[0]] = e[1];
}
}
const __rcMap = Object.create(null);
for(const t of ['resultCodes']){
for(const e of Object.entries(wasm.ctype[t])){
__rcMap[e[1]] = e[0];
}
}
/**
For the given integer, returns the SQLITE_xxx result code as a
string, or undefined if no such mapping is found.
*/
capi.sqlite3_wasm_rc_str = (rc)=>__rcMap[rc];
/* Bind all registered C-side structs... */
for(const s of wasm.ctype.structs){
capi[s.name] = sqlite3.StructBinder(s);
}
}
}/*end C constant imports*/
});

View File

@ -999,11 +999,14 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck();
}
navigator.storage.getDirectory().then((d)=>{
W.onerror = W._originalOnError;
delete W._originalOnError;
sqlite3.opfs = opfsUtil;
opfsUtil.rootDirectory = d;
log("End of OPFS sqlite3_vfs setup.", opfsVfs);
promiseResolve(sqlite3);
});
}catch(e){
error(e);
promiseReject(e);

View File

@ -99,7 +99,9 @@
};
const stdout = (...args)=>wMsg('stdout', args);
const stderr = (...args)=>wMsg('stderr', args);
const toss = (...args)=>{
throw new Error(args.join(' '));
};
const fixmeOPFS = "(FIXME: won't work with vanilla OPFS.)";
self.onerror = function(/*message, source, lineno, colno, error*/) {
@ -136,7 +138,7 @@
const S = fiddleModule.sqlite3;
/* We need to call sqlite3_shutdown() in order to avoid numerous
legitimate warnings from the shell about it being initialized
after sqlite3_initialize() has been called. This mean ,
after sqlite3_initialize() has been called. This means,
however, that any initialization done by the JS code may need
to be re-done (e.g. re-registration of dynamically-loaded
VFSes). */
@ -156,7 +158,7 @@
if(S.opfs){
stdout("\nOPFS is available. To open a persistent db, use:\n\n",
" .open file:name?vfs=opfs\n\nbut note that some",
"features (e.g. export) do not yet work with OPFS.");
"features (e.g. upload) do not yet work with OPFS.");
S.opfs.reregisterVfs();
}
stdout('\nEnter ".help" for usage hints.');
@ -208,6 +210,69 @@
}
};
/**
Exports the shell's current db file in such a way that it can
export DBs hosted in the Emscripten-supplied FS or in native OPFS
(and, hypothetically, kvvfs). Throws on error. On success returns
a Blob containing the whole db contents.
Bug: xFileSize() is returning garbage for the default VFS but
works in OPFS. Thus for exporting that impl we'll use the
fiddleModule.FS API for the time being.
*/
const exportDbFileToBlob = function(){
const S = fiddleModule.sqlite3, capi = S.capi, wasm = capi.wasm;
const pDb = wasm.xCall("fiddle_db_handle");
if(!pDb) toss("No db is opened.");
const scope = wasm.scopedAllocPush();
try{
const ppFile = wasm.scopedAlloc(12/*sizeof(i32 + i64)*/);
const pFileSize = ppFile + 4;
wasm.setMemValue(ppFile, 0, 'i32');
let rc = capi.sqlite3_file_control(
pDb, "main", capi.SQLITE_FCNTL_FILE_POINTER, ppFile
);
if(rc) toss("Cannot get sqlite3_file handle.");
const jFile = new capi.sqlite3_file(wasm.getPtrValue(ppFile));
const jIom = new capi.sqlite3_io_methods(jFile.$pMethods);
console.warn("jIom =",jIom);
const xFileSize = wasm.functionEntry(jIom.$xFileSize);
const xRead = wasm.functionEntry(jIom.$xRead);
wasm.setMemValue(pFileSize, 0n, 'i64');
//stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize);
rc = xFileSize( jFile.pointer, pFileSize );
if(rc) toss("Cannot get db file size.");
//stderr("nFileSize =",wasm.getMemValue(pFileSize,'i64'),"pFileSize =",pFileSize);
const nFileSize = Number( wasm.getMemValue(pFileSize,'i64') );
if(nFileSize <= 0n || nFileSize>=Number.MAX_SAFE_INTEGER){
toss("Unexpected DB size:",nFileSize);
}
//stderr("nFileSize =",nFileSize,"pFileSize =",pFileSize);
const nIobuf = 1024 * 4;
const iobuf = wasm.scopedAlloc(nIobuf);
let nPos = 0;
const blobList = [];
const heap = wasm.heap8u();
for( ; nPos < nFileSize; nPos += nIobuf ){
rc = xRead(jFile.pointer, iobuf, nIobuf, BigInt(nPos));
if(rc){
if(capi.SQLITE_IOERR_SHORT_READ === rc){
//stderr('rc =',rc,'nPos =',nPos,'nIobuf =',nIobuf,'nFileSize =',nFileSize);
rc = ((nPos + nIobuf) < nFileSize) ? rc : 0/*assume EOF*/;
}
if(rc) toss("xRead() SQLITE_xxx error #"+rc,capi.sqlite3_wasm_rc_str(rc));
}
blobList.push(heap.slice(iobuf, iobuf+nIobuf));
}
return new Blob(blobList);
}catch(e){
console.error('exportDbFileToBlob()',e);
stderr("exportDbFileToBlob():",e.message);
}finally{
wasm.scopedAllocPop(scope);
}
}/*exportDbFileToBlob()*/;
self.onmessage = function f(ev){
ev = ev.data;
if(!f.cache){
@ -233,14 +298,22 @@
*/
case 'db-export': {
const fn = Sqlite3Shell.dbFilename();
stdout("Exporting",fn+".",fixmeOPFS);
stdout("Exporting",fn+".");
const fn2 = fn ? fn.split(/[/\\]/).pop() : null;
try{
if(!fn2) throw new Error("DB appears to be closed.");
if(fiddleModule.sqlite3.capi.wasm.xCall(
'fiddle_db_is_opfs'
)){
exportDbFileToBlob().arrayBuffer().then((buffer)=>{
wMsg('db-export',{filename: fn2, buffer}, [buffer]);
});
}else{
const buffer = fiddleModule.FS.readFile(
fn, {encoding:"binary"}
).buffer;
wMsg('db-export',{filename: fn2, buffer}, [buffer]);
}
}catch(e){
/* Post a failure message so that UI elements disabled
during the export can be re-enabled. */

View File

@ -15,40 +15,49 @@
"make"d first. The intent is that <em>this</em> page be run
using:</div>
<blockquote><pre>althttpd -page index.html</pre></blockquote>
<div>and the individual tests be started in their own tab.</div>
<div>Warnings and Caveats:
<div>and the individual tests be started in their own tab.
Warnings and Caveats:
<ul class='warning'>
<li>Some of these pages require that
the web server emit the so-called COOP and COEP headers. The
default build of althttpd <em>does not</em>.
the web server emit the so-called
<a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Opener-Policy'>COOP</a>
and
<a href='https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cross-Origin-Embedder-Policy'>COEP</a>
headers. The default build
of <a href='https://sqlite.org/althttpd'>althttpd</a>
<em>does not</em>.
</li>
<li>Any OPFS-related pages require very recent version of
Chrome or Chromium (v102 at least, possibly newer). OPFS
support in the other major browsers is pending. Development
and testing is currently done against a dev-channel release
of Chrome (v107 as of 2022-09-26).
</li>
<li>Any OPFS-related pages require very recent
version of Chrome or Chromium (v102 at least, possibly
newer). OPFS support in the other major browsers is
pending.</li>
<li>Whether or not WASMFS/OPFS support is enabled on any given
page may depend on build-time options which are <em>off by
default</em> because they currently (as of 2022-09-08) break
with Worker-based pages. Similarly, WASMFS does not work on
some platforms, e.g. Raspberry Pi 4.
with Worker-based pages.
<!-- Similarly, WASMFS does not work on some platforms,
e.g. Raspberry Pi 4 (for which there is no Chrome
build, so it's currently a moot point). -->
</li>
</ul>
</div>
<div>The tests...
<div>The tests and demos...
<ul id='test-list'>
<li>High-level apps and demos...
<ul>
<li><a href='fiddle/fiddle.html'>fiddle</a> is an HTML front-end
to a wasm build of the sqlite3 shell.</li>
<li><a href='demo-123.html'>demo-123</a> provides a
no-nonsense example of adding sqlite3 support to a
web page. </li>
<li><a href='demo-123-worker.html'>demo-123-worker</a> is the
same as <code>demo-123</code> but loads and run sqlite3 from
a Worker thread.</li>
no-nonsense example of adding sqlite3 support to a web
page in the UI thread.</li>
<li><a href='demo-123-worker.html'>demo-123-worker</a> is
the same as <code>demo-123</code> but loads and runs
sqlite3 from a Worker thread.</li>
<li><a href='demo-kvvfs1.html'>demo-kvvfs1</a>: very basic
demo of using the key-value vfs for storing a persistent db
in JS localStorage or sessionStorage.</li>
demo of using the key-value VFS for storing a persistent db
in JS <code>localStorage</code> or <code>sessionStorage</code>.</li>
</ul>
</li>
<li>speedtest1 ports (sqlite3's primary benchmarking tool)...

View File

@ -809,9 +809,7 @@ common:
A read-only numeric property which is the "pointer" returned by the
configured allocator when this object is constructed. After
`dispose()` (inherited from [StructType][]) is called, this property
has the `undefined` value. When passing instances of this struct to
C-bound code, `pointer` is the value which must be passed in place
of a C-side struct pointer. When calling C-side code which takes a
has the `undefined` value. When calling C-side code which takes a
pointer to a struct of this type, simply pass it `myStruct.pointer`.
<a name='appendices'></a>

View File

@ -1,5 +1,5 @@
C wasm:\schange\sStructBinder\ssignature\sfor\ssqlite3_file::pMethods\sfrom\s'P'\sto\s'p'\sto\seliminate\san\sunnecessary\sand\sinconsistent\slevel\sof\smagic.
D 2022-09-26T11:34:31.822
C Get\sfiddle\sdb\sexport\sworking\sfor\sOPFS\sVFS.\sAdd\sroot\sdir\shandle\sto\sthe\smain\sOPFS\sVFS\sworker\sto\senable\screation\sof\scertain\sutility\sfunctions\swithout\sdelegating\sto\sthe\sasync\sworker.\sAdd\ssqlite3.capi.sqlite3_wasm_rc_str()\sto\smap\sinteger\sresult\scodes\sback\sto\stheir\sSQLITE_xxx\scounterparts.\sMinor\sdoc\stouchups.
D 2022-09-26T11:38:58.930
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -472,7 +472,7 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205
F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 105f6f7f211f49dea8fa6ee8b7b56492d5f9237ab0d1c1b3c970df11e4d0df02
F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 48af0d812e2e1099798ae0e27f1d4836d572d0d068c83b3e2088ef4d97c45c2f
F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle 0e88c8cfc3719e4b7e74980d9da664c709e68acf863e48386cda376edfd3bfb0
F ext/wasm/GNUmakefile 34a84e30e6b25e24959a8264e9dec020dffa82d96879dc55ad65d3c31c95d3b1
F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52
@ -482,9 +482,9 @@ F ext/wasm/api/README.md d876597edd2b9542b6ea031adaaff1c042076fde7b670b1dc6d8a87
F ext/wasm/api/post-js-footer.js b64319261d920211b8700004d08b956a6c285f3b0bba81456260a713ed04900c
F ext/wasm/api/post-js-header.js 2e5c886398013ba2af88028ecbced1e4b22dc96a86467f1ecc5ba9e64ef90a8b
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js cfff894bdf98a6c579975d09dd45471b0e3399f08a6f9e44a22646e8403196ed
F ext/wasm/api/sqlite3-api-glue.js fe5ca21ac519e6411f5e7a6403d06fe92c51ef81cca8e07ea8895df8ec9c2e4e
F ext/wasm/api/sqlite3-api-oo1.js f974e79d9af8f26bf33928c5730b0988cc706d14f59a5fe36394739b92249841
F ext/wasm/api/sqlite3-api-opfs.js 5585dc80aea9df54c3d5d3a6c62771bf741f21b23706330ba62571c57ec07abf
F ext/wasm/api/sqlite3-api-opfs.js 151f8bab445915dd4fa8fe3329accc5250351e7bf0af1518cb5cc49886c123d2
F ext/wasm/api/sqlite3-api-prologue.js 76db12cce58ec6724ec01a977dfbedabfd4916e915a6e7679ffc24dd52eef64e
F ext/wasm/api/sqlite3-api-worker1.js 2eeb2a24e1a90322d84a9b88a99919b806623de62792436446099c0988f2030b
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@ -502,12 +502,12 @@ F ext/wasm/demo-kvvfs1.html 7d4f28873de67f51ac18c584b7d920825139866a96049a49c424
F ext/wasm/demo-kvvfs1.js e884ea35022d772c0d1dd884b40011413696438394f605c6cd4808cfb1642a4a
F ext/wasm/fiddle.make fd56fa21bada6ecbf860686a9a789ebda7cc3d9b60835927000fcb00246ea50f
F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
F ext/wasm/fiddle/fiddle-worker.js d3e4d1e442a9a86cc34f8bd646059d848cf3345b5220883379268b03b3c3cdfa
F ext/wasm/fiddle/fiddle-worker.js 42c31065660e57844afa796bcf120666348c060fc4c656c335151b62da7fc60d
F ext/wasm/fiddle/fiddle.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2
F ext/wasm/fiddle/fiddle.js aa44051be6e48c53fd23c829177d43f557dcc6f0998ccfcbae7c473ff405f0c6
F ext/wasm/index.html 8b4b7ea052d558262c8466f94326fb455c21049b2d1d3577ed0a5fce15101ba8
F ext/wasm/index.html a39d14f6de6bd1294d39522fcf40eca3c9f0b6da44bb8df6948378e9f30468f5
F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215
F ext/wasm/jaccwabyt/jaccwabyt.md 447cc02b598f7792edaa8ae6853a7847b8178a18ed356afacbdbf312b2588106
F ext/wasm/jaccwabyt/jaccwabyt.md 9aa6951b529a8b29f578ec8f0355713c39584c92cf1708f63ba0cf917cb5b68e
F ext/wasm/jaccwabyt/jaccwabyt_test.c 39e4b865a33548f943e2eb9dd0dc8d619a80de05d5300668e9960fff30d0d36f
F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e568d2c2307c416902059339c06f19
F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
@ -615,7 +615,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960
F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c d69dfb5b082f9a25e6700e152ddb3d942359b847b1df504eb09f9b4531844f8d
F src/shell.c.in c4625fa9493cf815ff15e62bee5af93e91bbdb1a0c4e2c5a34a3744df3d5daaf
F src/shell.c.in 00a72221c8b2df823f3a9d712457ae8bebe627b5989fefa1ccfe099eafab5139
F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d
@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P bcec4f964a7b02f59be05286ff715bac654a32b19f05a743e402f4cdb207cab8
R 793f9e89b06d2236f21d1ef1114135f2
P 85f2e877e53899860af4dc6630044b471a9c7c82faba1f4e1e60ae991460b943
R 021d11845b25e048635a4d675c2c9f1f
U stephan
Z 73b30ac4eb53afa297401f4b1b820a88
Z 684fe82520a24a3caf4c9fdd7b640f27
# Remove this line to create a well-formed Fossil manifest.

View File

@ -1 +1 @@
85f2e877e53899860af4dc6630044b471a9c7c82faba1f4e1e60ae991460b943
9b2244e1c8a40efe6547094a1b57acc8f2173145a8731abb0e36268ce0a8ef41

View File

@ -12632,27 +12632,13 @@ int fiddle_experiment(int a,int b){
return a + b;
}
/* Only for emcc experimentation purposes.
Define this function in JS using:
emcc ... --js-library somefile.js
containing:
mergeInto(LibraryManager.library, {
my_foo: function(){
console.debug("my_foo()",arguments);
}
});
/*
** Returns a pointer to the current DB handle.
*/
/*extern void my_foo(sqlite3 *);*/
/* Only for emcc experimentation purposes. */
sqlite3 * fiddle_the_db(){
printf("fiddle_the_db(%p)\n", (const void*)globalDb);
/*my_foo(globalDb);*/
sqlite3 * fiddle_db_handle(){
return globalDb;
}
/* Only for emcc experimentation purposes. */
sqlite3 * fiddle_db_arg(sqlite3 *arg){
printf("fiddle_db_arg(%p)\n", (const void*)arg);
@ -12702,6 +12688,55 @@ void fiddle_reset_db(void){
sqlite3_free(zFilename);
}
/*
** This is a bit of a workaround: returns true if the current db uses
** the "opfs" VFS, else false.
*/
int fiddle_db_is_opfs(void){
sqlite3_vfs * pVfs = 0;
if( 0!=shellState.db ){
sqlite3_file_control( shellState.db, "main",
SQLITE_FCNTL_VFS_POINTER, &pVfs );
}
return pVfs ? 0==strcmp("opfs", pVfs->zName) : 0;
}
#if 0
/* TODO: test whether this impl works properly wrt xSize() in the
** unix-none VFS. The JS impl works fine for OPFS but not unix-none
** because xSize() is returning a garbage size.
*/
int fiddle_export_db( int (*callback)(unsigned const char *zOut, int n) ){
sqlite3_int64 nSize = 0;
sqlite3_int64 nPos = 0;
sqlite3_vfs * pVfs = 0;
sqlite3_file * pFile = 0;
unsigned char buf[1024 * 4];
const int nBuf = (int)sizeof(buf);
int rc = shellState.db
? sqlite3_file_control(shellState.db, "main",
SQLITE_FCNTL_VFS_POINTER, &pVfs)
: 0;
if( rc ) return rc;
else if( 0==pVfs ) return SQLITE_NOTFOUND;
rc = sqlite3_file_control(shellState.db, "main",
SQLITE_FCNTL_FILE_POINTER, &pFile);
if( rc ) return rc;
rc = pFile->pMethods->xFileSize(pFile, &nSize);
if(rc) return rc;
for( ; 0==rc && nPos<nSize; nPos += nBuf ){
rc = pFile->pMethods->xRead(pFile, buf, nBuf, nPos);
if(SQLITE_IOERR_SHORT_READ == rc){
rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/;
}
if(rc) return rc;
nPos += nBuf;
rc = callback(buf, nBuf);
}
return rc;
}
#endif
/*
** Trivial exportable function for emscripten. It processes zSql as if
** it were input to the sqlite3 shell and redirects all output to the