mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cut the speedtest1 runtime of the OPFS VFS proxy by approximately 3/4ths via xRead/xWrite buffer-copying optimizations. Still slower than the WASMFS impl by approx. 1/5th.

Browse Source 
FossilOrigin-Name: fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51
This commit is contained in:
stephan 2022-09-19 17:09:09 +00:00
parent 61418d5a67
commit f815011a08
7 changed files with 107 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
ext/wasm/api/sqlite3-api-opfs.js
View File

@ -134,6 +134,34 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
by the Promise setup and, on success, installed as sqlite3.opfs.
*/
const opfsUtil = Object.create(null);
/**
Not part of the public API. Solely for internal/development
use.
*/
opfsUtil.metrics = {
dump: function(){
let k, n = 0, t = 0;
for(k in metrics){
const m = metrics[k];
n += m.count;
t += m.time;
m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0;
m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0;
}
console.log("metrics for",self.location.href,":",metrics,
"\nTotal of",n,"op(s) for",t,"ms");
},
reset: function(){
let k;
const r = (m)=>(m.count = m.time = m.wait = 0);
for(k in state.opIds){
r(metrics[k] = Object.create(null));
}
[ // timed routines which are not in state.opIds
'xFileControl'
].forEach((k)=>r(metrics[k] = Object.create(null)));
}
}/*metrics*/;
/**
State which we send to the async-api Worker or share with it.
@ -165,6 +193,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.fbInt64Offset =
state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
state.opIds = Object.create(null);
const metrics = Object.create(null);
{
let i = 0;
state.opIds.xAccess = i++;
@ -180,6 +209,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
opfsUtil.metrics.reset();
}
state.sq3Codes = Object.create(null);
@ -207,9 +237,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
given operation's signature in the async API counterpart.
*/
const opRun = (op,args)=>{
const t = performance.now();
Atomics.store(state.opSABView, state.opIds[op], -1);
wMsg(op, args);
Atomics.wait(state.opSABView, state.opIds[op], -1);
metrics[op].wait += performance.now() - t;
return Atomics.load(state.opSABView, state.opIds[op]);
};
@ -338,7 +370,20 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
tgt.ondispose.__removeFuncList.push(memKey, pFunc);
return (n,f)=>callee(tgt, n, f);
}/*installMethod*/;
const opTimer = Object.create(null);
opTimer.op = undefined;
opTimer.start = undefined;
const mTimeStart = (op)=>{
opTimer.start = performance.now();
opTimer.op = op;
//metrics[op] || toss("Maintenance required: missing metrics for",op);
++metrics[op].count;
};
const mTimeEnd = ()=>(
metrics[opTimer.op].time += performance.now() - opTimer.start
);
/**
Impls for the sqlite3_io_methods methods. Maintenance reminder:
members are in alphabetical order to simplify finding them.
@ -351,6 +396,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
return 0;
},
xClose: function(pFile){
mTimeStart('xClose');
let rc = 0;
const f = __openFiles[pFile];
if(f){
@ -358,22 +404,29 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
rc = opRun('xClose', pFile);
if(f.sq3File) f.sq3File.dispose();
}
mTimeEnd();
return rc;
},
xDeviceCharacteristics: function(pFile){
//debug("xDeviceCharacteristics(",pFile,")");
return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
},
xFileControl: function(pFile,op,pArg){
//debug("xFileControl(",arguments,") is a no-op");
xFileControl: function(pFile, opId, pArg){
mTimeStart('xFileControl');
if(capi.SQLITE_FCNTL_SYNC===opId){
return opRun('xSync', {fid:pFile, flags:0});
}
mTimeEnd();
return capi.SQLITE_NOTFOUND;
},
xFileSize: function(pFile,pSz64){
mTimeStart('xFileSize');
const rc = opRun('xFileSize', pFile);
if(!isWorkerErrCode(rc)){
const f = __openFiles[pFile];
wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0) ,'i64');
wasm.setMemValue(pSz64, f.sabViewFileSize.getBigInt64(0,true) ,'i64');
}
mTimeEnd();
return rc;
},
xLock: function(pFile,lockType){
@ -383,26 +436,31 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
},
xRead: function(pFile,pDest,n,offset){
/* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
mTimeStart('xRead');
const f = __openFiles[pFile];
let rc;
try {
// FIXME(?): block until we finish copying the xRead result buffer. How?
rc = opRun('xRead',{fid:pFile, n, offset});
if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){
let i = 0;
for(; i < n; ++i) wasm.setMemValue(pDest + i, f.sabView[i]);
// set() seems to be the fastest way to copy this...
wasm.heap8u().set(f.sabView.subarray(0, n), pDest);
}
}catch(e){
error("xRead(",arguments,") failed:",e,f);
rc = capi.SQLITE_IOERR_READ;
}
mTimeEnd();
return rc;
},
xSync: function(pFile,flags){
return opRun('xSync', {fid:pFile, flags});
return 0; // impl'd in xFileControl(). opRun('xSync', {fid:pFile, flags});
},
xTruncate: function(pFile,sz64){
return opRun('xTruncate', {fid:pFile, size: sz64});
mTimeStart('xTruncate');
const rc = opRun('xTruncate', {fid:pFile, size: sz64});
mTimeEnd();
return rc;
},
xUnlock: function(pFile,lockType){
//2022-09: OPFS handles lock when opened
@ -411,16 +469,19 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
},
xWrite: function(pFile,pSrc,n,offset){
/* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
mTimeStart('xWrite');
const f = __openFiles[pFile];
let rc;
try {
let i = 0;
// FIXME(?): block from here until we finish the xWrite. How?
for(; i < n; ++i) f.sabView[i] = wasm.getMemValue(pSrc+i);
return opRun('xWrite',{fid:pFile, n, offset});
f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n));
rc = opRun('xWrite',{fid:pFile, n, offset});
}catch(e){
error("xWrite(",arguments,") failed:",e,f);
return capi.SQLITE_IOERR_WRITE;
rc = capi.SQLITE_IOERR_WRITE;
}
mTimeEnd();
return rc;
}
}/*ioSyncWrappers*/;
@ -430,8 +491,10 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
*/
const vfsSyncWrappers = {
xAccess: function(pVfs,zName,flags,pOut){
mTimeStart('xAccess');
const rc = opRun('xAccess', wasm.cstringToJs(zName));
wasm.setMemValue(pOut, rc ? 0 : 1, 'i32');
mTimeEnd();
return 0;
},
xCurrentTime: function(pVfs,pOut){
@ -448,9 +511,11 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
return 0;
},
xDelete: function(pVfs, zName, doSyncDir){
mTimeStart('xDelete');
opRun('xDelete', {filename: wasm.cstringToJs(zName), syncDir: doSyncDir});
/* We're ignoring errors because we cannot yet differentiate
between harmless and non-harmless failures. */
mTimeEnd();
return 0;
},
xFullPathname: function(pVfs,zName,nOut,pOut){
@ -469,6 +534,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
},
//xSleep is optionally defined below
xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
mTimeStart('xOpen');
if(!f._){
f._ = {
fileTypes: {
@ -518,6 +584,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
args.sq3File.$pMethods = opfsIoMethods.pointer;
args.ba = new Uint8Array(args.sab);
}
mTimeEnd();
return rc;
}/*xOpen()*/
}/*vfsSyncWrappers*/;
@ -600,7 +667,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
};
opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype);
}
/**
Potential TODOs:
@ -670,6 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
}
}/*sanityCheck()*/;
W.onmessage = function({data}){
//log("Worker.onmessage:",data);
switch(data.type){

8
ext/wasm/common/whwasmutil.js
View File

@ -603,6 +603,10 @@ self.WhWasmUtilInstaller = function(target){
out) the pointer's value, else it will contain an essentially
random value.
ACHTUNG: calling this often, e.g. in a loop, can have a noticably
painful impact on performance. Rather than doing so, use
heapForSize() to fetch the heap object and read directly from it.
See: setMemValue()
*/
target.getMemValue = function(ptr, type='i8'){
@ -633,6 +637,10 @@ self.WhWasmUtilInstaller = function(target){
this function behaves as if the 3rd argument were `i32`.
This function returns itself.
ACHTUNG: calling this often, e.g. in a loop, can have a noticably
painful impact on performance. Rather than doing so, use
heapForSize() to fetch the heap object and assign directly to it.
*/
target.setMemValue = function f(ptr, value, type='i8'){
if (type.endsWith('*')) type = ptrIR;

5
ext/wasm/speedtest1-worker.html
View File

@ -260,9 +260,12 @@
flags["without"] = "rowid Use WITHOUT ROWID where appropriate";
const preselectedFlags = [
'big-transactions',
'memdb',
'singlethread'
];
if('opfs'!==urlParams.get('vfs')){
preselectedFlags.push('memdb');
}
Object.keys(flags).sort().forEach(function(f){
const opt = document.createElement('option');
eFlags.appendChild(opt);

2
ext/wasm/speedtest1-worker.js
View File

@ -45,7 +45,7 @@
const runSpeedtest = function(cliFlagsArray){
const scope = App.wasm.scopedAllocPush();
const dbFile = App.pDir+"/speedtest1.db";
const dbFile = App.pDir+"/speedtest1.sqlite3";
App.unlink(dbFile);
try{
const argv = [

3
ext/wasm/sqlite3-opfs-async-proxy.js
View File

@ -60,6 +60,7 @@ const logImpl = (level,...args)=>{
const log = (...args)=>logImpl(2, ...args);
const warn = (...args)=>logImpl(1, ...args);
const error = (...args)=>logImpl(0, ...args);
const metrics = Object.create(null);
warn("This file is very much experimental and under construction.",
self.location.pathname);
@ -220,7 +221,7 @@ const vfsAsyncImpls = {
let sz;
try{
sz = await fh.accessHandle.getSize();
fh.sabViewFileSize.setBigInt64(0, BigInt(sz));
fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
sz = 0;
}catch(e){
error("xFileSize():",e, fh);

20
manifest
View File

@ -1,5 +1,5 @@
C Remove\san\sobsolete\sfile.\sUpdate\swasmfs\sbuild\sfor\stoday's\schanges.\sFix\sthe\scase\sthat\sOPFS.xTruncate()\sis\spassed\sa\sBigInt\sinstead\sof\sa\sNumber.
D 2022-09-19T14:56:13.838
C Cut\sthe\sspeedtest1\sruntime\sof\sthe\sOPFS\sVFS\sproxy\sby\sapproximately\s3/4ths\svia\sxRead/xWrite\sbuffer-copying\soptimizations.\sStill\sslower\sthan\sthe\sWASMFS\simpl\sby\sapprox.\s1/5th.
D 2022-09-19T17:09:09.850
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
F ext/wasm/api/sqlite3-api-opfs.js e6b3a168f18b94f01fd21028ffdbaf4e576c53c2f2762032c9ee7d24b7a3bd99
F ext/wasm/api/sqlite3-api-opfs.js 627b2d8fcd37b6479d7f0e8d5ba891d00c1eaf9b8cf99981081c2bf2e7ed8560
F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@ -495,7 +495,7 @@ F ext/wasm/batch-runner.js 6f5b86e0b5519a9a941d9f17ee9c5ecdc63f452f157602fe7fdf8
F ext/wasm/common/SqliteTestUtil.js 529161a624265ba84271a52db58da022649832fa1c71309fb1e02cc037327a2b
F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
F ext/wasm/common/testing.css 3a5143699c2b73a85b962271e1a9b3241b30d90e30d895e4f55665e648572962
F ext/wasm/common/whwasmutil.js f7282ef36c9625330d4e6e82d1beec6678cd101e95e7108cd85db587a788c145
F ext/wasm/common/whwasmutil.js f64f94e2a3e1d8fd2139b4d80f00a3ab504985e38103c45b1ff15ec8b44fcce3
F ext/wasm/demo-123-worker.html e419b66495d209b5211ec64903b4cfb3ca7df20d652b41fcd28bf018a773234f
F ext/wasm/demo-123.html aa281d33b7eefa755f3122b7b5a18f39a42dc5fb69c8879171bf14b4c37c4ec4
F ext/wasm/demo-123.js 234655683e35a4543a23de7b10800d76b0369947b33e089e5613171fa7795afb
@ -513,13 +513,13 @@ F ext/wasm/jaccwabyt/jaccwabyt_test.exports 5ff001ef975c426ffe88d7d8a6e96ec725e5
F ext/wasm/scratchpad-wasmfs-main.html 20cf6f1a8f368e70d01e8c17200e3eaa90f1c8e1029186d836d14b83845fbe06
F ext/wasm/scratchpad-wasmfs-main.js f0836e3576df7a89390d777bb53e142e559e8a79becfb2a5a976490b05a1c4fa
F ext/wasm/speedtest1-wasmfs.html 9d8cd19eab8854d17f7129aa11607cae6f6d9857c505a4aef13000588583d93e
F ext/wasm/speedtest1-worker.html f7adcc68af76d347741ecd884e0c773f707fa2cdec87cfd9b2e147f5fc2c56c7
F ext/wasm/speedtest1-worker.js d7e02ff74c28c8f2ef19eb6585642d44b7c8e7b83ac19b7848b209a880945706
F ext/wasm/speedtest1-worker.html 7bd3ede33e08067f59e8d6e3dc25016fc930222b059d66b2554203bf262aba02
F ext/wasm/speedtest1-worker.js 11e7f68cedd2a83b0e638f94c1d2f58406ba672a7e88b66bff5d4f4284e8ba16
F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0aa0d828cf1d
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js 7ce90feb4ed582a7709f6a99a46fd43a240a1c1ad2f9f47efff1bd32136ab09c
F ext/wasm/sqlite3-opfs-async-proxy.js 7139530dd8a408f2897115d3ea04dfe03f175df66870b1e8a49733a77a06f970
F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 2ec7e09139a510b9fd29e4c97283b20740a00f369193c6fecbb734f187e81b48
R 1715d33dc730c3ab31438d0a66f7aa0a
P 976053925013bf1975f5f9222e28ba648af28e305bb6bdae600eb24d0e136bec
R 79e6bae24f2a8f4f545c24b5d2562e72
U stephan
Z 0f0e9b03805cac5450f028e863da1f77
Z 251c239274312fdfb8787b5b1f3a9b52
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
976053925013bf1975f5f9222e28ba648af28e305bb6bdae600eb24d0e136bec
fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51