mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace time-based auto-unlock of opfs sync handles with lock acquisition/release via sqlite3_io_methods::xLock/xUnlock().

Browse Source 
FossilOrigin-Name: 2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40
This commit is contained in:
stephan 2022-10-04 17:06:51 +00:00
parent ed3182f233
commit 9a55773b2f
6 changed files with 115 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
ext/wasm/api/sqlite3-api-opfs.js
View File

@ -169,6 +169,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
W.onerror = function(err){
// The error object doesn't contain any useful info when the
// failure is, e.g., that the remote script is 404.
error("Error initializing OPFS asyncer:",err);
promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons."));
};
const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/;
@ -202,7 +203,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
environment or the other when sqlite3_os_end() is called (_if_ it
gets called at all in a wasm build, which is undefined).
*/
/**
State which we send to the async-api Worker or share with it.
This object must initially contain only cloneable or sharable
@ -224,7 +224,12 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
*/
const state = Object.create(null);
state.verbose = options.verbose;
state.littleEndian = true;
state.littleEndian = (()=>{
const buffer = new ArrayBuffer(2);
new DataView(buffer).setInt16(0, 256, true /* littleEndian */);
// Int16Array uses the platform's endianness.
return new Int16Array(buffer)[0] === 256;
})();
/** Whether the async counterpart should log exceptions to
the serialization channel. That produces a great deal of
noise for seemingly innocuous things like xAccess() checks
@ -265,7 +270,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
between both workers. This worker writes to it and the other
listens for changes. */
state.opIds.whichOp = i++;
/* Slot for storing return values. This work listens to that
/* Slot for storing return values. This worker listens to that
slot and the other worker writes to it. */
state.opIds.rc = i++;
/* Each function gets an ID which this worker writes to
@ -278,11 +283,13 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
state.opIds.xDeleteNoWait = i++;
state.opIds.xFileControl = i++;
state.opIds.xFileSize = i++;
state.opIds.xLock = i++;
state.opIds.xOpen = i++;
state.opIds.xRead = i++;
state.opIds.xSleep = i++;
state.opIds.xSync = i++;
state.opIds.xTruncate = i++;
state.opIds.xUnlock = i++;
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opIds['opfs-async-metrics'] = i++;
@ -290,7 +297,6 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
opfsUtil.metrics.reset();
}
/**
SQLITE_xxx constants to export to the async worker
counterpart...
@ -304,10 +310,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE',
'SQLITE_LOCK_NONE',
'SQLITE_LOCK_SHARED',
'SQLITE_LOCK_RESERVED',
'SQLITE_LOCK_PENDING',
'SQLITE_LOCK_EXCLUSIVE',
'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_READONLY'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
].forEach((k)=>{
if(undefined === (state.sq3Codes[k] = capi[k])){
toss("Maintenance required: not found:",k);
}
});
/**
@ -605,7 +618,8 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
xCheckReservedLock: function(pFile,pOut){
// Exclusive lock is automatically acquired when opened
//warn("xCheckReservedLock(",arguments,") is a no-op");
wasm.setMemValue(pOut,1,'i32');
const f = __openFiles[pFile];
wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32');
return 0;
},
xClose: function(pFile){
@ -643,9 +657,17 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
return rc;
},
xLock: function(pFile,lockType){
//2022-09: OPFS handles lock when opened
//warn("xLock(",arguments,") is a no-op");
return 0;
mTimeStart('xLock');
const f = __openFiles[pFile];
let rc = 0;
if( capi.SQLITE_LOCK_NONE === f.lockType ) {
rc = opRun('xLock', pFile, lockType);
if( 0===rc ) f.lockType = lockType;
}else{
f.lockType = lockType;
}
mTimeEnd();
return rc;
},
xRead: function(pFile,pDest,n,offset64){
/* int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst) */
@ -676,9 +698,16 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
return rc;
},
xUnlock: function(pFile,lockType){
//2022-09: OPFS handles lock when opened
//warn("xUnlock(",arguments,") is a no-op");
return 0;
mTimeStart('xUnlock');
const f = __openFiles[pFile];
let rc = 0;
if( capi.SQLITE_LOCK_NONE === lockType
&& f.lockType ){
rc = opRun('xUnlock', pFile, lockType);
}
if( 0===rc ) f.lockType = lockType;
mTimeEnd();
return rc;
},
xWrite: function(pFile,pSrc,n,offset64){
/* int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst) */
@ -696,7 +725,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
return rc;
}
}/*ioSyncWrappers*/;
/**
Impls for the sqlite3_vfs methods. Maintenance reminder: members
are in alphabetical order to simplify finding them.
@ -790,6 +819,7 @@ const installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri){
fh.sabView = state.sabFileBufView;
fh.sq3File = new sqlite3_file(pFile);
fh.sq3File.$pMethods = opfsIoMethods.pointer;
fh.lockType = capi.SQLITE_LOCK_NONE;
}
mTimeEnd();
return rc;
@ -1061,5 +1091,12 @@ installOpfsVfs.defaultProxyUri =
//self.location.pathname.replace(/[^/]*$/, "sqlite3-opfs-async-proxy.js");
"sqlite3-opfs-async-proxy.js";
//console.warn("sqlite3.installOpfsVfs.defaultProxyUri =",sqlite3.installOpfsVfs.defaultProxyUri);
self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>installOpfsVfs());
self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{
try{
return installOpfsVfs();
}catch(e){
console.error("installOpfsVfs() exception:",e);
throw e;
}
});
}/*sqlite3ApiBootstrap.initializers.push()*/);

15
ext/wasm/api/sqlite3-api-prologue.js
View File

@ -1312,14 +1312,25 @@ self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
// Is it okay to resolve these in parallel or do we need them
// to resolve in order? We currently only have 1, so it
// makes no difference.
lip = lip.map((f)=>f(sqlite3).catch(()=>{}));
lip = lip.map((f)=>f(sqlite3).catch((e)=>{
console.error("An async sqlite3 initializer failed:",e);
}));
//let p = lip.shift();
//while(lip.length) p = p.then(lip.shift());
//return p.then(()=>sqlite3);
return Promise.all(lip).then(()=>sqlite3);
}
};
sqlite3ApiBootstrap.initializers.forEach((f)=>f(sqlite3));
try{
sqlite3ApiBootstrap.initializers.forEach((f)=>{
f(sqlite3);
});
}catch(e){
/* If we don't report this here, it can get completely swallowed
up and disappear into the abyss of Promises and Workers. */
console.error("sqlite3 bootstrap initializer threw:",e);
throw e;
}
delete sqlite3ApiBootstrap.initializers;
sqlite3ApiBootstrap.sqlite3 = sqlite3;
return sqlite3;

78
ext/wasm/sqlite3-opfs-async-proxy.js
View File

@ -41,6 +41,9 @@ if(self.window === self){
}else if(!navigator.storage.getDirectory){
toss("This API requires navigator.storage.getDirectory.");
}
//warn("This file is very much experimental and under construction.",self.location.pathname);
/**
Will hold state copied to this object from the syncronous side of
this API.
@ -97,8 +100,6 @@ metrics.dump = ()=>{
console.log("Serialization metrics:",metrics.s11n);
};
//warn("This file is very much experimental and under construction.",self.location.pathname);
/**
Map of sqlite3_file pointers (integers) to metadata related to a
given OPFS file handles. The pointers are, in this side of the
@ -142,8 +143,7 @@ const getDirForFilename = async function f(absFilename, createDirs = false){
/**
Returns the sync access handle associated with the given file
handle object (which must be a valid handle object), lazily opening
it if needed. Timestamps the handle for use in relinquishing it
during idle time.
it if needed.
In order to help alleviate cross-tab contention for a dabase,
if an exception is thrown while acquiring the handle, this routine
@ -177,13 +177,12 @@ const getSyncHandle = async (fh)=>{
}
log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms');
}
fh.syncHandleTime = performance.now();
return fh.syncHandle;
};
const closeSyncHandle = async (fh)=>{
if(fh.syncHandle){
//warn("Closing sync handle for",fh.filenameAbs);
log("Closing sync handle for",fh.filenameAbs);
const h = fh.syncHandle;
delete fh.syncHandle;
return h.close();
@ -239,6 +238,7 @@ const wTimeEnd = ()=>(
*/
let flagAsyncShutdown = false;
/**
Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods
methods. Maintenance reminder: members are in alphabetical order
@ -373,6 +373,20 @@ const vfsAsyncImpls = {
storeAndNotify('xFileSize', sz);
mTimeEnd();
},
xLock: async function(fid,lockType){
mTimeStart('xLock');
const fh = __openFiles[fid];
let rc = 0;
if( !fh.syncHandle ){
try { await getSyncHandle(fh) }
catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR;
}
}
storeAndNotify('xLock',rc);
mTimeEnd();
},
xOpen: async function(fid/*sqlite3_file pointer*/, filename, flags){
const opName = 'xOpen';
mTimeStart(opName);
@ -473,6 +487,23 @@ const vfsAsyncImpls = {
storeAndNotify('xTruncate',rc);
mTimeEnd();
},
xUnlock: async function(fid,lockType){
mTimeStart('xUnlock');
let rc = 0;
const fh = __openFiles[fid];
if( state.sq3Codes.SQLITE_LOCK_NONE===lockType
&& fh.syncHandle ){
try { await closeSyncHandle(fh) }
catch(e){
state.s11n.storeException(1,e);
rc = state.sq3Codes.SQLITE_IOERR;
/* Maybe we want to not report this? "Destructors do not
throw." */
}
}
storeAndNotify('xUnlock',rc);
mTimeEnd();
},
xWrite: async function(fid,n,offset){
mTimeStart('xWrite');
let rc;
@ -495,7 +526,7 @@ const vfsAsyncImpls = {
storeAndNotify('xWrite',rc);
mTimeEnd();
}
};
}/*vfsAsyncImpls*/;
const initS11n = ()=>{
/**
@ -617,21 +648,7 @@ const waitLoop = async function f(){
We need to wake up periodically to give the thread a chance
to do other things.
*/
const waitTime = 500;
/**
relinquishTime defines the_approximate_ number of ms after which
a db sync access handle will be relinquished so that we do not
hold a persistent lock on it. When the following loop times out
while waiting, every (approximate) increment of this value it
will relinquish any db handles which have been idle for at least
this much time.
Reaquisition of a sync handle seems to take an average of
0.6-0.9ms on this dev machine but takes anywhere from 1-3ms every
once in a while (maybe 1 time in 5 or 10). Outliers as long as
7ms have been witnessed, but they're rare.
*/
const relinquishTime = 500;
const waitTime = 1000;
let lastOpTime = performance.now();
let now;
while(!flagAsyncShutdown){
@ -639,21 +656,6 @@ const waitLoop = async function f(){
if('timed-out'===Atomics.wait(
state.sabOPView, state.opIds.whichOp, 0, waitTime
)){
if(relinquishTime &&
(lastOpTime + relinquishTime <= (now = performance.now()))){
for(const fh of Object.values(__openFiles)){
if(fh.syncHandle && (
now - relinquishTime >= fh.syncHandleTime
)){
log("Relinquishing for timeout:",fh.filenameAbs);
await closeSyncHandle(fh)
/* Testing shows that we have to wait on this async
op to finish, else we might try to re-open it
before the close has run. The FS layer does not
retain the order those operations, apparently. */;
}
}
}
continue;
}
lastOpTime = performance.now();
@ -719,4 +721,4 @@ navigator.storage.getDirectory().then(function(d){
}
};
wMsg('opfs-async-loaded');
}).catch((e)=>error(e));
}).catch((e)=>error("error initializing OPFS asyncer:",e));

7
ext/wasm/test-opfs-vfs.js
View File

@ -34,10 +34,6 @@ const tryOpfsVfs = async function(sqlite3){
const wait = async (ms)=>{
return new Promise((resolve)=>setTimeout(resolve, ms));
};
const waitForRelinquish = async ()=>{
log("Waiting briefly to test sync handle relinquishing...");
return wait(1500);
};
const urlArgs = new URL(self.location.href).searchParams;
const dbFile = "my-persistent.db";
@ -45,13 +41,11 @@ const tryOpfsVfs = async function(sqlite3){
const db = new opfs.OpfsDb(dbFile,'ct');
log("db file:",db.filename);
await waitForRelinquish();
try{
if(opfs.entryExists(dbFile)){
let n = db.selectValue("select count(*) from sqlite_schema");
log("Persistent data found. sqlite_schema entry count =",n);
}
await waitForRelinquish();
db.transaction((db)=>{
db.exec({
sql:[
@ -63,7 +57,6 @@ const tryOpfsVfs = async function(sqlite3){
(performance.now() |0) / 4]
});
});
await waitForRelinquish();
log("count(*) from t =",db.selectValue("select count(*) from t"));
// Some sanity checks of the opfs utility functions...

18
manifest
View File

@ -1,5 +1,5 @@
C Tweaks\sto\sthe\sopfs\sasync\swait/relinquish\stimes.
D 2022-10-04T11:14:23.298
C Replace\stime-based\sauto-unlock\sof\sopfs\ssync\shandles\swith\slock\sacquisition/release\svia\ssqlite3_io_methods::xLock/xUnlock().
D 2022-10-04T17:06:51.812
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -487,8 +487,8 @@ F ext/wasm/api/pre-js.js 2db711eb637991b383fc6b5c0f3df65ec48a7201e5730e304beba8d
F ext/wasm/api/sqlite3-api-cleanup.js 5d22d1d3818ecacb23bfa223d5970cd0617d8cdbb48c8bc4bbd463f05b021a99
F ext/wasm/api/sqlite3-api-glue.js a5a1bd620e2e2c26b6e843cf439548b35c5bb5ed21b24b89a412e0b0a8592c42
F ext/wasm/api/sqlite3-api-oo1.js ac1e08d36bdfb5aa0a2d75b7d4c892fd51819d34c932370c3282810672bcc086
F ext/wasm/api/sqlite3-api-opfs.js 0db417af18fdcd39171366d9ac9e7bf57c5236aea4d2de26c759e65a22b7c446
F ext/wasm/api/sqlite3-api-prologue.js baf88cf253a35153674b068b5125238993ea2a7804152d238b5e40140b506ef5
F ext/wasm/api/sqlite3-api-opfs.js 78fd272d6ec4fe6da93910b978f693fff63beed8e0e5e7dab42a25dbdfefca4b
F ext/wasm/api/sqlite3-api-prologue.js 587e6aa373bd85e83bcc3fb82b3636b01aaf2caefa39679e02574e766560d711
F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
F ext/wasm/api/sqlite3-wasm.c 0c8cb242b80b8de27b74f05ed781f427958e04c9339c4720f30c3a7e78ee6242
@ -522,11 +522,11 @@ F ext/wasm/speedtest1.html e4cb5d722b494104fc1249e7c008ca018f820a784833c51004c95
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js 0861d0eff1ffb441b4ad98781a37825252ae2f08482cc5b29c27f34bcf952049
F ext/wasm/sqlite3-opfs-async-proxy.js 40fc1f779e18e08b31404ef832381ffd36b9d11447007aaf96223198324e4467
F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8
F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
F ext/wasm/test-opfs-vfs.js 0115d56f3f2a6475040dffe75a6851a353b1690708795944cf0fe1890bc7ff54
F ext/wasm/test-opfs-vfs.js 56c3d725044c668fa7910451e96c1195d25ad95825f9ac79f747a7759d1973d0
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
F ext/wasm/testing-worker1-promiser.js bd788e33c1807e0a6dda9c9a9d784bd3350ca49c9dd8ae2cc8719b506b6e013e
F ext/wasm/testing1.html 50575755e43232dbe4c2f97c9086b3118eb91ec2ee1fae931e6d7669fb17fcae
@ -2029,8 +2029,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 9d488081fc82a9abe3e81c7897fe6b29b6b337f62d0c62cb9cec7517bd54b53f
R 616ce8132e7f98fd04bd3d25f88d4168
P 35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b
R 447ab40d8f9d9bacdb3bbf1fc5e8416f
U stephan
Z da44b1ca0ef51fa028d53641db451066
Z 1f6ebc0e7b8d0de3ff6621f210427c47
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
35f33c23e5849de1c43c4499ee0a7fa11d26ae34949c1e820c3fa8e8873f9c2b
2625b7cfe1640c1d7e779ec1f37db970541598c0dc3e22e5eecf3c772d95ad40