mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

OPFS VFS: moved i/o buffer from per-file to the VFS, and related refactoring, in prep for experimentation with a new inter-worker comms model.

Browse Source 
FossilOrigin-Name: d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d
This commit is contained in:
stephan 2022-09-20 01:28:47 +00:00
parent aec046a264
commit c4b87be3e8
4 changed files with 65 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
ext/wasm/api/sqlite3-api-opfs.js
View File

@ -188,17 +188,22 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
const state = Object.create(null);
state.verbose = options.verbose;
state.fileBufferSize =
1024 * 64 + 8 /* size of aFileHandle.sab. 64k = max sqlite3 page
size. The additional bytes are space for
holding BigInt results, since we cannot store
those via the Atomics API (which only works on
an Int32Array). */;
1024 * 64 /* size of aFileHandle.sab. 64k = max sqlite3 page
size. */;
state.sabOffsetS11n = state.fileBufferSize;
state.sabIO = new SharedArrayBuffer(
state.fileBufferSize
+ 4096/* arg/result serialization */
+ 8 /* to be removed - porting crutch */
);
state.fbInt64Offset =
state.fileBufferSize - 8 /*spot in fileHandle.sab to store an int64 result */;
state.sabIO.byteLength - 8 /*spot in fileHandle.sab to store an int64 result.
to be removed. Porting crutch. */;
state.opIds = Object.create(null);
const metrics = Object.create(null);
{
let i = 0;
state.opIds.nothing = i++;
state.opIds.xAccess = i++;
state.opIds.xClose = i++;
state.opIds.xDelete = i++;
@ -211,7 +216,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
state.opIds.xTruncate = i++;
state.opIds.xWrite = i++;
state.opIds.mkdir = i++;
state.opSAB = new SharedArrayBuffer(i * 4/*sizeof int32*/);
state.sabOP = new SharedArrayBuffer(i * 4/*sizeof int32*/);
state.opIds.xFileControl = state.opIds.xSync /* special case */;
opfsUtil.metrics.reset();
}
@ -225,7 +230,9 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
'SQLITE_IOERR_WRITE', 'SQLITE_IOERR_FSYNC',
'SQLITE_IOERR_TRUNCATE', 'SQLITE_IOERR_DELETE',
'SQLITE_IOERR_ACCESS', 'SQLITE_IOERR_CLOSE',
'SQLITE_IOERR_DELETE'
'SQLITE_IOERR_DELETE',
'SQLITE_OPEN_CREATE', 'SQLITE_OPEN_DELETEONCLOSE',
'SQLITE_OPEN_READONLY'
].forEach(function(k){
state.sq3Codes[k] = capi[k] || toss("Maintenance required: not found:",k);
state.sq3Codes._reverse[capi[k]] = k;
@ -236,17 +243,17 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
/**
Runs the given operation in the async worker counterpart, waits
for its response, and returns the result which the async worker
writes to the given op's index in state.opSABView. The 2nd argument
writes to the given op's index in state.sabOPView. The 2nd argument
must be a single object or primitive value, depending on the
given operation's signature in the async API counterpart.
*/
const opRun = (op,args)=>{
const t = performance.now();
Atomics.store(state.opSABView, state.opIds[op], -1);
Atomics.store(state.sabOPView, state.opIds[op], -1);
wMsg(op, args);
Atomics.wait(state.opSABView, state.opIds[op], -1);
Atomics.wait(state.sabOPView, state.opIds[op], -1);
metrics[op].wait += performance.now() - t;
return Atomics.load(state.opSABView, state.opIds[op]);
return Atomics.load(state.sabOPView, state.opIds[op]);
};
/**
@ -574,10 +581,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
args.fid = pFile;
args.filename = zName;
args.sab = new SharedArrayBuffer(state.fileBufferSize);
args.fileType = f._.getFileType(args.filename, flags);
args.create = !!(flags & capi.SQLITE_OPEN_CREATE);
args.deleteOnClose = !!(flags & capi.SQLITE_OPEN_DELETEONCLOSE);
args.readOnly = !!(flags & capi.SQLITE_OPEN_READONLY);
args.flags = flags;
const rc = opRun('xOpen', args);
if(!rc){
/* Recall that sqlite3_vfs::xClose() will be called, even on
@ -586,8 +590,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32');
}
__openFiles[pFile] = args;
args.sabView = new Uint8Array(args.sab);
args.sabViewFileSize = new DataView(args.sab, state.fbInt64Offset, 8);
args.sabView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
args.sabViewFileSize = new DataView(state.sabIO, state.fbInt64Offset, 8);
args.sq3File = new sqlite3_file(pFile);
args.sq3File.$pMethods = opfsIoMethods.pointer;
args.ba = new Uint8Array(args.sab);
@ -615,7 +619,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
assume it's sane and use it, otherwise install a JS-based
one. */
vfsSyncWrappers.xSleep = function(pVfs,ms){
Atomics.wait(state.opSABView, state.opIds.xSleep, 0, ms);
Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms);
return 0;
};
}
@ -702,8 +706,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
log("xAccess(",dbFile,") exists ?=",rc);
rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile,
fid, openFlags, pOut);
log("open rc =",rc,"state.opSABView[xOpen] =",
state.opSABView[state.opIds.xOpen]);
log("open rc =",rc,"state.sabOPView[xOpen] =",
state.sabOPView[state.opIds.xOpen]);
if(isWorkerErrCode(rc)){
error("open failed with code",rc);
return;
@ -733,7 +737,7 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
log("waking up from xSleep()");
}
rc = ioSyncWrappers.xClose(fid);
log("xClose rc =",rc,"opSABView =",state.opSABView);
log("xClose rc =",rc,"sabOPView =",state.sabOPView);
log("Deleting file:",dbFile);
vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234);
vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut);
@ -767,7 +771,8 @@ sqlite3.installOpfsVfs = function callee(asyncProxyUri = callee.defaultProxyUri)
toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS");
}
capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods);
state.opSABView = new Int32Array(state.opSAB);
state.sabOPView = new Int32Array(state.sabOP);
state.sabFileBufView = new Uint8Array(state.sabFileBufView, 0, state.fileBufferSize);
if(options.sanityChecks){
warn("Running sanity checks because of opfs-sanity-check URL arg...");
sanityCheck();

58
ext/wasm/sqlite3-opfs-async-proxy.js
View File

@ -131,8 +131,8 @@ const getDirForPath = async function f(absFilename, createDirs = false){
*/
const storeAndNotify = (opName, value)=>{
log(opName+"() is notify()ing w/ value:",value);
Atomics.store(state.opSABView, state.opIds[opName], value);
Atomics.notify(state.opSABView, state.opIds[opName]);
Atomics.store(state.sabOPView, state.opIds[opName], value);
Atomics.notify(state.sabOPView, state.opIds[opName]);
};
/**
@ -214,6 +214,12 @@ const vfsAsyncImpls = {
}
mTimeEnd();
},
xDelete: async function(...args){
mTimeStart('xDelete');
const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
storeAndNotify('xDelete', rc);
mTimeEnd();
},
xDeleteNoWait: async function({filename, syncDir, recursive = false}){
/* The syncDir flag is, for purposes of the VFS API's semantics,
ignored here. However, if it has the value 0x1234 then: after
@ -248,12 +254,6 @@ const vfsAsyncImpls = {
}
return rc;
},
xDelete: async function(...args){
mTimeStart('xDelete');
const rc = await vfsAsyncImpls.xDeleteNoWait(...args);
storeAndNotify('xDelete', rc);
mTimeEnd();
},
xFileSize: async function(fid){
mTimeStart('xFileSize');
log("xFileSize(",arguments,")");
@ -261,6 +261,9 @@ const vfsAsyncImpls = {
let sz;
try{
sz = await fh.accessHandle.getSize();
if(!fh.sabViewFileSize){
fh.sabViewFileSize = new DataView(state.sabIO,state.fbInt64Offset,8);
}
fh.sabViewFileSize.setBigInt64(0, BigInt(sz), true);
sz = 0;
}catch(e){
@ -272,16 +275,15 @@ const vfsAsyncImpls = {
},
xOpen: async function({
fid/*sqlite3_file pointer*/,
sab/*file-specific SharedArrayBuffer*/,
filename,
fileType = undefined /*mainDb, mainJournal, etc.*/,
create = false, readOnly = false, deleteOnClose = false
flags
}){
const opName = 'xOpen';
mTimeStart(opName);
log(opName+"(",arguments[0],")");
const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags);
const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags);
try{
if(create) readOnly = false;
log(opName+"(",arguments[0],")");
let hDir, filenamePart;
try {
[hDir, filenamePart] = await getDirForPath(filename, !!create);
@ -290,20 +292,8 @@ const vfsAsyncImpls = {
mTimeEnd();
return;
}
const hFile = await hDir.getFileHandle(filenamePart, {create: !!create});
log(opName,"filenamePart =",filenamePart, 'hDir =',hDir);
const fobj = __openFiles[fid] = Object.create(null);
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.fileType = fileType;
fobj.sab = sab;
fobj.sabView = new Uint8Array(sab,0,state.fbInt64Offset);
fobj.sabViewFileSize = new DataView(sab,state.fbInt64Offset,8);
fobj.create = !!create;
fobj.readOnly = !!readOnly;
fobj.deleteOnClose = !!deleteOnClose;
const hFile = await hDir.getFileHandle(filenamePart, {create});
const fobj = Object.create(null);
/**
wa-sqlite, at this point, grabs a SyncAccessHandle and
assigns it to the accessHandle prop of the file state
@ -311,6 +301,14 @@ const vfsAsyncImpls = {
places that limitation on it.
*/
fobj.accessHandle = await hFile.createSyncAccessHandle();
__openFiles[fid] = fobj;
fobj.filenameAbs = filename;
fobj.filenamePart = filenamePart;
fobj.dirHandle = hDir;
fobj.fileHandle = hFile;
fobj.sabView = state.sabFileBufView;
fobj.readOnly = create ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags);
fobj.deleteOnClose = deleteOnClose;
storeAndNotify(opName, 0);
}catch(e){
error(opName,e);
@ -395,8 +393,10 @@ navigator.storage.getDirectory().then(function(d){
state.verbose = opt.verbose ?? 2;
state.fileBufferSize = opt.fileBufferSize;
state.fbInt64Offset = opt.fbInt64Offset;
state.opSAB = opt.opSAB;
state.opSABView = new Int32Array(state.opSAB);
state.sabOP = opt.sabOP;
state.sabOPView = new Int32Array(state.sabOP);
state.sabIO = opt.sabIO;
state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize);
state.opIds = opt.opIds;
state.sq3Codes = opt.sq3Codes;
Object.keys(vfsAsyncImpls).forEach((k)=>{

14
manifest
View File

@ -1,5 +1,5 @@
C Further\smetrics\sand\sbuffer-copy\soptimizations\sin\sthe\sOPFS\sproxy,\sbut\swith\slittle\seffect.
D 2022-09-19T18:22:29.467
C OPFS\sVFS:\smoved\si/o\sbuffer\sfrom\sper-file\sto\sthe\sVFS,\sand\srelated\srefactoring,\sin\sprep\sfor\sexperimentation\swith\sa\snew\sinter-worker\scomms\smodel.
D 2022-09-20T01:28:47.162
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -484,7 +484,7 @@ F ext/wasm/api/post-js-header.js 0e853b78db83cb1c06b01663549e0e8b4f377f12f5a2d9a
F ext/wasm/api/sqlite3-api-cleanup.js 8564a6077cdcaea9a9f428a019af8a05887f0131e6a2a1e72a7ff1145fadfe77
F ext/wasm/api/sqlite3-api-glue.js 366d580c8e5bf7fcf4c6dee6f646c31f5549bd417ea03a59a0acca00e8ecce30
F ext/wasm/api/sqlite3-api-oo1.js 2d13dddf0d2b4168a9249f124134d37924331e5b55e05dba18b6d661fbeefe48
F ext/wasm/api/sqlite3-api-opfs.js df3d085a55be11a0b7bce3d42ea1b721ff8aaba93659d0ec48c3327dde384596
F ext/wasm/api/sqlite3-api-opfs.js f8027fb4af1c24fcfad31889f8a5ccfa3b96d6e812d3495b13833bef57034046
F ext/wasm/api/sqlite3-api-prologue.js 0d2639387b94c30f492d4aea6e44fb7b16720808678464559458fd2ae3759655
F ext/wasm/api/sqlite3-api-worker1.js ee4cf149cbacb63d06b536674f822aa5088b7e022cdffc69f1f36cebe2f9fea0
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
@ -519,7 +519,7 @@ F ext/wasm/speedtest1.html 512addeb3c27c94901178b7bcbde83a6f95c093f9ebe16a2959a0
F ext/wasm/split-speedtest1-script.sh a3e271938d4d14ee49105eb05567c6a69ba4c1f1293583ad5af0cd3a3779e205 x
F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d82675bd63d9c2d97a15f0
F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5
F ext/wasm/sqlite3-opfs-async-proxy.js f1a270e7a8adeb80c73e0b345e472e574e694d9eaec6588ce3671438b79eb351
F ext/wasm/sqlite3-opfs-async-proxy.js 7ebc36915cd61bd4f067a4823307f4d4eb2678a173aaae470c534e8fe9cda650
F ext/wasm/sqlite3-worker1-promiser.js 4fd0465688a28a75f1d4ee4406540ba494f49844e3cad0670d0437a001943365
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5
@ -2026,8 +2026,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P fb7f287310d74a3e236125ae9c49b859f9263c29ae85161c1bcf9dd0778d8a51
R 946ef52a6b1d5b2d6cbf3b063dacd006
P d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96
R e5a1ff53ebe902985ab8e87bba2c71fb
U stephan
Z f42132e655dd81eba8154f54be0c97de
Z d05962442428c5dae722ab00dca47e48
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
d1f1fe6f1c60640f7770dfb9245c459a09b8d24ec2ddf664dff77c810bd51f96
d4d63e4580ad8d497310608175308c03c517e051d7865cb66aa0b10356612d7d