mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

More work on porting the sahpool bits. Revert to random backing-store names because it works better with the capacity-reduction algorithm.

Browse Source 
FossilOrigin-Name: b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411
This commit is contained in:
stephan 2023-07-15 01:02:38 +00:00
parent bee3213145
commit 100bc4429a
5 changed files with 255 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ext/wasm/GNUmakefile
View File

@ -46,12 +46,12 @@
# $(eval), or at least centralize the setup of the numerous vars
# related to each build variant $(JS_BUILD_MODES).
#
#default: all
default: quick
SHELL := $(shell which bash 2>/dev/null)
MAKEFILE := $(lastword $(MAKEFILE_LIST))
CLEAN_FILES :=
DISTCLEAN_FILES := ./--dummy--
default: all
#default: quick
release: oz
# JS_BUILD_MODES exists solely to reduce repetition in documentation
# below.

287
ext/wasm/api/sqlite3-vfs-opfs-sahpool.js
View File

@ -63,15 +63,14 @@
'use strict';
globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const installOpfsVfs = async function(sqlite3){
const pToss = (...args)=>Promise.reject(new Error(args.join(' ')));
if(!globalThis.FileSystemHandle ||
!globalThis.FileSystemDirectoryHandle ||
!globalThis.FileSystemFileHandle ||
!globalThis.FileSystemFileHandle.prototype.createSyncAccessHandle ||
!navigator?.storage?.getDirectory){
return pToss("Missing required OPFS APIs.");
return Promise.reject(new Error("Missing required OPFS APIs."));
}
const thePromise = new Promise(function(promiseResolve, promiseReject_){
return new Promise(async function(promiseResolve, promiseReject_){
const verbosity = 3;
const loggers = [
sqlite3.config.error,
@ -130,10 +129,13 @@ const installOpfsVfs = async function(sqlite3){
'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null)
);
const getFilename = function(ndx){
return 'sahpool-'+('00'+ndx).substr(-3);
}
const getFilename = false
? (ndx)=>'sahpool-'+('00'+ndx).substr(-3)
: ()=>Math.random().toString(36).slice(2)
/**
All state for the VFS.
*/
const SAHPool = Object.assign(Object.create(null),{
/* OPFS dir in which VFS metadata is stored. */
vfsDir: ".sqlite3-opfs-sahpool",
@ -148,24 +150,33 @@ const installOpfsVfs = async function(sqlite3){
addCapacity: async function(n){
const cap = this.getCapacity();
for(let i = cap; i < cap+n; ++i){
const name = getFilename(i);
const name = getFilename(i)
/* Reminder: because of how removeCapacity() works, we
really want random names. At this point in the dev
process that fills up the OPFS with randomly-named files
each time the page is reloaded, so delay the return to
random names until we've gotten far enough to eliminate
that problem. */;
const h = await this.dirHandle.getFileHandle(name, {create:true});
let ah = await h.createSyncAccessHandle();
if(0===i){
/* Ensure that this client has the "all-synchronous"
OPFS API and fail if they don't. */
if(undefined !== ah.close()){
toss("OPFS API is too old for opfs-sahpool:",
"it has an async close() method.");
}
ah = await h.createSyncAccessHandle();
}
const ah = await h.createSyncAccessHandle();
this.mapAH2Name.set(ah,name);
this.setAssociatedPath(ah, '', 0);
}
},
setAssociatedPath: function(accessHandle, path, flags){
// TODO
removeCapacity: async function(n){
let nRm = 0;
for(const ah of Array.from(this.availableAH)){
if(nRm === n || this.getFileCount() === this.getCapacity()){
break;
}
const name = this.mapAH2Name.get(ah);
ah.close();
await this.dirHandle.removeEntry(name);
this.mapAH2Name.delete(ah);
this.availableAH.delete(ah);
++nRm;
}
return nRm;
},
releaseAccessHandles: function(){
for(const ah of this.mapAH2Name.keys()) ah.close();
@ -174,7 +185,98 @@ const installOpfsVfs = async function(sqlite3){
this.availableAH.clear();
},
acquireAccessHandles: async function(){
// TODO
const files = [];
for await (const [name,h] of this.dirHandle){
if('file'===h.kind){
files.push([name,h]);
}
}
await Promise.all(files.map(async ([name,h])=>{
const ah = await h.createSyncAccessHandle()
/*TODO: clean up and fail vfs init on error*/;
this.mapAH2Name.set(ah, name);
const path = this.getAssociatedPath(ah);
if(path){
this.mapPath2AH.set(path, ah);
}else{
this.availableAH.add(ah);
}
}));
},
gapBody: new Uint8Array(HEADER_CORPUS_SIZE),
textDecoder: new TextDecoder(),
getAssociatedPath: function(sah){
const body = this.gapBody;
sah.read(body, {at: 0});
// Delete any unexpected files left over by previous
// untimely errors...
const dv = new DataView(body.buffer, body.byteOffset);
const flags = dv.getUint32(HEADER_OFFSET_FLAGS);
if(body[0] &&
((flags & capi.SQLITE_OPEN_DELETEONCLOSE) ||
(flags & PERSISTENT_FILE_TYPES)===0)){
warn(`Removing file with unexpected flags ${flags.toString(16)}`);
this.setAssociatedPath(sah, '', 0);
return '';
}
const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4);
sah.read(fileDigest, {at: HEADER_OFFSET_DIGEST});
const compDigest = this.computeDigest(body);
if(fileDigest.every((v,i) => v===compDigest[i])){
// Valid digest
const pathBytes = body.findIndex((v)=>0===v);
if(0===pathBytes){
// This file is unassociated, so ensure that it's empty
// to avoid leaving stale db data laying around.
sah.truncate(HEADER_OFFSET_DATA);
}
return this.textDecoder.decode(body.subarray(0,pathBytes));
}else{
// Invalid digest
warn('Disassociating file with bad digest.');
this.setAssociatedPath(sah, '', 0);
return '';
}
},
textEncoder: new TextEncoder(),
setAssociatedPath: function(sah, path, flags){
const body = this.gapBody;
const enc = this.textEncoder.encodeInto(path, body);
if(HEADER_MAX_PATH_SIZE <= enc.written){
toss("Path too long:",path);
}
const dv = new DataView(body.buffer, body.byteOffset);
dv.setUint32(HEADER_OFFSET_FLAGS, flags);
const digest = this.computeDigest(body);
sah.write(body, {at: 0});
sah.write(digest, {at: HEADER_OFFSET_DIGEST});
sah.flush();
if(path){
this.mapPath2AH.set(path, sah);
this.availableAH.delete(sah);
}else{
// This is not a persistent file, so eliminate the contents.
sah.truncate(HEADER_OFFSET_DATA);
this.mapPath2AH.delete(path);
this.availableAH.add(sah);
}
},
computeDigest: function(byteArray){
if(!byteArray[0]){
// Deleted file
return new Uint32Array([0xfecc5f80, 0xaccec037]);
}
let h1 = 0xdeadbeef;
let h2 = 0x41c6ce57;
for(const v of byteArray){
h1 = 31 * h1 + (v * 307);
h2 = 31 * h2 + (v * 307);
}
return new Uint32Array([h1>>>0, h2>>>0]);
},
reset: async function(){
await this.isReady;
@ -187,8 +289,20 @@ const installOpfsVfs = async function(sqlite3){
this.dirHandle = h;
this.releaseAccessHandles();
await this.acquireAccessHandles();
},
getPath: function(arg) {
if(wasm.isPtr(arg)) arg = wasm.cstrToJs(arg);
return ((arg instanceof URL)
? arg
: new URL(arg, 'file://localhost/')).pathname;
},
deletePath: function(path) {
const sah = this.mapPath2AH.get(path);
if(sah) {
// Un-associate the SQLite path from the OPFS file.
this.setAssociatedPath(sah, '', 0);
}
}
// much more TODO
})/*SAHPool*/;
sqlite3.SAHPool = SAHPool/*only for testing*/;
// Much, much more TODO...
@ -201,8 +315,21 @@ const installOpfsVfs = async function(sqlite3){
return 0;
},
xClose: function(pFile){
let rc = 0;
return rc;
const file = SAHPool.mapId2File.get(pFile);
if(file) {
try{
log(`xClose ${file.path}`);
file.sah.flush();
SAHPool.mapId2File.delete(pFIle);
if(file.flags & capi.SQLITE_OPEN_DELETEONCLOSE){
SAHPool.deletePath(file.path);
}
}catch(e){
error("xClose() failed:",e.message);
return capi.SQLITE_IOERR;
}
}
return 0;
},
xDeviceCharacteristics: function(pFile){
return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN;
@ -211,32 +338,74 @@ const installOpfsVfs = async function(sqlite3){
return capi.SQLITE_NOTFOUND;
},
xFileSize: function(pFile,pSz64){
let rc = 0;
return rc;
const file = SAHPool.mapId2File(pFile);
const size = file.sah.getSize() - HEADER_OFFSET_DATA;
//log(`xFileSize ${file.path} ${size}`);
wasm.poke64(pSz64, BigInt(size));
return 0;
},
xLock: function(pFile,lockType){
let rc = capi.SQLITE_IOERR_LOCK;
let rc = capi.SQLITE_IOERR;
return rc;
},
xRead: function(pFile,pDest,n,offset64){
let rc = capi.SQLITE_IOERR_READ;
return rc;
const file = SAHPool.mapId2File.get(pFile);
log(`xRead ${file.path} ${n} ${offset64}`);
try {
const nRead = file.sah.read(
pDest, {at: HEADER_OFFSET_DATA + offset64}
);
if(nRead < n){
wasm.heap8u().fill(0, pDest + nRead, pDest + n);
return capi.SQLITE_IOERR_SHORT_READ;
}
return 0;
}catch(e){
error("xRead() failed:",e.message);
return capi.SQLITE_IOERR;
}
},
xSectorSize: function(pFile){
return SECTOR_SIZE;
},
xSync: function(pFile,flags){
let rc = capi.SQLITE_IOERR_FSYNC;
return rc;
const file = SAHPool.mapId2File.get(pFile);
//log(`xSync ${file.path} ${flags}`);
try{
file.sah.flush();
return 0;
}catch(e){
error("xSync() failed:",e.message);
return capi.SQLITE_IOERR;
}
},
xTruncate: function(pFile,sz64){
let rc = capi.SQLITE_IOERR_TRUNCATE;
return rc;
},
xUnlock: function(pFile,lockType){
let rc = capi.SQLITE_IOERR_UNLOCK;
return rc;
const file = SAHPool.mapId2File.get(pFile);
//log(`xTruncate ${file.path} ${iSize}`);
try{
file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64));
return 0;
}catch(e){
error("xTruncate() failed:",e.message);
return capi.SQLITE_IOERR;
}
},
/**xUnlock: function(pFile,lockType){
return capi.SQLITE_IOERR;
},*/
xWrite: function(pFile,pSrc,n,offset64){
let rc = capi.SQLITE_IOERR_WRITE;
return rc;
const file = SAHPool.mapId2File(pFile);
//log(`xWrite ${file.path} ${n} ${offset64}`);
try{
const nBytes = file.sah.write(
pSrc, { at: HEADER_OFFSET_DATA + Number(offset64) }
);
return nBytes === n ? 0 : capi.SQLITE_IOERR;
return 0;
}catch(e){
error("xWrite() failed:",e.message);
return capi.SQLITE_IOERR;
}
}
}/*ioSyncWrappers*/;
@ -246,8 +415,9 @@ const installOpfsVfs = async function(sqlite3){
*/
const vfsSyncWrappers = {
xAccess: function(pVfs,zName,flags,pOut){
const rc = capi.SQLITE_ERROR;
return rc;
const name = this.getPath(zName);
wasm.poke32(pOut, SAHPool.mapPath2AH.has(name) ? 1 : 0);
return 0;
},
xCurrentTime: function(pVfs,pOut){
wasm.poke(pOut, 2440587.5 + (new Date().getTime()/86400000),
@ -260,8 +430,13 @@ const installOpfsVfs = async function(sqlite3){
return 0;
},
xDelete: function(pVfs, zName, doSyncDir){
const rc = capi.SQLITE_ERROR;
return rc;
try{
SAHPool.deletePath(SAHPool.getPath(zName));
return 0;
}catch(e){
error("Error xDelete()ing file:",e.message);
return capi.SQLITE_IOERR_DELETE;
}
},
xFullPathname: function(pVfs,zName,nOut,pOut){
const i = wasm.cstrncpy(pOut, zName, nOut);
@ -301,6 +476,31 @@ const installOpfsVfs = async function(sqlite3){
};
}
/**
Ensure that the client has a "fully-sync" SAH impl,
else reject the promise. Returns true on success,
else false.
*/
const affirmHasSyncAPI = async function(){
try {
const dh = await navigator.storage.getDirectory();
const fn = '.opfs-sahpool-sync-check-'+Math.random().toString(36).slice(2);
const fh = await dh.getFileHandle(fn, { create: true });
const ah = await fh.createSyncAccessHandle();
const close = ah.close();
await close;
await dh.removeEntry(fn);
if(close?.then){
toss("The local OPFS API is too old for opfs-sahpool:",
"it has an async FileSystemSyncAccessHandle.close() method.");
}
return true;
}catch(e){
promiseReject(e);
return false;
}
};
if(!(await affirmHasSyncAPI())) return;
SAHPool.isReady = SAHPool.reset().then(async ()=>{
if(0===SAHPool.getCapacity()){
await SAHPool.addCapacity(DEFAULT_CAPACITY);
@ -314,8 +514,7 @@ const installOpfsVfs = async function(sqlite3){
log("opfs-sahpool VFS initialized.");
promiseResolve(sqlite3);
}).catch(promiseReject);
})/*thePromise*/;
return thePromise;
})/*return Promise*/;
}/*installOpfsVfs()*/;
globalThis.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{

2
ext/wasm/tester1.c-pp.js
View File

@ -1288,7 +1288,7 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
if(1){
const vfsList = capi.sqlite3_js_vfs_list();
T.assert(vfsList.length>1);
log("vfsList =",vfsList);
//log("vfsList =",vfsList);
wasm.scopedAllocCall(()=>{
const vfsArg = (v)=>wasm.xWrap.testConvertArg('sqlite3_vfs*',v);
for(const v of vfsList){

16
manifest
View File

@ -1,5 +1,5 @@
C Switch\sopfs-sahpool\sto\suse\sdeterministic\sbacking-store\sfile\snames.\sDelay\sVFS\sregistration\suntil\safter\sthe\spool's\sfiles\sare\sall\sopened.\sFail\svfs\sinit\sif\sthe\sclient's\sOPFS\sAPI\shas\san\sasync\sFileSystemSyncAccessHandle.close()\smethod\s(indicating\sthat\sit's\soutdated).
D 2023-07-14T23:02:58.852
C More\swork\son\sporting\sthe\ssahpool\sbits.\sRevert\sto\srandom\sbacking-store\snames\sbecause\sit\sworks\sbetter\swith\sthe\scapacity-reduction\salgorithm.
D 2023-07-15T01:02:38.317
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -482,7 +482,7 @@ F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3
F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c
F ext/wasm/GNUmakefile b3d23a5f85dd9baff57c525a8fbd8f8ba2cdb41f2988fc93fb5e7746a276d8ab
F ext/wasm/GNUmakefile 1712a1fbf615dea49dfdc9cf9057c268df3a6887930aec8d9249f02a2059f2b2
F ext/wasm/README-dist.txt 6382cb9548076fca472fb3330bbdba3a55c1ea0b180ff9253f084f07ff383576
F ext/wasm/README.md ef39861aa21632fdbca0bdd469f78f0096f6449a720f3f39642594af503030e9
F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api d6a5078f48a5301ed17b9a30331075d9b2506e1360c1f0dee0c7816c10acd9ab
@ -502,7 +502,7 @@ F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b17386
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 961bbc3ccc1fa4e91d6519a96e8811ad7ae60173bd969fee7775dacb6eee1da2
F ext/wasm/api/sqlite3-v-helper.js e5c202a9ecde9ef818536d3f5faf26c03a1a9f5192b1ddea8bdabf30d75ef487
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js a54fee5e4112dce167bbcddd496f7817846cc269f859237460a835fe693b9c39
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.js 54beb56c4cc403c4bc9cf38704186e330b38fe6f24a86516a21d9d6239769b05
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 891f3a18d9ac9b0422b32fd975319dfcd0af5a8ca392f0cce850524e51b49c87
F ext/wasm/api/sqlite3-wasm.c 12a096d8e58a0af0589142bae5a3c27a0c7e19846755a1a37d2c206352fbedda
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
@ -549,7 +549,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
F ext/wasm/tester1.c-pp.js c021334b946f657251b2cc2cde11b2ff6c4f059fd5a0f4f07ea2d7564409c2cd
F ext/wasm/tester1.c-pp.js 4420eb97b6b4fc79e4e156b4b8010dd9f373365f4230dd76d823fb04ce28ffde
F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@ -2044,8 +2044,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ef96e6b586825a2b3ed011174309cba8ce1031876c86dc59ed87ab9bbc64f57f
R 1c797ce49382346884871d0f3bb17cf9
P 7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e
R ce448f7649c77f7ac682168b6f7b78d5
U stephan
Z de96eab25d9db47795591a43ac723ba9
Z 9bbeb2300d5ae5fa66a5962c456126f8
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
7c6697ededee9a64962ac6fd78934c6d6e39258b9558a03c1a6c02bf3be1759e
b4e005fd38b06b8d2f2317b955b93807e80a6a18db5f06d7747978d3bfa11411