mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extend the importDb() method of both OPFS VFSes to (A) support reading in an async streaming fashion via a callback and (B) automatically disable WAL mode in the imported db.

Browse Source 
FossilOrigin-Name: 9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2
This commit is contained in:
stephan 2023-08-18 14:16:26 +00:00
parent abfe646c12
commit ccbfe97cd5
6 changed files with 223 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
ext/wasm/api/sqlite3-api-prologue.js
View File

@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
isSharedTypedArray,
toss: function(...args){throw new Error(args.join(' '))},
toss3,
typedArrayPart
};
typedArrayPart,
/**
Given a byte array or ArrayBuffer, this function throws if the
lead bytes of that buffer do not hold a SQLite3 database header,
else it returns without side effects.
Added in 3.44.
*/
affirmDbHeader: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const header = "SQLite format 3";
if( header.length > bytes.byteLength ){
toss3("Input does not contain an SQLite3 database header.");
}
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss3("Input does not contain an SQLite3 database header.");
}
}
},
/**
Given a byte array or ArrayBuffer, this function throws if the
database does not, at a cursory glance, appear to be an SQLite3
database. It only examines the size and header, but further
checks may be added in the future.
Added in 3.44.
*/
affirmIsDb: function(bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!==0) {
toss3("Byte array size",n,"is invalid for an SQLite3 db.");
}
util.affirmDbHeader(bytes);
}
}/*util*/;
Object.assign(wasm, {
/**

61
ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js
View File

@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
const toss3 = sqlite3.util.toss3;
const initPromises = Object.create(null);
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
// Config opts for the VFS...
const SECTOR_SIZE = 4096;
@ -869,9 +870,48 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
return b;
}
//! Impl for importDb() when its 2nd arg is a function.
async importDbChunked(name, callback){
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
sah.truncate(0);
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: HEADER_OFFSET_DATA + nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array(2), {
at: HEADER_OFFSET_DATA + 18
}/*force db out of WAL mode*/);
}catch(e){
this.setAssociatedPath(sah, '', 0);
}
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
return nWrote;
}
//! Documented elsewhere in this file.
importDb(name, bytes){
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
@ -882,16 +922,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
toss("Input does not contain an SQLite database header.");
}
}
const sah = this.#mapFilenameToSAH.get(name)
|| this.nextAvailableSAH()
|| toss("No available handles to import to.");
const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
if(nWrote != n){
this.setAssociatedPath(sah, '', 0);
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}else{
sah.write(new Uint8Array([0,0]), {at: HEADER_OFFSET_DATA+18}
/* force db out of WAL mode */);
this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
}
return nWrote;
}
}/*class OpfsSAHPool*/;
@ -1098,6 +1138,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
automatically clean up any non-database files so importing them
is pointless.
If passed a function for its second argument, its behavior
changes to asynchronous and it imports its data in chunks fed to
it by the given callback function. It calls the callback (which
may be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. The result of the resolved Promise when
called this way is the size of the resulting database.
On succes this routine rewrites the database header bytes in the
output file (not the input array) to force disabling of WAL mode.
On a write error, the handle is removed from the pool and made
available for re-use.

95
ext/wasm/api/sqlite3-vfs-opfs.c-pp.js
View File

@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
const error = (...args)=>logImpl(0, ...args);
const toss = sqlite3.util.toss;
const capi = sqlite3.capi;
const util = sqlite3.util;
const wasm = sqlite3.wasm;
const sqlite3_vfs = capi.sqlite3_vfs;
const sqlite3_file = capi.sqlite3_file;
@ -1168,40 +1169,98 @@ const installOpfsVfs = function callee(options){
doDir(opt.directory, 0);
};
/**
impl of importDb() when it's given a function as its second
argument.
*/
const importDbChunked = async function(filename, callback){
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
const sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
let nWrote = 0, chunk, checkedHeader = false, err = false;
try{
while( undefined !== (chunk = await callback()) ){
if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
if( 0===nWrote && chunk.byteLength>=15 ){
util.affirmDbHeader(chunk);
checkedHeader = true;
}
sah.write(chunk, {at: nWrote});
nWrote += chunk.byteLength;
}
if( nWrote < 512 || 0!==nWrote % 512 ){
toss("Input size",nWrote,"is not correct for an SQLite database.");
}
if( !checkedHeader ){
const header = new Uint8Array(20);
sah.read( header, {at: 0} );
util.affirmDbHeader( header );
}
sah.write(new Uint8Array(2), {at: 18}/*force db out of WAL mode*/);
return nWrote;
}catch(e){
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally {
await sah.close();
}
};
/**
Asynchronously imports the given bytes (a byte array or
ArrayBuffer) into the given database file.
If passed a function for its second argument, its behaviour
changes to async and it imports its data in chunks fed to it by
the given callback function. It calls the callback (which may
be async) repeatedly, expecting either a Uint8Array or
ArrayBuffer (to denote new input) or undefined (to denote
EOF). For so long as the callback continues to return
non-undefined, it will append incoming data to the given
VFS-hosted database file. When called this way, the resolved
value of the returned Promise is the number of bytes written to
the target file.
It very specifically requires the input to be an SQLite3
database and throws if that's not the case. It does so in
order to prevent this function from taking on a larger scope
than it is specifically intended to. i.e. we do not want it to
become a convenience for importing arbitrary files into OPFS.
Throws on error. Resolves to the number of bytes written.
This routine rewrites the database header bytes in the output
file (not the input array) to force disabling of WAL mode.
On error this throws and the state of the input file is
undefined (it depends on where the exception was triggered).
On success, resolves to the number of bytes written.
*/
opfsUtil.importDb = async function(filename, bytes){
if( bytes instanceof Function ){
return importDbChunked(filename, bytes);
}
if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
util.affirmIsDb(bytes);
const n = bytes.byteLength;
if(n<512 || n%512!=0){
toss("Byte array size is invalid for an SQLite db.");
}
const header = "SQLite format 3";
for(let i = 0; i < header.length; ++i){
if( header.charCodeAt(i) !== bytes[i] ){
toss("Input does not contain an SQLite database header.");
}
}
const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
const sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
const nWrote = sah.write(bytes, {at: 0});
sah.close();
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
let sah, err, nWrote = 0;
try {
const hFile = await hDir.getFileHandle(fnamePart, {create:true});
sah = await hFile.createSyncAccessHandle();
sah.truncate(0);
nWrote = sah.write(bytes, {at: 0});
if(nWrote != n){
toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
}
sah.write(new Uint8Array(2), {at: 18}) /* force db out of WAL mode */;
return nWrote;
}catch(e){
await hDir.removeEntry( fnamePart ).catch(()=>{});
throw e;
}finally{
if( sah ) await sah.close();
}
return nWrote;
};
if(sqlite3.oo1){

45
ext/wasm/tester1.c-pp.js
View File

@ -2939,8 +2939,27 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
let db;
try {
const exp = this.opfsDbExport;
const filename = this.opfsDbFile;
delete this.opfsDbExport;
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(this.opfsDbFile, exp);
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, exp);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
db.close();
this.opfsUnlink(filename);
T.assert(!(await sqlite3.opfs.entryExists(filename)));
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 512, end = exp.byteLength;
const reader = async function(){
if(cursor >= exp.byteLength){
return undefined;
}
const rv = exp.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
this.opfsImportSize = await sqlite3.oo1.OpfsDb.importDb(filename, reader);
db = new sqlite3.oo1.OpfsDb(this.opfsDbFile);
T.assert(6 === db.selectValue('select count(*) from p')).
assert( this.opfsImportSize == exp.byteLength );
@ -3059,8 +3078,9 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
const dbytes = u1.exportFile(dbName);
T.assert(dbytes.length >= 4096);
const dbName2 = '/exported.db';
u1.importDb(dbName2, dbytes);
T.assert( 2 == u1.getFileCount() );
let nWrote = u1.importDb(dbName2, dbytes);
T.assert( 2 == u1.getFileCount() )
.assert( dbytes.byteLength == nWrote );
let db2 = new u1.OpfsSAHPoolDb(dbName2);
T.assert(db2 instanceof sqlite3.oo1.DB)
.assert(3 === db2.selectValue('select count(*) from t'));
@ -3069,6 +3089,25 @@ globalThis.sqlite3InitModule = sqlite3InitModule;
.assert(false === u1.unlink(dbName2))
.assert(1 === u1.getFileCount())
.assert(1 === u1.getFileNames().length);
// Try again with a function as an input source:
let cursor = 0;
const blockSize = 1024, end = dbytes.byteLength;
const reader = async function(){
if(cursor >= dbytes.byteLength){
return undefined;
}
const rv = dbytes.subarray(cursor, cursor+blockSize>end ? end : cursor+blockSize);
cursor += blockSize;
return rv;
};
nWrote = await u1.importDb(dbName2, reader);
T.assert( 2 == u1.getFileCount() );
db2 = new u1.OpfsSAHPoolDb(dbName2);
T.assert(db2 instanceof sqlite3.oo1.DB)
.assert(3 === db2.selectValue('select count(*) from t'));
db2.close();
T.assert(true === u1.unlink(dbName2))
.assert(dbytes.byteLength == nWrote);
}
T.assert(true === u1.unlink(dbName))

18
manifest
View File

@ -1,5 +1,5 @@
C Add\snote\sabout\sthe\scurrent\sthreading\slimitation\sto\sext/jni/README.md.\sNo\scode\schanges.
D 2023-08-15T13:01:20.690
C Extend\sthe\simportDb()\smethod\sof\sboth\sOPFS\sVFSes\sto\s(A)\ssupport\sreading\sin\san\sasync\sstreaming\sfashion\svia\sa\scallback\sand\s(B)\sautomatically\sdisable\sWAL\smode\sin\sthe\simported\sdb.
D 2023-08-18T14:16:26.669
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -543,13 +543,13 @@ F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057af
F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
F ext/wasm/api/sqlite3-api-glue.js b65e546568f1dfb35205b9792feb5146a6323d71b55cda58e2ed30def6dd52f3
F ext/wasm/api/sqlite3-api-oo1.js 9678dc4d9a5d39632b6ffe6ea94a023119260815bf32f265bf5f6c36c9516db8
F ext/wasm/api/sqlite3-api-prologue.js 5f283b096b98bfb1ee2f2201e7ff0489dff00e29e1030c30953bdb4f5b87f4bd
F ext/wasm/api/sqlite3-api-prologue.js ef6f67c5ea718490806e5e17d2644b8b2f6e6ba5284d23dc1fbfd14d401c1ab5
F ext/wasm/api/sqlite3-api-worker1.js 9f32af64df1a031071912eea7a201557fe39b1738645c0134562bb84e88e2fec
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
F ext/wasm/api/sqlite3-opfs-async-proxy.js 8cf8a897726f14071fae6be6648125162b256dfb4f96555b865dbb7a6b65e379
F ext/wasm/api/sqlite3-v-helper.js 7daa0eab0a513a25b05e9abae7b5beaaa39209b3ed12f86aeae9ef8d2719ed25
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js abb69b5e008961026bf5ff433d7116cb046359af92a5daf73208af2e7ac80ae7
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js e04fc2fda6a0200ef80efdbb4ddfa0254453558adb17ec3a230f93d2bf1d711c
F ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js 561463ac5380e4ccf1839a1922e6d7a5585660f32e3b9701a270b78cd35566cf
F ext/wasm/api/sqlite3-vfs-opfs.c-pp.js 656952a75c36d96e3248b03ae26d6a7f8d6ff31e66432c63e1c0bb021f1234ab
F ext/wasm/api/sqlite3-wasm.c d4d4c2b349b43b7b861e6d2994299630fb79e07573ea6b61e28e8071b7d16b61
F ext/wasm/api/sqlite3-worker1-promiser.c-pp.js bc06df0d599e625bde6a10a394e326dc68da9ff07fa5404354580f81566e591f
F ext/wasm/api/sqlite3-worker1.c-pp.js da509469755035e919c015deea41b4514b5e84c12a1332e6cc8d42cb2cc1fb75
@ -595,7 +595,7 @@ F ext/wasm/test-opfs-vfs.html 1f2d672f3f3fce810dfd48a8d56914aba22e45c6834e262555
F ext/wasm/test-opfs-vfs.js f09266873e1a34d9bdb6d3981ec8c9e382f31f215c9fd2f9016d2394b8ae9b7b
F ext/wasm/tester1-worker.html ebc4b820a128963afce328ecf63ab200bd923309eb939f4110510ab449e9814c
F ext/wasm/tester1.c-pp.html 1c1bc78b858af2019e663b1a31e76657b73dc24bede28ca92fbe917c3a972af2
F ext/wasm/tester1.c-pp.js 64eb0ee6e695d5638d0f758f31a0ca2231e627ca5d768de3d8b44f9f494de8d4
F ext/wasm/tester1.c-pp.js 9e0f4da49f02753a73a5f931bfb9b1458175518daa3fec40b5ebdc06c285539c
F ext/wasm/tests/opfs/concurrency/index.html 0802373d57034d51835ff6041cda438c7a982deea6079efd98098d3e42fbcbc1
F ext/wasm/tests/opfs/concurrency/test.js a98016113eaf71e81ddbf71655aa29b0fed9a8b79a3cdd3620d1658eb1cc9a5d
F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2
@ -2091,8 +2091,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 00ac653562a66aad3112ea322d08be68e05e6bf7413c814dd3f81bf850fcf43b
R 68ba6514d93093aa325f9f497a3147a9
P 653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
R 6b40a7c1cd4b822a33b7a897821b1648
U stephan
Z 5cbd8ac0238d9d299017246e54070fc0
Z b0064c5da7f02d0dcda913e568fa1924
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
653ed92dc39185cdedfab3ea518bc7ec2d2826120e5fa4cbdee3343301396184
9b1398c96a4fd0b59e65faa8d5c98de4129f0f0357732f12cb2f5c53a08acdc2