Optimize sqlite3.oo1.DB.exec() for the rowMode='object' case to avoid converting the object property keys (column names) from native code to JS for each row. This speeds up large data sets considerably and addresses the report in [forum:3632183d2470617d|forum post 3632183d2470617d].
FossilOrigin-Name: 8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759
This commit is contained in:
stephan
parent
f895173c5f
commit
7fd9b53308
@ -433,40 +433,56 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
|
||||
out.returnVal = ()=>opt.resultRows;
|
||||
}
|
||||
if(opt.callback || opt.resultRows){
|
||||
switch((undefined===opt.rowMode)
|
||||
? 'array' : opt.rowMode) {
|
||||
case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break;
|
||||
case 'array': out.cbArg = (stmt)=>stmt.get([]); break;
|
||||
case 'stmt':
|
||||
if(Array.isArray(opt.resultRows)){
|
||||
toss3("exec(): invalid rowMode for a resultRows array: must",
|
||||
"be one of 'array', 'object',",
|
||||
"a result column number, or column name reference.");
|
||||
}
|
||||
out.cbArg = (stmt)=>stmt;
|
||||
switch((undefined===opt.rowMode) ? 'array' : opt.rowMode) {
|
||||
case 'object':
|
||||
out.cbArg = (stmt,cache)=>{
|
||||
if( !cache.columnNames ) cache.columnNames = stmt.getColumnNames([]);
|
||||
/* https://sqlite.org/forum/forumpost/3632183d2470617d:
|
||||
conversion of rows to objects (key/val pairs) is
|
||||
somewhat expensive for large data sets because of the
|
||||
native-to-JS conversion of the column names. If we
|
||||
instead cache the names and build objects from that
|
||||
list of strings, it can run twice as fast. The
|
||||
difference is not noticeable for small data sets but
|
||||
becomes human-perceivable when enough rows are
|
||||
involved. */
|
||||
const row = stmt.get([]);
|
||||
const rv = Object.create(null);
|
||||
for( const i in cache.columnNames ) rv[cache.columnNames[i]] = row[i];
|
||||
return rv;
|
||||
};
|
||||
break;
|
||||
case 'array': out.cbArg = (stmt)=>stmt.get([]); break;
|
||||
case 'stmt':
|
||||
if(Array.isArray(opt.resultRows)){
|
||||
toss3("exec(): invalid rowMode for a resultRows array: must",
|
||||
"be one of 'array', 'object',",
|
||||
"a result column number, or column name reference.");
|
||||
}
|
||||
out.cbArg = (stmt)=>stmt;
|
||||
break;
|
||||
default:
|
||||
if(util.isInt32(opt.rowMode)){
|
||||
out.cbArg = (stmt)=>stmt.get(opt.rowMode);
|
||||
break;
|
||||
default:
|
||||
if(util.isInt32(opt.rowMode)){
|
||||
out.cbArg = (stmt)=>stmt.get(opt.rowMode);
|
||||
break;
|
||||
}else if('string'===typeof opt.rowMode
|
||||
&& opt.rowMode.length>1
|
||||
&& '$'===opt.rowMode[0]){
|
||||
/* "$X": fetch column named "X" (case-sensitive!). Prior
|
||||
to 2022-12-14 ":X" and "@X" were also permitted, but
|
||||
having so many options is unnecessary and likely to
|
||||
cause confusion. */
|
||||
const $colName = opt.rowMode.substr(1);
|
||||
out.cbArg = (stmt)=>{
|
||||
const rc = stmt.get(Object.create(null))[$colName];
|
||||
return (undefined===rc)
|
||||
? toss3(capi.SQLITE_NOTFOUND,
|
||||
"exec(): unknown result column:",$colName)
|
||||
: rc;
|
||||
};
|
||||
break;
|
||||
}
|
||||
toss3("Invalid rowMode:",opt.rowMode);
|
||||
}else if('string'===typeof opt.rowMode
|
||||
&& opt.rowMode.length>1
|
||||
&& '$'===opt.rowMode[0]){
|
||||
/* "$X": fetch column named "X" (case-sensitive!). Prior
|
||||
to 2022-12-14 ":X" and "@X" were also permitted, but
|
||||
having so many options is unnecessary and likely to
|
||||
cause confusion. */
|
||||
const $colName = opt.rowMode.substr(1);
|
||||
out.cbArg = (stmt)=>{
|
||||
const rc = stmt.get(Object.create(null))[$colName];
|
||||
return (undefined===rc)
|
||||
? toss3(capi.SQLITE_NOTFOUND,
|
||||
"exec(): unknown result column:",$colName)
|
||||
: rc;
|
||||
};
|
||||
break;
|
||||
}
|
||||
toss3("Invalid rowMode:",opt.rowMode);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
@ -884,10 +900,15 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
|
||||
and names. */) ? 0 : 1;
|
||||
evalFirstResult = false;
|
||||
if(arg.cbArg || resultRows){
|
||||
const cbArgCache = Object.create(null)
|
||||
/* 2nd arg for arg.cbArg, used by (at least) row-to-object
|
||||
converter */;
|
||||
for(; stmt.step(); stmt._lockedByExec = false){
|
||||
if(0===gotColNames++) stmt.getColumnNames(opt.columnNames);
|
||||
if(0===gotColNames++){
|
||||
stmt.getColumnNames(cbArgCache.columnNames = (opt.columnNames || []));
|
||||
}
|
||||
stmt._lockedByExec = true;
|
||||
const row = arg.cbArg(stmt);
|
||||
const row = arg.cbArg(stmt,cbArgCache);
|
||||
if(resultRows) resultRows.push(row);
|
||||
if(callback && false === callback.call(opt, row, stmt)){
|
||||
break;
|
||||
|
||||
15
manifest
15
manifest
@ -1,5 +1,5 @@
|
||||
C Add\sthe\s"interstage-heuristic"\sthat\sattempts\sto\savoid\swildly\sinefficient\nqueries\sthat\suse\stable\sscans\sinstead\sof\sindex\slookups\sbecause\sthe\soutput\nrow\sestimates\sare\sinaccurate.
|
||||
D 2024-04-04T14:26:42.961
|
||||
C Optimize\ssqlite3.oo1.DB.exec()\sfor\sthe\srowMode='object'\scase\sto\savoid\sconverting\sthe\sobject\sproperty\skeys\s(column\snames)\sfrom\snative\scode\sto\sJS\sfor\seach\srow.\sThis\sspeeds\sup\slarge\sdata\ssets\sconsiderably\sand\saddresses\sthe\sreport\sin\s[forum:3632183d2470617d|forum\spost\s3632183d2470617d].
|
||||
D 2024-04-04T22:53:09.385
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -606,7 +606,7 @@ F ext/wasm/api/post-js-header.js 04dc12c3edd666b64a1b4ef3b6690c88dcc653f26451fd4
|
||||
F ext/wasm/api/pre-js.c-pp.js ad906703f7429590f2fbf5e6498513bf727a1a4f0ebfa057afb08161d7511219
|
||||
F ext/wasm/api/sqlite3-api-cleanup.js d235ad237df6954145404305040991c72ef8b1881715d2a650dda7b3c2576d0e
|
||||
F ext/wasm/api/sqlite3-api-glue.js 2d35660c52dcb4bb16d00c56553d34e7caa6ad30083938b515e6f9aa0b312fbb
|
||||
F ext/wasm/api/sqlite3-api-oo1.js 7f3bcf0549ac44cde4b9da0b642d771916738d3f6781fb8a1757c50a91e506c0
|
||||
F ext/wasm/api/sqlite3-api-oo1.js 365b3ae01a461dc974796823652ef1ecb1a9fac5df295ee1a78002cc77afb0d8
|
||||
F ext/wasm/api/sqlite3-api-prologue.js 93a72b07b2a5d964d2edc76a90b439ece49298bd7ba60a1c6ae5d4878213701e
|
||||
F ext/wasm/api/sqlite3-api-worker1.js 8d9c0562831f62218170a3373468d8a0b7a6503b5985e309b69bf71187b525cf
|
||||
F ext/wasm/api/sqlite3-license-version-header.js 0c807a421f0187e778dc1078f10d2994b915123c1223fe752b60afdcd1263f89
|
||||
@ -2184,9 +2184,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 797cda7ddcceb140330d58892c3e73d28df72b638df00fd48f07dfcba7706c5f 186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e
|
||||
R 869b12f296c2e2b0f215f1adb6cf5942
|
||||
T +closed 186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e
|
||||
U drh
|
||||
Z 93e4543ac83acbe7df90dc3896cd40af
|
||||
P 7bf49e2c54c9f6f336416f01c0e76aaf70f1e2f3fd612232e5a33ae5dabe0900
|
||||
R 32cab23d7481e1a388944c536dd694cd
|
||||
U stephan
|
||||
Z 96ae2cb4464d7637da1ca6fd97c9b197
|
||||
# Remove this line to create a well-formed Fossil manifest.
|
||||
|
||||
@ -1 +1 @@
|
||||
7bf49e2c54c9f6f336416f01c0e76aaf70f1e2f3fd612232e5a33ae5dabe0900
|
||||
8b41ef8690001eb299f5b7182c28f5318333bff5b505e1d59d6e6f4556b1c759
|
||||
Loading…
x
Reference in New Issue
Block a user