mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add batch-runner.js for running batch SQL scripts with timing info.

Browse Source 
FossilOrigin-Name: 11f3ed61150c5940da6c157e5063e70c3aa0628dfd0023c47bb65b00af74ab1f
This commit is contained in:
stephan 2022-08-29 12:39:34 +00:00
parent f2e624ea91
commit 2cae138fba
6 changed files with 333 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ext/wasm/GNUmakefile
View File

@ -194,15 +194,17 @@ ifneq (0,$(ENABLE_WASMFS))
emcc.jsflags += -pthread -sWASMFS -sPTHREAD_POOL_SIZE=2
emcc.cflags += '-DSQLITE_DEFAULT_UNIX_VFS="unix-none"'
emcc.environment := $(emcc.environment),worker
emcc.jsflags += -sINITIAL_MEMORY=128450560
else
emcc.jsflags += -sALLOW_MEMORY_GROWTH
# emcc: warning: USE_PTHREADS + ALLOW_MEMORY_GROWTH may run non-wasm code
# slowly, see https://github.com/WebAssembly/design/issues/1271
# [-Wpthreads-mem-growth]
emcc.jsflags += -sINITIAL_MEMORY=13107200
#emcc.jsflags += -sINITIAL_MEMORY=64225280
# ^^^^ 64MB is not enough for WASMFS/OPFS test runs using batch-runner.js
endif
emcc.jsflags += $(emcc.environment)
#emcc.jsflags += -sINITIAL_MEMORY=13107200
emcc.jsflags += -sINITIAL_MEMORY=64225280
#emcc.jsflags += -sTOTAL_STACK=4194304
emcc.jsflags += -sEXPORT_NAME=sqlite3InitModule
emcc.jsflags += -sGLOBAL_BASE=4096 # HYPOTHETICALLY keep func table indexes from overlapping w/ heap addr.
@ -285,6 +287,18 @@ all: $(sqlite3.js)
# End main Emscripten-based module build
########################################################################
########################################################################
# Bits for use with batch-runner.js...
speedtest1 := ../../speedtest1
$(speedtest1):
$(MAKE) -C ../.. speedtest1
speedtest1.sql: $(speedtest1)
$(speedtest1) --script $@
batch-sql.in := $(sort $(wildcard *.sql))
batch-runner.list: $(batch-sql.in) $(MAKEFILE) speedtest1.sql
bash split-speedtest1-script.sh speedtest1.sql
ls -1 *.sql | sort > $@
batch: batch-runner.list
########################################################################
# fiddle_remote is the remote destination for the fiddle app. It

59
ext/wasm/batch-runner.html Normal file
View File

@ -0,0 +1,59 @@
<!doctype html>
<html lang="en-us">
<head>
<meta charset="utf-8">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<link rel="shortcut icon" href="data:image/x-icon;," type="image/x-icon">
<link rel="stylesheet" href="common/emscripten.css"/>
<link rel="stylesheet" href="common/testing.css"/>
<title>sqlite3-api batch SQL runner</title>
</head>
<body>
<header id='titlebar'><span>sqlite3-api batch SQL runner</span></header>
<!-- emscripten bits -->
<figure id="module-spinner">
<div class="spinner"></div>
<div class='center'><strong>Initializing app...</strong></div>
<div class='center'>
On a slow internet connection this may take a moment. If this
message displays for "a long time", intialization may have
failed and the JavaScript console may contain clues as to why.
</div>
</figure>
<div class="emscripten" id="module-status">Downloading...</div>
<div class="emscripten">
<progress value="0" max="100" id="module-progress" hidden='1'></progress>
</div><!-- /emscripten bits -->
<p class='warning'>ACHTUNG: this file requires a generated input list
file. Run "make batch" from this directory to generate it.
</p>
<p class='warning'>WARNING: if the WASMFS/OPFS layer crashes, this page may
become completely unresponsive and need to be closed and
reloaded to recover.
</p>
<hr>
<div>
<select id='sql-select'>
<option disabled selected>Populated via script code</option>
</select>
<button id='sql-run'>Run selected SQL</button>
<button id='output-clear'>Clear output</button>
</div>
<hr>
<div id='test-output'></div>
<script src="sqlite3.js"></script>
<script src="common/SqliteTestUtil.js"></script>
<script src="batch-runner.js"></script>
<style>
.warning { color: firebrick; }
#test-output {
border: 1px inset;
padding: 0.25em;
max-height: 20em;
overflow: auto;
white-space: break-spaces;
}
</style>
</body>
</html>

237
ext/wasm/batch-runner.js Normal file
View File

@ -0,0 +1,237 @@
/*
2022-08-29
The author disclaims copyright to this source code. In place of a
legal notice, here is a blessing:
* May you do good and not evil.
* May you find forgiveness for yourself and forgive others.
* May you share freely, never taking more than you give.
***********************************************************************
A basic batch SQL running for sqlite3-api.js. This file must be run in
main JS thread and sqlite3.js must have been loaded before it.
*/
'use strict';
(function(){
const T = self.SqliteTestUtil;
const toss = function(...args){throw new Error(args.join(' '))};
const debug = console.debug.bind(console);
const App = {
e: {
output: document.querySelector('#test-output'),
selSql: document.querySelector('#sql-select'),
btnRun: document.querySelector('#sql-run'),
btnClear: document.querySelector('#output-clear')
},
log: console.log.bind(console),
warn: console.warn.bind(console),
cls: function(){this.e.output.innerHTML = ''},
logHtml2: function(cssClass,...args){
const ln = document.createElement('div');
if(cssClass) ln.classList.add(cssClass);
ln.append(document.createTextNode(args.join(' ')));
this.e.output.append(ln);
},
logHtml: function(...args){
console.log(...args);
if(1) this.logHtml2('', ...args);
},
logErr: function(...args){
console.error(...args);
if(1) this.logHtml2('error', ...args);
},
openDb: function(fn){
if(this.pDb){
toss("Already have an opened db.");
}
const capi = this.sqlite3.capi, wasm = capi.wasm;
const stack = wasm.scopedAllocPush();
let pDb = 0;
try{
const oFlags = capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE;
const ppDb = wasm.scopedAllocPtr();
const rc = capi.sqlite3_open_v2(fn, ppDb, oFlags, null);
pDb = wasm.getPtrValue(ppDb)
}finally{
wasm.scopedAllocPop(stack);
}
this.logHtml("Opened db:",capi.sqlite3_db_filename(pDb, 'main'));
return this.pDb = pDb;
},
closeDb: function(){
if(this.pDb){
this.sqlite3.capi.sqlite3_close_v2(this.pDb);
this.pDb = undefined;
}
},
loadSqlList: async function(){
const sel = this.e.selSql;
sel.innerHTML = '';
this.blockControls(true);
const infile = 'batch-runner.list';
this.logHtml("Loading list of SQL files:", infile);
let txt;
try{
const r = await fetch(infile);
if(404 === r.status){
toss("Missing file '"+infile+"'.");
}
if(!r.ok) toss("Loading",infile,"failed:",r.statusText);
txt = await r.text();
}catch(e){
this.logErr(e.message);
throw e;
}finally{
this.blockControls(false);
}
const list = txt.split('\n');
let opt;
if(0){
opt = document.createElement('option');
opt.innerText = "Select file to evaluate...";
opt.value = '';
opt.disabled = true;
opt.selected = true;
sel.appendChild(opt);
}
list.forEach(function(fn){
opt = document.createElement('option');
opt.value = opt.innerText = fn;
sel.appendChild(opt);
});
this.logHtml("Loaded",infile);
},
/** Fetch ./fn and return its contents as a Uint8Array. */
fetchFile: async function(fn){
this.logHtml("Fetching",fn,"...");
let sql;
try {
const r = await fetch(fn);
if(!r.ok) toss("Fetch failed:",r.statusText);
sql = new Uint8Array(await r.arrayBuffer());
}catch(e){
this.logErr(e.message);
throw e;
}
this.logHtml("Fetched",sql.length,"bytes from",fn);
return sql;
},
checkRc: function(rc){
if(rc){
toss("Prepare failed:",this.sqlite3.capi.sqlite3_errmsg(this.pDb));
}
},
blockControls: function(block){
[
this.e.selSql, this.e.btnRun, this.e.btnClear
].forEach((e)=>e.disabled = block);
},
/** Fetch ./fn and eval it as an SQL blob. */
evalFile: async function(fn){
const sql = await this.fetchFile(fn);
this.logHtml("Running",fn,'...');
const capi = this.sqlite3.capi, wasm = capi.wasm;
let pStmt = 0, pSqlBegin;
const stack = wasm.scopedAllocPush();
const metrics = Object.create(null);
metrics.prepTotal = metrics.stepTotal = 0;
metrics.stmtCount = 0;
this.blockControls(true);
// Use setTimeout() so that the above log messages run before the loop starts.
setTimeout((function(){
metrics.timeStart = performance.now();
try {
let t;
let sqlByteLen = sql.byteLength;
const [ppStmt, pzTail] = wasm.scopedAllocPtr(2);
pSqlBegin = wasm.alloc( sqlByteLen + 1/*SQL + NUL*/);
let pSql = pSqlBegin;
const pSqlEnd = pSqlBegin + sqlByteLen;
wasm.heap8().set(sql, pSql);
wasm.setMemValue(pSql + sqlByteLen, 0);
while(wasm.getMemValue(pSql,'i8')){
pStmt = 0;
wasm.setPtrValue(ppStmt, 0);
wasm.setPtrValue(pzTail, 0);
t = performance.now();
let rc = capi.sqlite3_prepare_v3(
this.pDb, pSql, sqlByteLen, 0, ppStmt, pzTail
);
metrics.prepTotal += performance.now() - t;
this.checkRc(rc);
++metrics.stmtCount;
pStmt = wasm.getPtrValue(ppStmt);
pSql = wasm.getPtrValue(pzTail);
sqlByteLen = pSqlEnd - pSql;
if(!pStmt) continue/*empty statement*/;
t = performance.now();
rc = capi.sqlite3_step(pStmt);
metrics.stepTotal += performance.now() - t;
switch(rc){
case capi.SQLITE_ROW:
case capi.SQLITE_DONE: break;
default: this.checkRc(rc); toss("Not reached.");
}
}
}catch(e){
this.logErr(e.message);
throw e;
}finally{
wasm.dealloc(pSqlBegin);
wasm.scopedAllocPop(stack);
this.blockControls(false);
}
metrics.timeEnd = performance.now();
metrics.timeTotal = (metrics.timeEnd - metrics.timeStart);
this.logHtml("Metrics:");//,JSON.stringify(metrics, undefined, ' '));
this.logHtml("prepare() count:",metrics.stmtCount);
this.logHtml("Time in prepare_v2():",metrics.prepTotal,"ms",
"("+(metrics.prepTotal / metrics.stmtCount),"ms per prepare())");
this.logHtml("Time in step():",metrics.stepTotal,"ms",
"("+(metrics.stepTotal / metrics.stmtCount),"ms per step())");
this.logHtml("Total runtime:",metrics.timeTotal,"ms");
this.logHtml("Overhead (time - prep - step):",
(metrics.timeTotal - metrics.prepTotal - metrics.stepTotal)+"ms");
}.bind(this)), 10);
},
run: function(sqlite3){
this.sqlite3 = sqlite3;
const capi = sqlite3.capi, wasm = capi.wasm;
this.logHtml("Loaded module:",capi.sqlite3_libversion(), capi.sqlite3_sourceid());
this.logHtml("WASM heap size =",wasm.heap8().length);
this.logHtml("WARNING: if the WASMFS/OPFS layer crashes, this page may",
"become unresponsive and need to be closed and ",
"reloaded to recover.");
const pDir = capi.sqlite3_web_persistent_dir();
const dbFile = pDir ? pDir+"/speedtest.db" : ":memory:";
if(pDir){
// We initially need a clean db file, so...
capi.sqlite3_wasm_vfs_unlink(dbFile);
}
this.openDb(dbFile);
this.loadSqlList();
const who = this;
this.e.btnClear.addEventListener('click', ()=>this.cls(), false);
this.e.btnRun.addEventListener('click', function(){
if(!who.e.selSql.value) return;
who.evalFile(who.e.selSql.value);
}, false);
}
}/*App*/;
self.sqlite3TestModule.initSqlite3().then(function(theEmccModule){
self._MODULE = theEmccModule /* this is only to facilitate testing from the console */;
App.run(theEmccModule.sqlite3);
});
})();

11
ext/wasm/split-speedtest1-script.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
# Expects $1 to be a (speedtest1 --script) output file. Output is a
# series of SQL files extracted from that file.
infile=${1:?arg = speedtest1 --script output file}
testnums=$(grep -e '^-- begin test' "$infile" | cut -d' ' -f4)
#echo testnums=$testnums
for n in $testnums; do
ofile=$(printf "speedtest1-%03d.sql" $n)
sed -n -e "/^-- begin test $n\$/,/^-- end test $n\$/p" $infile > $ofile
echo -e "$n\t$ofile"
done

15
manifest
View File

@ -1,5 +1,5 @@
C Add\sget/setPtrValue()\sto\sthe\scommon\swasm\sutils.
D 2022-08-29T12:31:57.599
C Add\sbatch-runner.js\sfor\srunning\sbatch\sSQL\sscripts\swith\stiming\sinfo.
D 2022-08-29T12:39:34.874
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -474,7 +474,7 @@ F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb
F ext/wasm/EXPORTED_FUNCTIONS.fiddle db7a4602f043cf4a5e4135be3609a487f9f1c83f05778bfbdf93766be4541b96
F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle a004bd5eeeda6d3b28d16779b7f1a80305bfe009dfc7f0721b042967f0d39d02
F ext/wasm/GNUmakefile 12b4cefa1653df172ac54400f83b626b4969b4007da2ce523ebc27c8726656b0
F ext/wasm/GNUmakefile 61c0d9dcda9ca174907f87770c9440b26408dc415d2e11a3cbcae017358fd08c
F ext/wasm/README.md e1ee1e7c321c6a250bf78a84ca6f5882890a237a450ba5a0649c7a8399194c52
F ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api 77ef4bcf37e362b9ad61f9c175dfc0f1b3e571563fb311b96581cf422ee6a8ec
F ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api 1ec3c73e7d66e95529c3c64ac3de2470b0e9e7fbf7a5b41261c367cf4f1b7287
@ -489,6 +489,8 @@ F ext/wasm/api/sqlite3-api-prologue.js 2d5c5d3355f55eefe51922cec5bfedbec0f8300db
F ext/wasm/api/sqlite3-api-worker1.js 73579555563b789785ae83724014eaf31811073aad9be6596c8336ffb51edd71
F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9
F ext/wasm/api/sqlite3-wasm.c 0d81282eaeff2a6e9fc5c28a388c5c5b45cf25a9393992fa511ac009b27df982
F ext/wasm/batch-runner.html f9068c4b4222d0c11ba0590391892e4d7576ef1a4fb76974ba0bd3b80c6217f9
F ext/wasm/batch-runner.js 57f325e40812a89f8d47c918963f278d7a249d158cf0d245d75e74577af638c8
F ext/wasm/common/SqliteTestUtil.js eb96275bed43fdb364b7d65bcded0ca5e22aaacff120d593d1385f852f486247
F ext/wasm/common/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f
F ext/wasm/common/testing.css 572cf1ffae0b6eb7ca63684d3392bf350217a07b90e7a896e4fa850700c989b0
@ -508,6 +510,7 @@ F ext/wasm/scratchpad-opfs-main.js 69e960e9161f6412fd0c30f355d4112f1894d6609eb43
F ext/wasm/scratchpad-opfs-worker.html 66c1d15d678f3bd306373d76b61c6c8aef988f61f4a8dd40185d452f9c6d2bf5
F ext/wasm/scratchpad-opfs-worker.js 3ec2868c669713145c76eb5877c64a1b20741f741817b87c907a154b676283a9
F ext/wasm/scratchpad-opfs-worker2.js 5f2237427ac537b8580b1c659ff14ad2621d1694043eaaf41ae18dbfef2e48c0
F ext/wasm/split-speedtest1-script.sh dc187a66b692e21343a84767627e9e408dde504d9e92d4dfa63a2b43cedab292 x
F ext/wasm/sqlite3-worker1-promiser.js 92b8da5f38439ffec459a8215775d30fa498bc0f1ab929ff341fc3dd479660b9
F ext/wasm/sqlite3-worker1.js 0c1e7626304543969c3846573e080c082bf43bcaa47e87d416458af84f340a9e
F ext/wasm/testing-worker1-promiser.html 6eaec6e04a56cf24cf4fa8ef49d78ce8905dde1354235c9125dca6885f7ce893
@ -2009,8 +2012,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P ef0b7ef2d2e19b1f9437fdd7e24f040f662d9907d1fa17c6a3892fcf091b849e
R b442ab29e0d8d5bef6fe7f9423ce5596
P 24d70fc458c6002d5ff3c9f8ce7c66bde299b32aca6417c2dd1236e1412b036d
R ed1761d31b9a32171fc27afe1d89c1b7
U stephan
Z 5511f58972906137802924f203cb8dde
Z 4abbcdb5df49f5d67360636a18d94c2b
# Remove this line to create a well-formed Fossil manifest.

2
manifest.uuid
View File

@ -1 +1 @@
24d70fc458c6002d5ff3c9f8ce7c66bde299b32aca6417c2dd1236e1412b036d
11f3ed61150c5940da6c157e5063e70c3aa0628dfd0023c47bb65b00af74ab1f