Update testrunner.tcl so that it can run the test suites defined in permutation.test.
FossilOrigin-Name: 15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa
This commit is contained in:
dan
parent
e834484d89
commit
ca0720a9d8
15
manifest
15
manifest
@ -1,5 +1,5 @@
|
||||
C When\sapplying\sthe\somit-ORDER-BY\soptimization,\sdefer\sdeleting\sthe\sAST\sof\nthe\sdeleted\sORDER\sBY\sclause\suntil\safter\scode\sgeneration\sends.
|
||||
D 2022-07-14T01:48:27.069
|
||||
C Update\stestrunner.tcl\sso\sthat\sit\scan\srun\sthe\stest\ssuites\sdefined\sin\spermutation.test.
|
||||
D 2022-07-14T18:09:56.687
|
||||
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
|
||||
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
|
||||
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
|
||||
@ -1507,8 +1507,7 @@ F test/temptable2.test d2940417496e2b9548e01d09990763fbe88c316504033256d51493e1f
|
||||
F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
|
||||
F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
|
||||
F test/tester.tcl 76771269dcc20b2c2d1d6f1175dd50d1eebddc004aebac865483f1829a5cd398
|
||||
F test/testrunner.tcl da340a87fd8b5c94e0d2196b086b96a90de0ce16bfdb71c1254d543bcee1614a
|
||||
F test/testset.tcl 27a6bbbc93cbbcf442c57e3c023e6b5d7304dc415e09eb0e9ac61edd0e6c1fbe
|
||||
F test/testrunner.tcl 711ccfb6309e63877b20910f76e8b5c8c9e45176cce0efd6f8a40b4e4d44d0e9
|
||||
F test/thread001.test b61a29dd87cf669f5f6ac96124a7c97d71b0c80d9012746072055877055cf9ef
|
||||
F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58
|
||||
F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7
|
||||
@ -1981,8 +1980,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
|
||||
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
|
||||
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
|
||||
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
|
||||
P 14918f28221a3124b78a490fbb483279551ccc5a0032ea854ff0ac365684cc60
|
||||
R d2c6af8b43cf81f08214974216426964
|
||||
U drh
|
||||
Z 663440ae7ea06bca4d8f20ca5b3783d1
|
||||
P f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba
|
||||
R f0a96981c3415711d35db8d014ac10fa
|
||||
U dan
|
||||
Z 26ecdc9b9c72535e31216f99411f29ef
|
||||
# Remove this line to create a well-formed Fossil manifest.
|
||||
|
||||
@ -1 +1 @@
|
||||
f22f95b838873f1d2a320afe3d0f4e4847948fcd343097b93a9f684a6f66d6ba
|
||||
15ce937ef42491c503cb91f5bb3ce4dc5cdb3a927ff2f2b873fb6bf96808f3aa
|
||||
@ -1,21 +1,47 @@
|
||||
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
# Usage:
|
||||
#
|
||||
proc usage {} {
|
||||
set a0 testrunner.tcl
|
||||
|
||||
puts stderr "Usage: $a0 ?SWITCHES? ?all|veryquick? ?PATTERNS?"
|
||||
set ::argv [list]
|
||||
uplevel [list source $::testdir/permutations.test]
|
||||
|
||||
puts stderr "Usage: $a0 ?SWITCHES? ?PERMUTATION? ?PATTERNS?"
|
||||
puts stderr ""
|
||||
puts stderr "where SWITCHES are:"
|
||||
puts stderr " --jobs NUMBER-OF-JOBS"
|
||||
puts stderr ""
|
||||
puts stderr "available PERMUTATION values are:"
|
||||
set ii 0
|
||||
foreach name [lsort [array names ::testspec]] {
|
||||
if {($ii % 3)==0} { puts -nonewline stderr " " }
|
||||
puts -nonewline stderr [format "% -22s" $name]
|
||||
if {($ii % 3)==2} { puts stderr "" }
|
||||
incr ii
|
||||
}
|
||||
puts stderr ""
|
||||
puts stderr ""
|
||||
puts stderr "Examples:"
|
||||
puts stderr " $a0 # Run veryquick.test tests"
|
||||
puts stderr " $a0 all # Run all tests"
|
||||
puts stderr " $a0 veryquick rtree% # Run all test scripts from veryquick.test that match 'rtree%'"
|
||||
puts stderr " $a0 alter% fts5% # Run all test scripts that match 'alter%' or 'rtree%'"
|
||||
puts stderr " 1) Run the veryquick tests:"
|
||||
puts stderr " $a0"
|
||||
puts stderr " 2) Run all test scripts in the source tree:"
|
||||
puts stderr " $a0 full"
|
||||
puts stderr " 2) Run the 'memsubsys1' permutation:"
|
||||
puts stderr " $a0 memsubsys1"
|
||||
puts stderr " 3) Run all permutations usually run by \[make fulltest\]"
|
||||
puts stderr " $a0 release"
|
||||
puts stderr " 4) Run all scripts that match the pattern 'select%':"
|
||||
puts stderr " $a0 select%"
|
||||
puts stderr " $a0 all select%"
|
||||
puts stderr " $a0 full select%"
|
||||
puts stderr " 5) Run all scripts that are part of the veryquick permutation and match the pattern 'select%':"
|
||||
puts stderr " $a0 veryquick select%"
|
||||
puts stderr " 6) Run the 'memsubsys1' permutation, but just those scripts that match 'window%':"
|
||||
puts stderr " $a0 memsubsys1 window%"
|
||||
puts stderr " 7) Run all the permutations, but only the scripts that match either 'fts5%' or 'rtree%':"
|
||||
puts stderr " $a0 release fts5% rtree%"
|
||||
|
||||
exit 1
|
||||
}
|
||||
@ -30,13 +56,15 @@ set R(schema) {
|
||||
DROP TABLE IF EXISTS malloc;
|
||||
|
||||
CREATE TABLE script(
|
||||
filename TEXT PRIMARY KEY, -- full path to test script
|
||||
config TEXT,
|
||||
filename TEXT, -- full path to test script
|
||||
state TEXT CHECK( state IN ('ready', 'running', 'done') ),
|
||||
testfixtureid, -- Id of process that ran script
|
||||
time INTEGER, -- Time in ms
|
||||
nerr INTEGER, -- if 'done', the number of errors
|
||||
ntest INTEGER, -- if 'done', the number of tests
|
||||
output TEXT -- full output of test script
|
||||
output TEXT, -- full output of test script
|
||||
PRIMARY KEY(config, filename)
|
||||
);
|
||||
|
||||
CREATE TABLE malloc(
|
||||
@ -95,7 +123,6 @@ set R(leaker) "" ;# Name of first script to leak memory
|
||||
set R(patternlist) [list]
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
source $testdir/testset.tcl
|
||||
|
||||
# Parse the command line options. There are two ways to invoke this
|
||||
# script - to create a helper or coordinator process. If there are
|
||||
@ -140,15 +167,86 @@ if {$R(helper)==0} {
|
||||
usage
|
||||
}
|
||||
} else {
|
||||
lappend R(patternlist) [string map {% * _ .} $a]
|
||||
lappend R(patternlist) [string map {% *} $a]
|
||||
}
|
||||
}
|
||||
|
||||
set argv [list]
|
||||
}
|
||||
source $testdir/permutations.test
|
||||
|
||||
source $testdir/tester.tcl
|
||||
db close
|
||||
#-------------------------------------------------------------------------
|
||||
# Return a list of tests to run. Each element of the list is itself a
|
||||
# list of two elements - the name of a permuations.test configuration
|
||||
# followed by the full path to a test script. i.e.:
|
||||
#
|
||||
# {CONFIG FILENAME} {CONFIG FILENAME} ...
|
||||
#
|
||||
proc testset_patternlist {patternlist} {
|
||||
|
||||
set first [lindex $patternlist 0]
|
||||
if {$first=="all"} { set first "full" }
|
||||
|
||||
if {$first=="release"} {
|
||||
|
||||
# The following mirrors the set of test suites invoked by "all.test".
|
||||
#
|
||||
set clist {
|
||||
full
|
||||
no_optimization memsubsys1 memsubsys2 singlethread
|
||||
multithread onefile utf16 exclusive persistent_journal
|
||||
persistent_journal_error no_journal no_journal_error
|
||||
autovacuum_ioerr no_mutex_try fullmutex journaltest
|
||||
inmemory_journal pcache0 pcache10 pcache50 pcache90
|
||||
pcache100 prepare mmap
|
||||
}
|
||||
ifcapable rbu { lappend clist rbu }
|
||||
if {$::tcl_platform(platform)=="unix"} {
|
||||
ifcapable !default_autovacuum {
|
||||
lappend clist autovacuum_crash
|
||||
}
|
||||
}
|
||||
set patternlist [lrange $patternlist 1 end]
|
||||
|
||||
} elseif {[info exists ::testspec($first)]} {
|
||||
set clist $first
|
||||
set patternlist [lrange $patternlist 1 end]
|
||||
} elseif { [llength $patternlist]==0 } {
|
||||
set clist veryquick
|
||||
} else {
|
||||
set clist full
|
||||
}
|
||||
|
||||
set testset [list]
|
||||
|
||||
foreach config $clist {
|
||||
catch { array unset O }
|
||||
array set O $::testspec($config)
|
||||
foreach f $O(-files) {
|
||||
if {[file pathtype $f]!="absolute"} {
|
||||
set f [file join $::testdir $f]
|
||||
}
|
||||
lappend testset [list $config [file normalize $f]]
|
||||
}
|
||||
}
|
||||
|
||||
if {[llength $patternlist]>0} {
|
||||
foreach t $testset {
|
||||
set tail [file tail [lindex $t 1]]
|
||||
foreach p $patternlist {
|
||||
if {[string match $p $tail]} {
|
||||
lappend ret $t
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
set ret $testset
|
||||
}
|
||||
|
||||
set ret
|
||||
}
|
||||
#--------------------------------------------------------------------------
|
||||
|
||||
|
||||
proc r_write_db {tcl} {
|
||||
@ -166,37 +264,63 @@ proc r_write_db {tcl} {
|
||||
proc make_new_testset {} {
|
||||
global R
|
||||
|
||||
set scripts [testset_patternlist $R(patternlist)]
|
||||
set tests [testset_patternlist $R(patternlist)]
|
||||
r_write_db {
|
||||
db eval $R(schema)
|
||||
foreach s $scripts {
|
||||
db eval { INSERT INTO script(filename, state) VALUES ($s, 'ready') }
|
||||
foreach t $tests {
|
||||
foreach {c s} $t {}
|
||||
db eval {
|
||||
INSERT INTO script(config, filename, state) VALUES ($c, $s, 'ready')
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Find the next job in the database and mark it as 'running'. Then return
|
||||
# a list consisting of the
|
||||
#
|
||||
# CONFIG FILENAME
|
||||
#
|
||||
# pair for the test.
|
||||
#
|
||||
proc get_next_test {} {
|
||||
global R
|
||||
set myid $R(helper_id)
|
||||
|
||||
r_write_db {
|
||||
set f [db one {
|
||||
SELECT filename FROM script WHERE state='ready' ORDER BY 1 LIMIT 1
|
||||
}]
|
||||
set f ""
|
||||
set c ""
|
||||
db eval {
|
||||
SELECT config, filename FROM script WHERE state='ready'
|
||||
ORDER BY config!='full', config, filename LIMIT 1
|
||||
} {
|
||||
set c $config
|
||||
set f $filename
|
||||
}
|
||||
if {$f!=""} {
|
||||
db eval {
|
||||
UPDATE script SET state='running', testfixtureid=$myid WHERE filename=$f
|
||||
UPDATE script SET state='running', testfixtureid=$myid
|
||||
WHERE (config, filename) = ($c, $f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $f
|
||||
if {$f==""} { return "" }
|
||||
list $c $f
|
||||
}
|
||||
|
||||
proc r_set_test_result {filename ms nerr ntest output} {
|
||||
proc r_testname {config filename} {
|
||||
set name [file tail $filename]
|
||||
if {$config!="" && $config!="full" && $config!="veryquick"} {
|
||||
set name "$config-$name"
|
||||
}
|
||||
return $name
|
||||
}
|
||||
|
||||
proc r_set_test_result {config filename ms nerr ntest output} {
|
||||
global R
|
||||
|
||||
set f [file tail $filename]
|
||||
set f [r_testname $config $filename]
|
||||
if {$nerr==0} {
|
||||
set msg "$f... Ok"
|
||||
} else {
|
||||
@ -211,14 +335,14 @@ proc r_set_test_result {filename ms nerr ntest output} {
|
||||
set nMalloc [lindex [sqlite3_status SQLITE_STATUS_MALLOC_COUNT 0] 1]
|
||||
set nByte [sqlite3_memory_used]
|
||||
if {($nByte>0 || $nMalloc>0) && $R(leaker)==""} {
|
||||
set R(leaker) $filename
|
||||
set R(leaker) $f
|
||||
}
|
||||
|
||||
r_write_db {
|
||||
db eval {
|
||||
UPDATE script
|
||||
SET state='done', output=$output, nerr=$nerr, ntest=$ntest, time=$ms
|
||||
WHERE filename=$filename;
|
||||
WHERE (config, filename)=($config, $filename);
|
||||
|
||||
INSERT INTO msg(msg) VALUES ($msg);
|
||||
}
|
||||
@ -408,7 +532,7 @@ proc r_final_report {} {
|
||||
# Create the text log file. This is just the concatenation of the
|
||||
# 'output' column of the database for every script that was run.
|
||||
set fd [open $R(logname) w]
|
||||
db eval {SELECT output FROM script ORDER BY filename} {
|
||||
db eval {SELECT output FROM script ORDER BY config!='full',config,filename} {
|
||||
puts $fd $output
|
||||
}
|
||||
close $fd
|
||||
@ -421,8 +545,8 @@ proc r_final_report {} {
|
||||
db eval { SELECT sum(nerr) AS nerr, sum(ntest) AS ntest FROM script } { }
|
||||
puts "$nerr errors from $ntest tests."
|
||||
if {$nerr>0} {
|
||||
db eval { SELECT filename FROM script WHERE nerr>0 } {
|
||||
lappend errlist [file tail $filename]
|
||||
db eval { SELECT config, filename FROM script WHERE nerr>0 } {
|
||||
lappend errlist [r_testname $config $filename]
|
||||
}
|
||||
puts "Errors in: $errlist"
|
||||
set errcode 1
|
||||
@ -431,8 +555,8 @@ proc r_final_report {} {
|
||||
# Check if any scripts were not run or did not finish. Print out a
|
||||
# line identifying them if there are any.
|
||||
set errlist [list]
|
||||
db eval { SELECT filename FROM script WHERE state!='done' } {
|
||||
lappend errlist [file tail $filename]
|
||||
db eval { SELECT config, filename FROM script WHERE state!='done' } {
|
||||
lappend errlist [r_testname $config $filename]
|
||||
}
|
||||
if {$errlist!=[list]} {
|
||||
puts "Tests DID NOT FINISH (crashed?): $errlist"
|
||||
@ -498,13 +622,31 @@ proc r_helper_readable {id chan} {
|
||||
}
|
||||
|
||||
if {$R(nHelperRunning)==0} {
|
||||
while { ""!=[set f [get_next_test]] } {
|
||||
while { ""!=[set t [get_next_test]] } {
|
||||
set R(output) ""
|
||||
set TC(count) 0
|
||||
set TC(errors) 0
|
||||
set ms [slave_test_file $f]
|
||||
|
||||
r_set_test_result $f $ms $TC(errors) $TC(count) $R(output)
|
||||
foreach {config filename} $t {}
|
||||
|
||||
array set O $::testspec($config)
|
||||
set ::G(perm:name) $config
|
||||
set ::G(perm:prefix) $O(-prefix)
|
||||
set ::G(isquick) 1
|
||||
set ::G(perm:dbconfig) $O(-dbconfig)
|
||||
set ::G(perm:presql) $O(-presql)
|
||||
|
||||
eval $O(-initialize)
|
||||
set ms [slave_test_file $filename]
|
||||
eval $O(-shutdown)
|
||||
|
||||
unset -nocomplain ::G(perm:sqlite3_args)
|
||||
unset ::G(perm:name)
|
||||
unset ::G(perm:prefix)
|
||||
unset ::G(perm:dbconfig)
|
||||
unset ::G(perm:presql)
|
||||
|
||||
r_set_test_result $config $filename $ms $TC(errors) $TC(count) $R(output)
|
||||
|
||||
if {$R(helper)==0} {
|
||||
foreach msg [r_get_messages] { puts $msg }
|
||||
|
||||
121
test/testset.tcl
121
test/testset.tcl
@ -1,121 +0,0 @@
|
||||
|
||||
|
||||
# Commands in this file:
|
||||
#
|
||||
# testset_all
|
||||
# Return a list of all test scripts designed to be run individually.
|
||||
#
|
||||
# testset_veryquick
|
||||
# The subset of [testset_all] meant to run as veryquick.test.
|
||||
#
|
||||
|
||||
set D(testdir) [file dir [file normalize [info script]]]
|
||||
|
||||
proc testset_all {} {
|
||||
global D
|
||||
set ret [list]
|
||||
|
||||
# The following tests are driver scripts that themselves run lots of other
|
||||
# test scripts. They should be ignored here.
|
||||
set drivers {
|
||||
all.test async.test quick.test veryquick.test
|
||||
memleak.test permutations.test soak.test fts3.test
|
||||
mallocAll.test rtree.test full.test extraquick.test
|
||||
session.test rbu.test
|
||||
}
|
||||
|
||||
set srcdir [file dirname $D(testdir)]
|
||||
set ret [glob -nocomplain \
|
||||
$srcdir/test/*.test \
|
||||
$srcdir/ext/rtree/*.test \
|
||||
$srcdir/ext/fts5/test/*.test \
|
||||
$srcdir/ext/expert/*.test \
|
||||
$srcdir/ext/session/*.test \
|
||||
]
|
||||
set ret [ts_filter $ret $drivers]
|
||||
return $ret
|
||||
}
|
||||
|
||||
proc testset_veryquick {} {
|
||||
set ret [testset_all]
|
||||
|
||||
set ret [ts_filter $ret {
|
||||
async2.test async3.test backup_ioerr.test corrupt.test
|
||||
corruptC.test crash.test crash2.test crash3.test crash4.test crash5.test
|
||||
crash6.test crash7.test delete3.test e_fts3.test fts3rnd.test
|
||||
fkey_malloc.test fuzz.test fuzz3.test fuzz_malloc.test in2.test loadext.test
|
||||
misc7.test mutex2.test notify2.test onefile.test pagerfault2.test
|
||||
savepoint4.test savepoint6.test select9.test
|
||||
speed1.test speed1p.test speed2.test speed3.test speed4.test
|
||||
speed4p.test sqllimits1.test tkt2686.test thread001.test thread002.test
|
||||
thread003.test thread004.test thread005.test trans2.test vacuum3.test
|
||||
incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test
|
||||
vtab_err.test walslow.test walcrash.test walcrash3.test
|
||||
walthread.test rtree3.test indexfault.test securedel2.test
|
||||
sort3.test sort4.test fts4growth.test fts4growth2.test
|
||||
bigsort.test walprotocol.test mmap4.test fuzzer2.test
|
||||
walcrash2.test e_fkey.test backup.test
|
||||
|
||||
fts4merge.test fts4merge2.test fts4merge4.test fts4check.test
|
||||
fts4merge5.test
|
||||
fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test
|
||||
fts3defer.test fts4langid.test fts3sort.test fts5unicode.test
|
||||
|
||||
rtree4.test
|
||||
sessionbig.test
|
||||
}]
|
||||
|
||||
set ret [ts_filter $ret {
|
||||
*malloc* *ioerr* *fault* *bigfile* *_err* *fts5corrupt* *fts5big* *fts5aj*
|
||||
}]
|
||||
|
||||
return $ret
|
||||
}
|
||||
|
||||
proc ts_filter {input exlist} {
|
||||
foreach f $input { set a($f) 1 }
|
||||
foreach e $exlist { array unset a */$e }
|
||||
array names a
|
||||
}
|
||||
|
||||
proc testset_patternlist {patternlist} {
|
||||
set nPat [llength $patternlist]
|
||||
|
||||
if {$nPat==0} {
|
||||
set scripts [testset_veryquick]
|
||||
} else {
|
||||
set ii 0
|
||||
set p0 [lindex $patternlist 0]
|
||||
|
||||
if {$p0=="veryquick"} {
|
||||
set scripts [testset_veryquick]
|
||||
incr ii
|
||||
} elseif {$p0=="all"} {
|
||||
set scripts [testset_all]
|
||||
incr ii
|
||||
} else {
|
||||
set scripts [testset_all]
|
||||
}
|
||||
|
||||
if {$nPat>$ii} {
|
||||
array set S [list]
|
||||
foreach f $scripts { set a([file tail $f]) $f }
|
||||
|
||||
foreach p [lrange $patternlist $ii end] {
|
||||
set nList [llength [array names a $p]]
|
||||
if {$nList==0} {
|
||||
puts stderr "Argument $p matches no scripts (typo?)"
|
||||
exit 1
|
||||
}
|
||||
foreach n [array names a $p] { set S($a($n)) 1 }
|
||||
}
|
||||
|
||||
set scripts [lsort [array names S]]
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
set scripts
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user