diff --git a/ext/ota/README.txt b/ext/ota/README.txt index 943d1822af..42ac29064c 100644 --- a/ext/ota/README.txt +++ b/ext/ota/README.txt @@ -5,21 +5,8 @@ User documentation is in sqlite3ota.h. SQLite Hacks ------------ -1) PRAGMA ota_mode: - This is a new flag pragma. If the flag is set: - - * INSERT/DELETE/UPDATE commands are prevented from updating any but the main - b-tree for each table (the PK index for WITHOUT ROWID tables or the - rowid b-tree for others). - - * The above statements do not check UNIQUE constraints - except those enforced - by the main b-tree. - - * All non-temporary triggers are disabled. - - -2) PRAGMA pager_ota_mode=1: +1) PRAGMA pager_ota_mode=1: This pragma sets a flag on the pager associated with the main database only. In a zipvfs system, this pragma is intercepted by zipvfs and the flag is set @@ -51,7 +38,7 @@ SQLite Hacks pager_ota_mode connections. If two or more such connections attempt to write simultaneously, the results are undefined. -3) PRAGMA pager_ota_mode=2: +2) PRAGMA pager_ota_mode=2: The pager_ota_mode pragma may also be set to 2 if the main database is open in WAL mode. This prevents SQLite from checkpointing the wal file as part @@ -60,14 +47,7 @@ SQLite Hacks The effects of setting pager_ota_mode=2 if the db is not in WAL mode are undefined. -4) sqlite3_index_writer() - - This new API function is used to create VMs that can insert or delete entries - from individual index b-trees within the database. The VMs apply affinities - and check that UNIQUE constraints are not violated before updating index - b-trees. - -5) sqlite3_ckpt_open/step/close() +3) sqlite3_ckpt_open/step/close() API for performing (and resuming) incremental checkpoints. diff --git a/ext/ota/ota1.test b/ext/ota/ota1.test index afb5fd5865..dec5f14a99 100644 --- a/ext/ota/ota1.test +++ b/ext/ota/ota1.test @@ -98,7 +98,7 @@ proc step_ota {target ota} { # proc step_ota_uri {target ota} { while 1 { - sqlite3ota ota file:$target?xyz=123 $ota + sqlite3ota ota file:$target?xyz=&abc=123 $ota set rc [ota step] ota close if {$rc != "SQLITE_OK"} break @@ -106,361 +106,376 @@ proc step_ota_uri {target ota} { set rc } -foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - } - 3 { - CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; - } - 4 { - CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - 5 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - 6 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b, a); - } - 7 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b, c); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(a, b, c, a, b, c); - } - - 8 { - CREATE TABLE t1(a PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b, c); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(a, b, c, a, b, c); - } - - 9 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); - CREATE INDEX i1 ON t1(b); - } - - 10 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b DESC); - } - - 11 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); - } - - 12 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; - } - - 13 { - CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; - } - - 14 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - - 15 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - } - - 16 { - CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b DESC, c, a); - } +foreach {tn3 create_vfs destroy_vfs} { + 1 {} {} + 2 { + sqlite3ota_create_vfs -default myota "" } { - reset_db - execsql $schema - - do_test 1.$tn2.$tn.1 { - create_ota1 ota.db - $cmd test.db ota.db - } {SQLITE_DONE} - - do_execsql_test 1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { - 1 2 3 - 2 two three - 3 {} 8.2 - } - do_execsql_test 1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { - 3 {} 8.2 - 1 2 3 - 2 two three - } - do_execsql_test 1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { - 1 2 3 - 3 {} 8.2 - 2 two three - } - - do_execsql_test 1.$tn2.$tn.5 { PRAGMA integrity_check } ok + sqlite3ota_destroy_vfs myota } -} - -#------------------------------------------------------------------------- -# Check that an OTA cannot be applied to a table that has no PK. -# -# UPDATE: At one point OTA required that all tables featured either -# explicit IPK columns or were declared WITHOUT ROWID. This has been -# relaxed so that external PRIMARY KEYs on tables with automatic rowids -# are now allowed. -# -# UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. -# However the input table must feature an "ota_rowid" column. -# -reset_db -create_ota1 ota.db -do_execsql_test 2.1 { CREATE TABLE t1(a, b, c) } -do_test 2.2 { - sqlite3ota ota test.db ota.db - ota step -} {SQLITE_ERROR} -do_test 2.3 { - list [catch { ota close } msg] $msg -} {1 {SQLITE_ERROR - table data_t1 requires ota_rowid column}} -reset_db -do_execsql_test 2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } -do_test 2.5 { - sqlite3ota ota test.db ota.db - ota step -} {SQLITE_OK} -do_test 2.6 { - list [catch { ota close } msg] $msg -} {0 SQLITE_OK} - -#------------------------------------------------------------------------- -# Check that if a UNIQUE constraint is violated the current and all -# subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA -# transaction is rolled back by the [ota close] that deletes the ota -# handle. -# -foreach {tn errcode errmsg schema} { - 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(3, 2, 1); - } - - 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); - INSERT INTO t1 VALUES(4, 2, 'three'); - } - - 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { - CREATE TABLE t1(a PRIMARY KEY, b, c); - INSERT INTO t1 VALUES(3, 2, 1); - } - - 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { - CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); - INSERT INTO t1 VALUES(4, 2, 'three'); - } - } { + + eval $create_vfs + + foreach {tn2 cmd} {1 run_ota 2 step_ota 3 step_ota_uri} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 5 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 6 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b, a); + } + 7 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 8 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 9 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); + CREATE INDEX i1 ON t1(b); + } + + 10 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC); + } + + 11 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); + } + + 12 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + } + + 13 { + CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; + } + + 14 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 15 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 16 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b DESC, c, a); + } + } { + reset_db + execsql $schema + + do_test $tn3.1.$tn2.$tn.1 { + create_ota1 ota.db + breakpoint + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { + 1 2 3 + 2 two three + 3 {} 8.2 + } + do_execsql_test $tn3.1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { + 3 {} 8.2 + 1 2 3 + 2 two three + } + do_execsql_test $tn3.1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { + 1 2 3 + 3 {} 8.2 + 2 two three + } + + do_execsql_test $tn3.1.$tn2.$tn.5 { PRAGMA integrity_check } ok + } + } + + #------------------------------------------------------------------------- + # Check that an OTA cannot be applied to a table that has no PK. + # + # UPDATE: At one point OTA required that all tables featured either + # explicit IPK columns or were declared WITHOUT ROWID. This has been + # relaxed so that external PRIMARY KEYs on tables with automatic rowids + # are now allowed. + # + # UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. + # However the input table must feature an "ota_rowid" column. + # reset_db - execsql $schema - set cksum [dbcksum db main] - - do_test 3.$tn.1 { - create_ota1 ota.db + create_ota1 ota.db + do_execsql_test $tn3.2.1 { CREATE TABLE t1(a, b, c) } + do_test $tn3.2.2 { sqlite3ota ota test.db ota.db - while {[set res [ota step]]=="SQLITE_OK"} {} - set res - } $errcode - - do_test 3.$tn.2 { ota step } $errcode - - do_test 3.$tn.3 { + ota step + } {SQLITE_ERROR} + do_test $tn3.2.3 { list [catch { ota close } msg] $msg - } [list 1 "$errcode - $errmsg"] + } {1 {SQLITE_ERROR - table data_t1 requires ota_rowid column}} + reset_db + do_execsql_test $tn3.2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } + do_test $tn3.2.5 { + sqlite3ota ota test.db ota.db + ota step + } {SQLITE_OK} + do_test $tn3.2.6 { + list [catch { ota close } msg] $msg + } {0 SQLITE_OK} - do_test 3.$tn.4 { dbcksum db main } $cksum -} + #------------------------------------------------------------------------- + # Check that if a UNIQUE constraint is violated the current and all + # subsequent [ota step] calls return SQLITE_CONSTRAINT. And that the OTA + # transaction is rolled back by the [ota close] that deletes the ota + # handle. + # + foreach {tn errcode errmsg schema} { + 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } -#------------------------------------------------------------------------- -# -foreach {tn2 cmd} {1 run_ota 2 step_ota} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - } - 3 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } - 4 { - CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } - 5 { - CREATE TABLE t1(a INT PRIMARY KEY, b, c); - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } + 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + + 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } - 6 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); - CREATE INDEX i1 ON t1(b DESC); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c DESC, b, c); - } - 7 { - CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; - CREATE INDEX i1 ON t1(b); - CREATE INDEX i2 ON t1(c, b); - CREATE INDEX i3 ON t1(c, b, c); - } } { reset_db execsql $schema - execsql { - INSERT INTO t1 VALUES(2, 'hello', 'world'); - INSERT INTO t1 VALUES(4, 'hello', 'planet'); - INSERT INTO t1 VALUES(6, 'hello', 'xyz'); - } - - do_test 4.$tn2.$tn.1 { - create_ota4 ota.db - $cmd test.db ota.db - } {SQLITE_DONE} - - do_execsql_test 4.$tn2.$tn.2 { - SELECT * FROM t1 ORDER BY a ASC; + set cksum [dbcksum db main] + + do_test $tn3.3.$tn.1 { + create_ota1 ota.db + sqlite3ota ota test.db ota.db + while {[set res [ota step]]=="SQLITE_OK"} {} + set res + } $errcode + + do_test $tn3.3.$tn.2 { ota step } $errcode + + do_test $tn3.3.$tn.3 { + list [catch { ota close } msg] $msg + } [list 1 "$errcode - $errmsg"] + + do_test $tn3.3.$tn.4 { dbcksum db main } $cksum + } + + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 4 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 5 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + + 6 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); + CREATE INDEX i1 ON t1(b DESC); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c DESC, b, c); + } + 7 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } } { - 1 2 3 - 3 8 9 - 6 hello xyz - } - - do_execsql_test 4.$tn2.$tn.3 { PRAGMA integrity_check } ok - } -} - -#------------------------------------------------------------------------- -# -foreach {tn2 cmd} {1 run_ota 2 step_ota} { - foreach {tn schema} { - 1 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); - } - 2 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - 3 { - CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - 4 { - CREATE TABLE t1(a PRIMARY KEY, b, c, d); - CREATE INDEX i1 ON t1(d); - CREATE INDEX i2 ON t1(d, c); - CREATE INDEX i3 ON t1(d, c, b); - CREATE INDEX i4 ON t1(b); - CREATE INDEX i5 ON t1(c); - CREATE INDEX i6 ON t1(c, b); - } - } { - reset_db - execsql $schema - execsql { - INSERT INTO t1 VALUES(1, 2, 3, 4); - INSERT INTO t1 VALUES(2, 5, 6, 7); - INSERT INTO t1 VALUES(3, 8, 9, 10); - } - - do_test 5.$tn2.$tn.1 { - create_ota5 ota.db - $cmd test.db ota.db - } {SQLITE_DONE} + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(2, 'hello', 'world'); + INSERT INTO t1 VALUES(4, 'hello', 'planet'); + INSERT INTO t1 VALUES(6, 'hello', 'xyz'); + } - do_execsql_test 5.$tn2.$tn.2 { - SELECT * FROM t1 ORDER BY a ASC; - } { - 1 2 3 5 - 2 5 10 5 - 3 11 9 10 + do_test $tn3.4.$tn2.$tn.1 { + create_ota4 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.4.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 + 3 8 9 + 6 hello xyz + } + + do_execsql_test $tn3.4.$tn2.$tn.3 { PRAGMA integrity_check } ok } - - do_execsql_test 5.$tn2.$tn.3 { PRAGMA integrity_check } ok } -} -#------------------------------------------------------------------------- -# Test some error cases: -# -# * A virtual table with no ota_rowid column. -# * A no-PK table with no ota_rowid column. -# * A PK table with an ota_rowid column. -# -ifcapable fts3 { - foreach {tn schema error} { - 1 { - CREATE TABLE t1(a, b); - CREATE TABLE ota.data_t1(a, b, ota_control); - } {SQLITE_ERROR - table data_t1 requires ota_rowid column} - - 2 { - CREATE VIRTUAL TABLE t1 USING fts4(a, b); - CREATE TABLE ota.data_t1(a, b, ota_control); - } {SQLITE_ERROR - table data_t1 requires ota_rowid column} - - 3 { - CREATE TABLE t1(a PRIMARY KEY, b); - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - 4 { - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - 5 { - CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; - CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); - } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} - - } { - reset_db - forcedelete ota.db - execsql { ATTACH 'ota.db' AS ota } - execsql $schema - - do_test 6.$tn { - list [catch { run_ota test.db ota.db } msg] $msg - } [list 1 $error] + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_ota 2 step_ota} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 5, 6, 7); + INSERT INTO t1 VALUES(3, 8, 9, 10); + } + + do_test $tn3.5.$tn2.$tn.1 { + create_ota5 ota.db + $cmd test.db ota.db + } {SQLITE_DONE} + + do_execsql_test $tn3.5.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 5 + 2 5 10 5 + 3 11 9 10 + } + + do_execsql_test $tn3.5.$tn2.$tn.3 { PRAGMA integrity_check } ok + } } + + #------------------------------------------------------------------------- + # Test some error cases: + # + # * A virtual table with no ota_rowid column. + # * A no-PK table with no ota_rowid column. + # * A PK table with an ota_rowid column. + # + ifcapable fts3 { + foreach {tn schema error} { + 1 { + CREATE TABLE t1(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 2 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b); + CREATE TABLE ota.data_t1(a, b, ota_control); + } {SQLITE_ERROR - table data_t1 requires ota_rowid column} + + 3 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 4 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + 5 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE ota.data_t1(a, b, ota_rowid, ota_control); + } {SQLITE_ERROR - table data_t1 may not have ota_rowid column} + + } { + reset_db + forcedelete ota.db + execsql { ATTACH 'ota.db' AS ota } + execsql $schema + + do_test $tn3.6.$tn { + list [catch { run_ota test.db ota.db } msg] $msg + } [list 1 $error] + } + } + + eval $destroy_vfs } diff --git a/ext/ota/ota2.test b/ext/ota/ota2.test deleted file mode 100644 index 3c273e348a..0000000000 --- a/ext/ota/ota2.test +++ /dev/null @@ -1,74 +0,0 @@ -# 2014 August 30 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl -set ::testprefix ota2 - -forcedelete {*}[glob -nocomplain test.db?*] - -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); -} {} -do_test 1.1 { glob test.db* } {test.db} - -do_execsql_test 1.2 { - PRAGMA pager_ota_mode = 1; - INSERT INTO t1 VALUES(3, 4); - INSERT INTO t1 VALUES(5, 6); - SELECT * FROM t1; -} {1 2 3 4 5 6} - -do_test 1.3 { lsort [glob test.db*] } {test.db test.db-oal} - -do_test 1.4 { - sqlite3 db2 test.db - db2 eval { SELECT * FROM t1 } -} {1 2} - -do_test 1.5 { - catchsql { INSERT INTO t1 VALUES(7, 8) } db2 -} {1 {database is locked}} - -db2 close -db close - -sqlite3 db test.db -do_execsql_test 1.6 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; -} {1 2 3 4 5 6} - -do_execsql_test 1.7 { - INSERT INTO t1 VALUES(7,8); - SELECT * FROM t1; -} {1 2 3 4 5 6 7 8} - -db close -sqlite3 db2 test.db - -do_test 1.8 { - execsql { BEGIN; SELECT * FROM t1 } db2 -} {1 2} -do_test 1.9 { - file rename test.db-oal test.db-wal - execsql { SELECT * FROM t1 } db2 -} {1 2} -do_test 1.10 { - execsql { COMMIT; SELECT * FROM t1 } db2 -} {1 2 3 4 5 6 7 8} - - -finish_test diff --git a/ext/ota/ota4.test b/ext/ota/ota4.test deleted file mode 100644 index a12cbd8b7b..0000000000 --- a/ext/ota/ota4.test +++ /dev/null @@ -1,128 +0,0 @@ -# 2014 August 30 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# -# Test some properties of the pager_ota_mode and ota_mode pragmas. -# - -if {![info exists testdir]} { - set testdir [file join [file dirname [info script]] .. .. test] -} -source $testdir/tester.tcl -set ::testprefix ota4 - -#------------------------------------------------------------------------- -# The following tests aim to verify some properties of the pager_ota_mode -# pragma: -# -# 1. Cannot set the pager_ota_mode flag on a WAL mode database. -# -# 2. Or if there is an open read transaction. -# -# 3. Cannot start a transaction with pager_ota_mode set if there -# is a WAL file in the file-system. -# -# 4. Or if the wal-mode flag is set in the database file header. -# -# 5. Cannot open a transaction with pager_ota_mode set if the database -# file has been modified by a rollback mode client since the *-oal -# file was started. -# - -do_execsql_test 1.1.1 { - PRAGMA journal_mode = wal; - SELECT * FROM sqlite_master; -} {wal} -do_catchsql_test 1.1.2 { - PRAGMA pager_ota_mode = 1 -} {1 {cannot set pager_ota_mode in wal mode}} - - -do_execsql_test 1.2.1 { - PRAGMA journal_mode = delete; - BEGIN; - SELECT * FROM sqlite_master; -} {delete} -do_catchsql_test 1.2.2 { - PRAGMA pager_ota_mode = 1 -} {1 {cannot set pager_ota_mode with open transaction}} -do_execsql_test 1.2.3 { - COMMIT; -} {} - - -do_execsql_test 1.3.1 { - PRAGMA journal_mode = wal; - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); -} {wal} -do_test 1.3.2 { - forcecopy test.db-wal test.db-bak - execsql { - PRAGMA journal_mode = delete; - PRAGMA pager_ota_mode = 1; - } - forcecopy test.db-bak test.db-wal - catchsql { - SELECT * FROM sqlite_master - } -} {1 {unable to open database file}} - -do_test 1.4.1 { - db close - forcedelete test.db-wal test.db-oal - sqlite3 db test.db - execsql { - PRAGMA journal_mode = wal; - PRAGMA pager_ota_mode = 1; - } - catchsql { - SELECT * FROM sqlite_master; - } -} {1 {unable to open database file}} - -do_test 1.5.1 { - forcedelete test.db-oal - reset_db - execsql { - PRAGMA journal_mode = delete; - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); - } - execsql { - PRAGMA pager_ota_mode = 1; - INSERT INTO t1 VALUES(3, 4); - } - db close - sqlite3 db test.db - execsql { - SELECT * FROM t1; - } -} {1 2} -do_execsql_test 1.5.2 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; - INSERT INTO t1 VALUES(5, 6); -} {1 2 3 4} -do_test 5.3 { - db close - sqlite3 db test.db - execsql { - INSERT INTO t1 VALUES(7, 8); - SELECT * FROM t1; - } -} {1 2 7 8} -do_catchsql_test 1.5.4 { - PRAGMA pager_ota_mode = 1; - SELECT * FROM t1; -} {1 {database is locked}} - -finish_test - diff --git a/ext/ota/ota6.test b/ext/ota/ota6.test index f15873629c..8027f36c5f 100644 --- a/ext/ota/ota6.test +++ b/ext/ota/ota6.test @@ -50,7 +50,7 @@ for {set nStep 1} {$nStep < 7} {incr nStep} { setup_test sqlite3ota ota test.db ota.db for {set i 0} {$i<$nStep} {incr i} {ota step} - + ota close sqlite3 db test.db execsql { INSERT INTO t1 VALUES(5, 'hello') } @@ -65,7 +65,7 @@ for {set nStep 1} {$nStep < 7} {incr nStep} { } {1 0} do_test 1.$nStep.4 { list [catch { ota close } msg] $msg - } {1 {SQLITE_BUSY - database is locked}} + } {1 {SQLITE_BUSY - database modified during ota update}} } for {set nStep 7} {$nStep < 8} {incr nStep} { @@ -122,7 +122,5 @@ for {set nStep 8} {$nStep < 20} {incr nStep} { } - finish_test - diff --git a/ext/ota/otaA.test b/ext/ota/otaA.test new file mode 100644 index 0000000000..9ba9606e30 --- /dev/null +++ b/ext/ota/otaA.test @@ -0,0 +1,82 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the OTA module. More specifically, it +# contains tests to ensure that it is an error to attempt to update +# a wal mode database via OTA. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +set ::testprefix otaA + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); +} +set ota_sql { + CREATE TABLE data_t1(a, b, c, ota_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.0 { + forcedelete test.db ota.db + + sqlite3 db test.db + db eval $db_sql + db eval { PRAGMA journal_mode = wal } + db close + + sqlite3 db ota.db + db eval $ota_sql + db close + + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} +do_test 1.1 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + +do_test 2.0 { + forcedelete test.db ota.db + + sqlite3 db test.db + db eval $db_sql + db close + + sqlite3 db ota.db + db eval $ota_sql + db close + + sqlite3ota ota test.db ota.db + ota step + ota close +} {SQLITE_OK} + +do_test 2.1 { + sqlite3 db test.db + db eval {PRAGMA journal_mode = wal} + db close + sqlite3ota ota test.db ota.db + ota step +} {SQLITE_ERROR} + +do_test 2.2 { + list [catch { ota close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + + +finish_test + diff --git a/ext/ota/sqlite3ota.c b/ext/ota/sqlite3ota.c index 20b9ede75b..de7f1250c5 100644 --- a/ext/ota/sqlite3ota.c +++ b/ext/ota/sqlite3ota.c @@ -62,6 +62,9 @@ ** Valid if STAGE==3. The blob to pass to sqlite3ckpt_start() to resume ** the incremental checkpoint. ** +** OTA_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. */ #define OTA_STATE_STAGE 1 #define OTA_STATE_TBL 2 @@ -69,9 +72,9 @@ #define OTA_STATE_ROW 4 #define OTA_STATE_PROGRESS 5 #define OTA_STATE_CKPT 6 +#define OTA_STATE_COOKIE 7 #define OTA_STAGE_OAL 1 -#define OTA_STAGE_COPY 2 #define OTA_STAGE_CKPT 3 #define OTA_STAGE_DONE 4 @@ -81,6 +84,8 @@ typedef struct OtaState OtaState; typedef struct OtaObjIter OtaObjIter; +typedef struct ota_vfs ota_vfs; +typedef struct ota_file ota_file; /* ** A structure to store values read from the ota_state table in memory. @@ -152,6 +157,7 @@ struct OtaObjIter { #define OTA_PK_WITHOUT_ROWID 4 #define OTA_PK_VTAB 5 + /* ** OTA handle. */ @@ -166,8 +172,37 @@ struct sqlite3ota { int nProgress; /* Rows processed for all objects */ OtaObjIter objiter; /* Iterator for skipping through tbl/idx */ sqlite3_ckpt *pCkpt; /* Incr-checkpoint handle */ + ota_file *pTargetFd; /* File handle open on target db */ + const char *zVfsName; /* Name of automatically created ota vfs */ }; +struct ota_vfs { + sqlite3_vfs base; /* ota VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; + const char *zOtaWal; +}; + +struct ota_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + ota_vfs *pOtaVfs; /* Pointer to the ota_vfs object */ + sqlite3ota *pOta; /* Pointer to ota object (ota target only) */ + + int openFlags; /* Flags this file was opened with */ + unsigned int iCookie; /* Cookie value for main db files */ + unsigned char iWriteVer; /* "write-version" value for main db files */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + const char *zWal; /* Wal filename for this db file */ + char *zDel; /* Delete this when closing file */ +}; + + +static void otaCreateVfs(sqlite3ota*, const char*); +static void otaDeleteVfs(sqlite3ota*); + /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return @@ -413,6 +448,20 @@ static int otaMPrintfExec(sqlite3ota *p, const char *zFmt, ...){ return p->rc; } +static void *otaMalloc(sqlite3ota *p, int nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + pRet = sqlite3_malloc(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + /* ** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that ** there is room for at least nCol elements. If an OOM occurs, store an @@ -422,17 +471,13 @@ static void otaAllocateIterArrays(sqlite3ota *p, OtaObjIter *pIter, int nCol){ int nByte = (2*sizeof(char*) + sizeof(int) + 2*sizeof(unsigned char)) * nCol; char **azNew; - assert( p->rc==SQLITE_OK ); - azNew = (char**)sqlite3_malloc(nByte); + azNew = (char**)otaMalloc(p, nByte); if( azNew ){ - memset(azNew, 0, nByte); pIter->azTblCol = azNew; pIter->azTblType = &azNew[nCol]; pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; pIter->abTblPk = (unsigned char*)&pIter->aiSrcOrder[nCol]; pIter->abNotNull = (unsigned char*)&pIter->abTblPk[nCol]; - }else{ - p->rc = SQLITE_NOMEM; } } @@ -969,17 +1014,14 @@ static char *otaObjIterGetSetlist( static char *otaObjIterGetBindlist(sqlite3ota *p, int nBind){ char *zRet = 0; - if( p->rc==SQLITE_OK ){ - int nByte = nBind*2 + 1; - zRet = sqlite3_malloc(nByte); - if( zRet==0 ){ - p->rc = SQLITE_NOMEM; - }else{ - int i; - for(i=0; irc==SQLITE_OK ); - sqlite3_close(p->db); - p->db = 0; + assert( p->db==0 ); - p->rc = sqlite3_open(p->zTarget, &p->db); + p->rc = sqlite3_open_v2(p->zTarget, &p->db, flags, p->zVfsName); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(p->db)); + }else{ + otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); + + /* Mark the database file just opened as an OTA target database. If + ** this call returns SQLITE_NOTFOUND, then the OTA vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->db, "main", SQLITE_FCNTL_OTA, (void*)p); + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("ota vfs not found"); + } + } } - otaMPrintfExec(p, "ATTACH %Q AS ota", p->zOta); } /* @@ -1506,8 +1560,10 @@ static void otaMoveOalFile(sqlite3ota *p){ /* Re-open the databases. */ otaObjIterFinalize(&p->objiter); - otaOpenDatabase(p); + sqlite3_close(p->db); + p->db = 0; p->eStage = OTA_STAGE_CKPT; + otaOpenDatabase(p); } sqlite3_free(zWal); @@ -1795,13 +1851,15 @@ static void otaSaveTransactionState(sqlite3ota *p){ "(%d, %Q), " "(%d, %d), " "(%d, %lld), " - "(%d, ?) ", + "(%d, ?), " + "(%d, %lld) ", OTA_STATE_STAGE, p->eStage, OTA_STATE_TBL, p->objiter.zTbl, OTA_STATE_IDX, p->objiter.zIdx, OTA_STATE_ROW, p->nStep, OTA_STATE_PROGRESS, p->nProgress, - OTA_STATE_CKPT + OTA_STATE_CKPT, + OTA_STATE_COOKIE, (sqlite3_int64)p->pTargetFd->iCookie ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -1848,7 +1906,7 @@ static void otaFreeState(OtaState *p){ static OtaState *otaLoadState(sqlite3ota *p){ const char *zSelect = "SELECT k, v FROM ota.ota_state"; OtaState *pRet = 0; - sqlite3_stmt *pStmt; + sqlite3_stmt *pStmt = 0; int rc; int rc2; @@ -1866,7 +1924,6 @@ static OtaState *otaLoadState(sqlite3ota *p){ case OTA_STATE_STAGE: pRet->eStage = sqlite3_column_int(pStmt, 1); if( pRet->eStage!=OTA_STAGE_OAL - && pRet->eStage!=OTA_STAGE_COPY && pRet->eStage!=OTA_STAGE_CKPT ){ p->rc = SQLITE_CORRUPT; @@ -1896,6 +1953,19 @@ static OtaState *otaLoadState(sqlite3ota *p){ ); break; + case OTA_STATE_COOKIE: + /* At this point (p->iCookie) contains the value of the change-counter + ** cookie (the thing that gets incremented when a transaction is + ** committed in rollback mode) currently stored on page 1 of the + ** database file. */ + if( pRet->eStage==OTA_STAGE_OAL + && p->pTargetFd->iCookie!=(unsigned int)sqlite3_column_int64(pStmt, 1) + ){ + rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during ota update"); + } + break; + default: rc = SQLITE_CORRUPT; break; @@ -1965,19 +2035,29 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p ){ OtaState *pState = 0; - /* Open the target database */ + /* Create the custom VFS */ memset(p, 0, sizeof(sqlite3ota)); - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zOta = &p->zTarget[nTarget+1]; - memcpy(p->zOta, zOta, nOta+1); - otaOpenDatabase(p); + otaCreateVfs(p, 0); + + /* Open the target database */ + if( p->rc==SQLITE_OK ){ + p->zTarget = (char*)&p[1]; + memcpy(p->zTarget, zTarget, nTarget+1); + p->zOta = &p->zTarget[nTarget+1]; + memcpy(p->zOta, zOta, nOta+1); + otaOpenDatabase(p); + } /* If it has not already been created, create the ota_state table */ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->db, OTA_CREATE_STATE, 0, 0, &p->zErrmsg); } + if( p->rc==SQLITE_OK && p->pTargetFd->iWriteVer>1 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot update wal mode database"); + } + if( p->rc==SQLITE_OK ){ pState = otaLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -1995,12 +2075,14 @@ sqlite3ota *sqlite3ota_open(const char *zTarget, const char *zOta){ if( p->rc==SQLITE_OK ){ if( p->eStage==OTA_STAGE_OAL ){ - const char *zScript = - "PRAGMA journal_mode=off;" - "PRAGMA pager_ota_mode=1;" - "BEGIN IMMEDIATE;" - ; - p->rc = sqlite3_exec(p->db, zScript, 0, 0, &p->zErrmsg); + ota_vfs *pOtaVfs = p->pTargetFd->pOtaVfs; + + sqlite3_mutex_enter(pOtaVfs->mutex); + assert( pOtaVfs->zOtaWal==0 ); + pOtaVfs->zOtaWal = p->pTargetFd->zWal; + p->rc = sqlite3_exec(p->db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + pOtaVfs->zOtaWal = 0; + sqlite3_mutex_leave(pOtaVfs->mutex); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -2081,13 +2163,10 @@ int sqlite3ota_close(sqlite3ota *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->db, "COMMIT", 0, 0, &p->zErrmsg); } - if( p->rc==SQLITE_OK && p->eStage==OTA_STAGE_CKPT ){ - p->rc = sqlite3_exec(p->db, "PRAGMA pager_ota_mode=2", 0, 0, &p->zErrmsg); - } - - /* Close the open database handle */ + /* Close the open database handle and VFS object. */ if( p->pCkpt ) sqlite3_ckpt_close(p->pCkpt, 0, 0); sqlite3_close(p->db); + otaDeleteVfs(p); otaEditErrmsg(p); rc = p->rc; @@ -2109,6 +2188,624 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){ return pOta->nProgress; } +/************************************************************************** +** Beginning of OTA VFS shim methods. The VFS shim modifies the behaviour +** of a standard VFS in the following ways: +** +** 1. Whenever the first page of a main database file is read or +** written, the value of the change-counter cookie is stored in +** ota_file.iCookie. Similarly, the value of the "write-version" +** database header field is stored in ota_file.iWriteVer. This ensures +** that the values are always trustworthy within an open transaction. +** +** 2. When the ota handle is in OTA_STAGE_OAL or OTA_STAGE_CKPT state, all +** EXCLUSIVE lock attempts on the target database fail. This prevents +** sqlite3_close() from running an automatic checkpoint. Until the +** ota handle reaches OTA_STAGE_DONE - at that point the automatic +** checkpoint may be required to delete the *-wal file. +** +** 3. In OTA_STAGE_OAL, the *-shm file is stored in memory. All xShmLock() +** calls are noops. This is just an optimization. +** +** 4. In OTA_STAGE_OAL mode, when SQLite calls xAccess() to check if a +** *-wal file associated with the target database exists, the following +** special handling applies: +** +** a) if the *-wal file does exist, return SQLITE_CANTOPEN. An OTA +** target database may not be in wal mode already. +** +** b) if the *-wal file does not exist, set the output parameter to +** non-zero (to tell SQLite that it does exist) anyway. +** +** 5. In OTA_STAGE_OAL mode, if SQLite tries to open a *-wal file +** associated with a target database, open the corresponding *-oal file +** instead. +*/ + +/* +** Close an ota file. +*/ +static int otaVfsClose(sqlite3_file *pFile){ + ota_file *p = (ota_file*)pFile; + int rc; + int i; + + /* Free the contents of the apShm[] array. And the array itself. */ + for(i=0; inShm; i++){ + sqlite3_free(p->apShm[i]); + } + sqlite3_free(p->apShm); + p->apShm = 0; + sqlite3_free(p->zDel); + + /* Close the underlying file handle */ + rc = p->pReal->pMethods->xClose(p->pReal); + return rc; +} + + +/* +** Read and return an unsigned 32-bit big-endian integer from the buffer +** passed as the only argument. +*/ +static unsigned int otaGetU32(unsigned char *aBuf){ + return ((unsigned int)aBuf[0] << 24) + + ((unsigned int)aBuf[1] << 16) + + ((unsigned int)aBuf[2] << 8) + + ((unsigned int)aBuf[3]); +} + +/* +** Read data from an otaVfs-file. +*/ +static int otaVfsRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ota_file *p = (ota_file*)pFile; + int rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ + /* These look like magic numbers. But they are stable, as they are part + ** of the definition of the SQLite file format, which may not change. */ + unsigned char *pBuf = (unsigned char*)zBuf; + p->iCookie = otaGetU32(&pBuf[24]); + p->iWriteVer = pBuf[19]; + } + return rc; +} + +/* +** Write data to an otaVfs-file. +*/ +static int otaVfsWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ota_file *p = (ota_file*)pFile; + int rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ + /* These look like magic numbers. But they are stable, as they are part + ** of the definition of the SQLite file format, which may not change. */ + unsigned char *pBuf = (unsigned char*)zBuf; + p->iCookie = otaGetU32(&pBuf[24]); + p->iWriteVer = pBuf[19]; + } + return rc; +} + +/* +** Truncate an otaVfs-file. +*/ +static int otaVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ota_file *p = (ota_file*)pFile; + return p->pReal->pMethods->xTruncate(p->pReal, size); +} + +/* +** Sync an otaVfs-file. +*/ +static int otaVfsSync(sqlite3_file *pFile, int flags){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xSync(p->pReal, flags); +} + +/* +** Return the current file-size of an otaVfs-file. +*/ +static int otaVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xFileSize(p->pReal, pSize); +} + +/* +** Lock an otaVfs-file. +*/ +static int otaVfsLock(sqlite3_file *pFile, int eLock){ + ota_file *p = (ota_file*)pFile; + sqlite3ota *pOta = p->pOta; + int rc = SQLITE_OK; + + if( pOta && eLock==SQLITE_LOCK_EXCLUSIVE + && (pOta->eStage==OTA_STAGE_OAL || pOta->eStage==OTA_STAGE_CKPT) + ){ + /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this + ** prevents it from checkpointing the database from sqlite3_close(). */ + rc = SQLITE_BUSY; + }else{ + rc = p->pReal->pMethods->xLock(p->pReal, eLock); + } + + return rc; +} + +/* +** Unlock an otaVfs-file. +*/ +static int otaVfsUnlock(sqlite3_file *pFile, int eLock){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xUnlock(p->pReal, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an otaVfs-file. +*/ +static int otaVfsCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xCheckReservedLock(p->pReal, pResOut); +} + +/* +** File control method. For custom operations on an otaVfs-file. +*/ +static int otaVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ + ota_file *p = (ota_file *)pFile; + int (*xControl)(sqlite3_file*,int,void*) = p->pReal->pMethods->xFileControl; + + if( op==SQLITE_FCNTL_OTA ){ + int rc; + sqlite3ota *pOta = (sqlite3ota*)pArg; + + /* First try to find another OTA vfs lower down in the vfs stack. If + ** one is found, this vfs will operate in pass-through mode. The lower + ** level vfs will do the special OTA handling. */ + rc = xControl(p->pReal, op, pArg); + + if( rc==SQLITE_NOTFOUND ){ + /* Now search for a zipvfs instance lower down in the VFS stack. If + ** one is found, this is an error. */ + void *dummy = 0; + rc = xControl(p->pReal, SQLITE_FCNTL_ZIPVFS_PAGER, &dummy); + if( rc==SQLITE_OK ){ + rc = SQLITE_ERROR; + pOta->zErrmsg = sqlite3_mprintf("ota/zipvfs setup error"); + }else if( rc==SQLITE_NOTFOUND ){ + pOta->pTargetFd = p; + p->pOta = pOta; + rc = SQLITE_OK; + } + } + return rc; + } + return xControl(p->pReal, op, pArg); +} + +/* +** Return the sector-size in bytes for an otaVfs-file. +*/ +static int otaVfsSectorSize(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xSectorSize(p->pReal); +} + +/* +** Return the device characteristic flags supported by an otaVfs-file. +*/ +static int otaVfsDeviceCharacteristics(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + return p->pReal->pMethods->xDeviceCharacteristics(p->pReal); +} + +/* +** Shared-memory methods are all pass-thrus. +*/ +static int otaVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ + ota_file *p = (ota_file*)pFile; + int rc = SQLITE_OK; + +#ifdef SQLITE_AMALGAMATION + assert( WAL_CKPT_LOCK==1 ); +#endif + + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ + /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from + ** taking this lock also prevents any checkpoints from occurring. + ** todo: really, it's not clear why this might occur, as + ** wal_autocheckpoint ought to be turned off. */ + if( ofst==1 && n==1 ) rc = SQLITE_BUSY; + }else{ + assert( p->nShm==0 ); + rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); + } + + return rc; +} + +static int otaVfsShmMap( + sqlite3_file *pFile, + int iRegion, + int szRegion, + int isWrite, + void volatile **pp +){ + ota_file *p = (ota_file*)pFile; + int rc = SQLITE_OK; + + /* If not in OTA_STAGE_OAL, allow this call to pass through. Or, if this + ** ota is in the OTA_STAGE_OAL state, use heap memory for *-shm space + ** instead of a file on disk. */ + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ + if( iRegion<=p->nShm ){ + int nByte = (iRegion+1) * sizeof(char*); + char **apNew = (char**)sqlite3_realloc(p->apShm, nByte); + if( apNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); + p->apShm = apNew; + p->nShm = iRegion+1; + } + } + + if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ + char *pNew = (char*)sqlite3_malloc(szRegion); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, szRegion); + p->apShm[iRegion] = pNew; + } + } + + if( rc==SQLITE_OK ){ + *pp = p->apShm[iRegion]; + }else{ + *pp = 0; + } + }else{ + assert( p->apShm==0 ); + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp); + } + + return rc; +} + +/* +** Memory barrier. +*/ +static void otaVfsShmBarrier(sqlite3_file *pFile){ + ota_file *p = (ota_file *)pFile; + p->pReal->pMethods->xShmBarrier(p->pReal); +} + +static int otaVfsShmUnmap(sqlite3_file *pFile, int delFlag){ + ota_file *p = (ota_file*)pFile; + int rc = SQLITE_OK; + + if( p->pOta && p->pOta->eStage==OTA_STAGE_OAL ){ + /* no-op */ + }else{ + rc = p->pReal->pMethods->xShmUnmap(p->pReal, delFlag); + } + return rc; +} + +/* +** Open an ota file handle. +*/ +static int otaVfsOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + static sqlite3_io_methods otavfs_io_methods = { + 2, /* iVersion */ + otaVfsClose, /* xClose */ + otaVfsRead, /* xRead */ + otaVfsWrite, /* xWrite */ + otaVfsTruncate, /* xTruncate */ + otaVfsSync, /* xSync */ + otaVfsFileSize, /* xFileSize */ + otaVfsLock, /* xLock */ + otaVfsUnlock, /* xUnlock */ + otaVfsCheckReservedLock, /* xCheckReservedLock */ + otaVfsFileControl, /* xFileControl */ + otaVfsSectorSize, /* xSectorSize */ + otaVfsDeviceCharacteristics, /* xDeviceCharacteristics */ + otaVfsShmMap, /* xShmMap */ + otaVfsShmLock, /* xShmLock */ + otaVfsShmBarrier, /* xShmBarrier */ + otaVfsShmUnmap /* xShmUnmap */ + }; + ota_vfs *pOtaVfs = (ota_vfs*)pVfs; + sqlite3_vfs *pRealVfs = pOtaVfs->pRealVfs; + ota_file *pFd = (ota_file *)pFile; + int rc = SQLITE_OK; + const char *zOpen = zName; + + memset(pFd, 0, sizeof(ota_file)); + pFd->pReal = (sqlite3_file*)&pFd[1]; + pFd->pOtaVfs = pOtaVfs; + pFd->openFlags = flags; + if( zName ){ + if( flags & SQLITE_OPEN_MAIN_DB ){ + /* A main database has just been opened. The following block sets + ** (pFd->zWal) to point to a buffer owned by SQLite that contains + ** the name of the *-wal file this db connection will use. SQLite + ** happens to pass a pointer to this buffer when using xAccess() + ** or xOpen() to operate on the *-wal file. */ + int n = strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + pFd->zWal = z; + } + else if( (flags & SQLITE_OPEN_WAL) && zName==pOtaVfs->zOtaWal ){ + char *zCopy = otaStrndup(zName, -1, &rc); + if( zCopy ){ + int nCopy = strlen(zCopy); + zCopy[nCopy-3] = 'o'; + zOpen = (const char*)(pFd->zDel = zCopy); + } + } + } + + if( rc==SQLITE_OK ){ + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + } + if( pFd->pReal->pMethods ){ + pFile->pMethods = &otavfs_io_methods; + } + + return rc; +} + +/* +** Delete the file located at zPath. +*/ +static int otaVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDelete(pRealVfs, zPath, dirSync); +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int otaVfsAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + ota_vfs *pOtaVfs = (ota_vfs*)pVfs; + sqlite3_vfs *pRealVfs = pOtaVfs->pRealVfs; + int rc; + + rc = pRealVfs->xAccess(pRealVfs, zPath, flags, pResOut); + + if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS && pOtaVfs->zOtaWal==zPath ){ + if( *pResOut ){ + rc = SQLITE_CANTOPEN; + }else{ + *pResOut = 1; + } + } + + return rc; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (DEVSYM_MAX_PATHNAME+1) bytes. +*/ +static int otaVfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xFullPathname(pRealVfs, zPath, nOut, zOut); +} + +#ifndef SQLITE_OMIT_LOAD_EXTENSION +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *otaVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlOpen(pRealVfs, zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void otaVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + pRealVfs->xDlError(pRealVfs, nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*otaVfsDlSym( + sqlite3_vfs *pVfs, + void *pArg, + const char *zSym +))(void){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlSym(pRealVfs, pArg, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void otaVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xDlClose(pRealVfs, pHandle); +} +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int otaVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xRandomness(pRealVfs, nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int otaVfsSleep(sqlite3_vfs *pVfs, int nMicro){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xSleep(pRealVfs, nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int otaVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + sqlite3_vfs *pRealVfs = ((ota_vfs*)pVfs)->pRealVfs; + return pRealVfs->xCurrentTime(pRealVfs, pTimeOut); +} + +static int otaVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return 0; +} + +void sqlite3ota_destroy_vfs(const char *zName){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); + if( pVfs ){ + sqlite3_vfs_unregister(pVfs); + sqlite3_free(pVfs); + } +} + +int sqlite3ota_create_vfs(const char *zName, const char *zParent){ + + /* Template for VFS */ + static sqlite3_vfs vfs_template = { + 1, /* iVersion */ + 0, /* szOsFile */ + 0, /* mxPathname */ + 0, /* pNext */ + 0, /* zName */ + 0, /* pAppData */ + otaVfsOpen, /* xOpen */ + otaVfsDelete, /* xDelete */ + otaVfsAccess, /* xAccess */ + otaVfsFullPathname, /* xFullPathname */ + + otaVfsDlOpen, /* xDlOpen */ + otaVfsDlError, /* xDlError */ + otaVfsDlSym, /* xDlSym */ + otaVfsDlClose, /* xDlClose */ + + otaVfsRandomness, /* xRandomness */ + otaVfsSleep, /* xSleep */ + otaVfsCurrentTime, /* xCurrentTime */ + otaVfsGetLastError, /* xGetLastError */ + 0, /* xCurrentTimeInt64 (version 2) */ + 0, 0, 0 /* Unimplemented version 3 methods */ + }; + + sqlite3_vfs *pParent; /* Parent VFS */ + ota_vfs *pNew = 0; /* Newly allocated VFS */ + int nName; + int rc = SQLITE_OK; + + nName = strlen(zName); + pParent = sqlite3_vfs_find(zParent); + if( pParent==0 ){ + rc = SQLITE_NOTFOUND; + }else{ + int nByte = sizeof(ota_vfs) + nName + 1; + pNew = (ota_vfs*)sqlite3_malloc(nByte); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, nByte); + } + } + + if( rc==SQLITE_OK ){ + char *zSpace; + memcpy(&pNew->base, &vfs_template, sizeof(sqlite3_vfs)); + pNew->base.mxPathname = pParent->mxPathname; + pNew->base.szOsFile = sizeof(ota_file) + pParent->szOsFile; + pNew->pRealVfs = pParent; + + pNew->base.zName = (const char*)(zSpace = (char*)&pNew[1]); + memcpy(zSpace, zName, nName); + + /* Register the new VFS (not as the default) */ + rc = sqlite3_vfs_register(&pNew->base, 0); + if( rc ){ + sqlite3_free(pNew); + } + } + + return rc; +} + +static void otaCreateVfs(sqlite3ota *p, const char *zParent){ + int rnd; + char zRnd[64]; + + assert( p->rc==SQLITE_OK ); + sqlite3_randomness(sizeof(int), (void*)&rnd); + sprintf(zRnd, "ota_vfs_%d", rnd); + p->rc = sqlite3ota_create_vfs(zRnd, zParent); + if( p->rc==SQLITE_NOTFOUND ){ + p->zErrmsg = sqlite3_mprintf("no such vfs: %s", zParent); + }else if( p->rc==SQLITE_OK ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd); + assert( pVfs ); + p->zVfsName = pVfs->zName; + } +} + +static void otaDeleteVfs(sqlite3ota *p){ + if( p->zVfsName ){ + sqlite3ota_destroy_vfs(p->zVfsName); + p->zVfsName = 0; + } +} + /**************************************************************************/ @@ -2235,9 +2932,76 @@ static int test_sqlite3ota( return TCL_OK; } +/* +** Tclcmd: sqlite3ota_create_vfs ?-default? NAME PARENT +*/ +static int test_sqlite3ota_create_vfs( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zName; + const char *zParent; + int rc; + + if( objc!=3 && objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-default? NAME PARENT"); + return TCL_ERROR; + } + + zName = Tcl_GetString(objv[objc-2]); + zParent = Tcl_GetString(objv[objc-1]); + if( zParent[0]=='\0' ) zParent = 0; + + rc = sqlite3ota_create_vfs(zName, zParent); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else if( objc==4 ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); + sqlite3_vfs_register(pVfs, 1); + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** Tclcmd: sqlite3ota_destroy_vfs NAME +*/ +static int test_sqlite3ota_destroy_vfs( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zName; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME"); + return TCL_ERROR; + } + + zName = Tcl_GetString(objv[1]); + sqlite3ota_destroy_vfs(zName); + return TCL_OK; +} + int SqliteOta_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3ota", test_sqlite3ota, 0, 0); + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aObjCmd[] = { + { "sqlite3ota", test_sqlite3ota }, + { "sqlite3ota_create_vfs", test_sqlite3ota_create_vfs }, + { "sqlite3ota_destroy_vfs", test_sqlite3ota_destroy_vfs }, + }; + int i; + for(i=0; ix.pSelect = pSelect; pInClause->flags |= EP_xIsSelect; - sqlite3ExprSetHeight(pParse, pInClause); + sqlite3ExprSetHeightAndFlags(pParse, pInClause); return pInClause; /* something went wrong. clean up anything allocated. */ diff --git a/src/expr.c b/src/expr.c index 25bd958ceb..5457a9c065 100644 --- a/src/expr.c +++ b/src/expr.c @@ -146,10 +146,25 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ break; } if( p->flags & EP_Collate ){ - if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){ + if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ p = p->pLeft; }else{ - p = p->pRight; + Expr *pNext = p->pRight; + /* The Expr.x union is never used at the same time as Expr.pRight */ + assert( p->x.pList==0 || p->pRight==0 ); + /* p->flags holds EP_Collate and p->pLeft->flags does not. And + ** p->x.pSelect cannot. So if p->x.pLeft exists, it must hold at + ** least one EP_Collate. Thus the following two ALWAYS. */ + if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){ + int i; + for(i=0; ALWAYS(ix.pList->nExpr); i++){ + if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ + pNext = p->x.pList->a[i].pExpr; + break; + } + } + } + p = pNext; } }else{ break; @@ -355,6 +370,9 @@ static void heightOfSelect(Select *p, int *pnHeight){ ** Expr.pSelect member has a height of 1. Any other expression ** has a height equal to the maximum height of any other ** referenced Expr plus one. +** +** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, +** if appropriate. */ static void exprSetHeight(Expr *p){ int nHeight = 0; @@ -362,8 +380,9 @@ static void exprSetHeight(Expr *p){ heightOfExpr(p->pRight, &nHeight); if( ExprHasProperty(p, EP_xIsSelect) ){ heightOfSelect(p->x.pSelect, &nHeight); - }else{ + }else if( p->x.pList ){ heightOfExprList(p->x.pList, &nHeight); + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); } p->nHeight = nHeight + 1; } @@ -372,8 +391,11 @@ static void exprSetHeight(Expr *p){ ** Set the Expr.nHeight variable using the exprSetHeight() function. If ** the height is greater than the maximum allowed expression depth, ** leave an error in pParse. +** +** Also propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. */ -void sqlite3ExprSetHeight(Parse *pParse, Expr *p){ +void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ exprSetHeight(p); sqlite3ExprCheckHeight(pParse, p->nHeight); } @@ -387,8 +409,17 @@ int sqlite3SelectExprHeight(Select *p){ heightOfSelect(p, &nHeight); return nHeight; } -#else - #define exprSetHeight(y) +#else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ +/* +** Propagate all EP_Propagate flags from the Expr.x.pList into +** Expr.flags. +*/ +void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ + if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); + } +} +#define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ /* @@ -490,11 +521,11 @@ void sqlite3ExprAttachSubtrees( }else{ if( pRight ){ pRoot->pRight = pRight; - pRoot->flags |= EP_Collate & pRight->flags; + pRoot->flags |= EP_Propagate & pRight->flags; } if( pLeft ){ pRoot->pLeft = pLeft; - pRoot->flags |= EP_Collate & pLeft->flags; + pRoot->flags |= EP_Propagate & pLeft->flags; } exprSetHeight(pRoot); } @@ -594,7 +625,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ } pNew->x.pList = pList; assert( !ExprHasProperty(pNew, EP_xIsSelect) ); - sqlite3ExprSetHeight(pParse, pNew); + sqlite3ExprSetHeightAndFlags(pParse, pNew); return pNew; } @@ -1209,6 +1240,21 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ sqlite3DbFree(db, pList); } +/* +** Return the bitwise-OR of all Expr.flags fields in the given +** ExprList. +*/ +u32 sqlite3ExprListFlags(const ExprList *pList){ + int i; + u32 m = 0; + if( pList ){ + for(i=0; inExpr; i++){ + m |= pList->a[i].pExpr->flags; + } + } + return m; +} + /* ** These routines are Walker callbacks used to check expressions to ** see if they are "constant" for some definition of constant. The @@ -1249,7 +1295,7 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ** and either pWalker->eCode==4 or 5 or the function has the ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){ return WRC_Continue; }else{ pWalker->eCode = 0; diff --git a/src/pager.c b/src/pager.c index 7cc35505f6..959f135b1d 100644 --- a/src/pager.c +++ b/src/pager.c @@ -615,18 +615,6 @@ struct PagerSavepoint { ** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode ** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. -** -** otaMode -** This variable is normally 0. It is set to 1 by the PagerSetOtaMode() -** function - as a result of a "PRAGMA pager_ota_mode=1" command. Once -** the *-oal file has been opened and it has been determined that the -** database file has not been modified since it was created, this variable -** is set to 2. -** -** It is also possible to use PagerSetOtaMode() to 2 if the database is -** already in WAL mode. In this case the only effect is to prevent the -** connection from checkpointing the db as part of an sqlite3PagerClose() -** call. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -642,9 +630,6 @@ struct Pager { u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ -#ifdef SQLITE_ENABLE_OTA - u8 otaMode; /* Non-zero if in ota_mode */ -#endif /************************************************************************** ** The following block contains those class members that change during @@ -720,16 +705,6 @@ struct Pager { #endif }; -/* -** Return the value of the pager otaMode flag (0, 1 or 2). Or, if -** SQLITE_ENABLE_OTA is not defined, return constant value 0. -*/ -#ifdef SQLITE_ENABLE_OTA -# define PagerOtaMode(pPager) ((pPager)->otaMode) -#else -# define PagerOtaMode(pPager) 0 -#endif - /* ** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains ** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS @@ -851,8 +826,6 @@ static int pagerUseWal(Pager *pPager){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif -static int pagerOpenWalInternal(Pager*, int*); - #ifndef NDEBUG /* ** Usage: @@ -2052,7 +2025,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode && !PagerOtaMode(pPager) + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); @@ -4007,9 +3980,7 @@ int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, - pPager->ckptSyncFlags, pPager->pageSize, (PagerOtaMode(pPager)?0:pTmp) - ); + sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); pPager->pWal = 0; #endif pager_reset(pPager); @@ -5210,11 +5181,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** mode. Otherwise, the following function call is a no-op. */ rc = pagerOpenWalIfPresent(pPager); - if( rc==SQLITE_OK && PagerOtaMode(pPager) ){ - int nWal = sqlite3Strlen30(pPager->zWal); - pPager->zWal[nWal-3] = 'o'; - rc = pagerOpenWalInternal(pPager, 0); - } #ifndef SQLITE_OMIT_WAL assert( pPager->pWal==0 || rc==SQLITE_OK ); @@ -5224,17 +5190,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseWal(pPager) ){ assert( rc==SQLITE_OK ); rc = pagerBeginReadTransaction(pPager); - if( rc==SQLITE_OK && PagerOtaMode(pPager)==1 ){ - rc = sqlite3WalCheckSalt(pPager->pWal, pPager->fd); - if( rc!=SQLITE_OK ){ - sqlite3WalClose(pPager->pWal, 0, 0, 0); - pPager->pWal = 0; - }else{ -#ifdef SQLITE_ENABLE_OTA - pPager->otaMode = 2; -#endif - } - } } if( pPager->eState==PAGER_OPEN && rc==SQLITE_OK ){ @@ -7129,7 +7084,7 @@ void sqlite3PagerClearCache(Pager *pPager){ */ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; - if( pPager->pWal && PagerOtaMode(pPager)==0 ){ + if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, @@ -7197,7 +7152,7 @@ static int pagerOpenWal(Pager *pPager){ */ if( rc==SQLITE_OK ){ rc = sqlite3WalOpen(pPager->pVfs, - pPager->fd, pPager->zWal, pPager->exclusiveMode || PagerOtaMode(pPager), + pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); } @@ -7206,15 +7161,23 @@ static int pagerOpenWal(Pager *pPager){ return rc; } + /* -** Open the WAL file if it is not open. If it is already open, set *pbOpen -** to 1 before returning. Return SQLITE_OK if successful, or an SQLite error -** code otherwise. +** The caller must be holding a SHARED lock on the database file to call +** this function. ** -** The difference between this function and sqlite3PagerOpenWal() is that -** PagerOpenWal() does not open the WAL file if the pager is in OTA mode. +** If the pager passed as the first argument is open on a real database +** file (not a temp file or an in-memory database), and the WAL file +** is not already open, make an attempt to open it now. If successful, +** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** not support the xShmXXX() methods, return an error code. *pbOpen is +** not modified in either case. +** +** If the pager is open on a temp-file (or in-memory database), or if +** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK +** without doing anything. */ -static int pagerOpenWalInternal( +int sqlite3PagerOpenWal( Pager *pPager, /* Pager object */ int *pbOpen /* OUT: Set to true if call is a no-op */ ){ @@ -7244,29 +7207,6 @@ static int pagerOpenWalInternal( return rc; } -/* -** The caller must be holding a SHARED lock on the database file to call -** this function. -** -** If the pager passed as the first argument is open on a real database -** file (not a temp file or an in-memory database), and the WAL file -** is not already open, make an attempt to open it now. If successful, -** return SQLITE_OK. If an error occurs or the VFS used by the pager does -** not support the xShmXXX() methods, return an error code. *pbOpen is -** not modified in either case. -** -** If the pager is open on a temp-file (or in-memory database), or if -** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK -** without doing anything. -*/ -int sqlite3PagerOpenWal( - Pager *pPager, /* Pager object */ - int *pbOpen /* OUT: Set to true if call is a no-op */ -){ - if( PagerOtaMode(pPager) ) return SQLITE_CANTOPEN_BKPT; - return pagerOpenWalInternal(pPager, pbOpen); -} - /* ** This function is called to close the connection to the log file prior ** to switching from WAL to rollback mode. @@ -7313,20 +7253,6 @@ int sqlite3PagerCloseWal(Pager *pPager){ return rc; } -/* -** This function is called by the wal.c module to obtain the 8 bytes of -** "salt" written into the wal file header. In OTA mode, this is a copy -** of bytes 24-31 of the database file. In non-OTA mode, it is 8 bytes -** of pseudo-random data. -*/ -void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt){ - if( PagerOtaMode(pPager) ){ - memcpy(aSalt, pPager->dbFileVers, 8); - }else{ - sqlite3_randomness(8, aSalt); - } -} - #endif /* !SQLITE_OMIT_WAL */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -7344,17 +7270,6 @@ int sqlite3PagerWalFramesize(Pager *pPager){ #endif #ifdef SQLITE_ENABLE_OTA -/* -** Set or clear the "OTA mode" flag. -*/ -int sqlite3PagerSetOtaMode(Pager *pPager, int iOta){ - assert( iOta==1 || iOta==2 ); - if( iOta==1 && (pPager->pWal || pPager->eState!=PAGER_OPEN) ){ - return SQLITE_ERROR; - } - pPager->otaMode = iOta; - return SQLITE_OK; -} /* ** Open an incremental checkpoint handle. diff --git a/src/pager.h b/src/pager.h index 289ef9d42b..10df8e0deb 100644 --- a/src/pager.h +++ b/src/pager.h @@ -210,8 +210,6 @@ void *sqlite3PagerCodec(DbPage *); # define enable_simulated_io_errors() #endif -int sqlite3PagerSetOtaMode(Pager *pPager, int bOta); -void sqlite3PagerWalSalt(Pager *pPager, u32 *aSalt); int sqlite3PagerWalCheckpointStart(sqlite3*, Pager*, u8*, int, sqlite3_ckpt**); #endif /* _PAGER_H_ */ diff --git a/src/parse.y b/src/parse.y index 544888a228..78e79cd361 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1078,7 +1078,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); if( A.pExpr ){ A.pExpr->x.pList = Y; - sqlite3ExprSetHeight(pParse, A.pExpr); + sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3ExprListDelete(pParse->db, Y); } @@ -1091,8 +1091,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); if( A.pExpr ){ A.pExpr->x.pSelect = X; - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); + ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3SelectDelete(pParse->db, X); } @@ -1103,8 +1103,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); if( A.pExpr ){ A.pExpr->x.pSelect = Y; - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); + ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3SelectDelete(pParse->db, Y); } @@ -1117,8 +1117,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); if( A.pExpr ){ A.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - ExprSetProperty(A.pExpr, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, A.pExpr); + ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3SrcListDelete(pParse->db, pSrc); } @@ -1130,8 +1130,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { Expr *p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); if( p ){ p->x.pSelect = Y; - ExprSetProperty(p, EP_xIsSelect); - sqlite3ExprSetHeight(pParse, p); + ExprSetProperty(p, EP_xIsSelect|EP_Subquery); + sqlite3ExprSetHeightAndFlags(pParse, p); }else{ sqlite3SelectDelete(pParse->db, Y); } @@ -1145,7 +1145,7 @@ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). { A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0, 0); if( A.pExpr ){ A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y; - sqlite3ExprSetHeight(pParse, A.pExpr); + sqlite3ExprSetHeightAndFlags(pParse, A.pExpr); }else{ sqlite3ExprListDelete(pParse->db, Y); sqlite3ExprDelete(pParse->db, Z); diff --git a/src/pragma.c b/src/pragma.c index d4c5ba51dc..ac217d4597 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -422,50 +422,6 @@ void sqlite3Pragma( } #endif /* !SQLITE_OMIT_PAGER_PRAGMAS && !SQLITE_OMIT_DEPRECATED */ - /* - ** PRAGMA [database.]pager_ota_mode=[01] - ** - ** This pragma sets a flag on the pager associated with the main database - ** only. The flag can only be set when there is no open transaction and - ** the pager does not already have an open WAL file. - ** - ** Once the flag has been set, it is not possible to open a regular WAL - ** file. If, when the next read-transaction is opened, a *-wal file is - ** found or the database header flags indicate that it is a wal-mode - ** database, SQLITE_CANTOPEN is returned. - ** - ** Otherwise, if no WAL file or flags are found, the pager opens the *-oal - ** file and uses it as a write-ahead-log with the *-shm data stored in - ** heap-memory. If the *-oal file already exists but the database file has - ** been modified since it was created, an SQLITE_BUSY_SNAPSHOT error is - ** returned and the read-transaction cannot be opened. - ** - ** Other clients see a rollback-mode database on which the pager_ota_mode - ** client is holding a SHARED lock. - */ -#ifdef SQLITE_ENABLE_OTA - case PragTyp_PAGER_OTA_MODE: { - Btree *pBt = pDb->pBt; - assert( pBt!=0 ); - if( zRight ){ - int iArg = sqlite3Atoi(zRight); - Pager *pPager = sqlite3BtreePager(pBt); - if( sqlite3BtreeIsInReadTrans(pBt) ){ - sqlite3ErrorMsg(pParse, - "cannot set pager_ota_mode with open transaction" - ); - }else if( sqlite3PagerWalSupported(pPager)==0 ){ - sqlite3ErrorMsg(pParse, - "cannot set pager_ota_mode without wal support" - ); - }else if( sqlite3PagerSetOtaMode(sqlite3BtreePager(pBt), iArg) ){ - sqlite3ErrorMsg(pParse, "cannot set pager_ota_mode in wal mode"); - } - } - break; - } -#endif /* SQLITE_ENABLE_OTA */ - #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) /* ** PRAGMA [database.]page_size diff --git a/src/pragma.h b/src/pragma.h index b2cf43911b..c9ae8e6eb8 100644 --- a/src/pragma.h +++ b/src/pragma.h @@ -28,24 +28,23 @@ #define PragTyp_PAGE_COUNT 22 #define PragTyp_MMAP_SIZE 23 #define PragTyp_PAGE_SIZE 24 -#define PragTyp_PAGER_OTA_MODE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 +#define PragTyp_SECURE_DELETE 25 +#define PragTyp_SHRINK_MEMORY 26 +#define PragTyp_SOFT_HEAP_LIMIT 27 +#define PragTyp_STATS 28 +#define PragTyp_SYNCHRONOUS 29 +#define PragTyp_TABLE_INFO 30 +#define PragTyp_TEMP_STORE 31 +#define PragTyp_TEMP_STORE_DIRECTORY 32 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 #define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { @@ -304,12 +303,6 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif -#if defined(SQLITE_ENABLE_OTA) - { /* zName: */ "pager_ota_mode", - /* ePragTyp: */ PragTyp_PAGER_OTA_MODE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, -#endif #if defined(SQLITE_DEBUG) { /* zName: */ "parser_trace", /* ePragTyp: */ PragTyp_PARSER_TRACE, @@ -463,4 +456,4 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 59 on by default, 73 total. */ +/* Number of pragmas: 59 on by default, 72 total. */ diff --git a/src/resolve.c b/src/resolve.c index d4bd548c93..47df7243f8 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -247,9 +247,10 @@ static int lookupName( testcase( pNC->ncFlags & NC_PartIdx ); testcase( pNC->ncFlags & NC_IsCheck ); if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){ - /* Silently ignore database qualifiers inside CHECK constraints and partial - ** indices. Do not raise errors because that might break legacy and - ** because it does not hurt anything to just ignore the database name. */ + /* Silently ignore database qualifiers inside CHECK constraints and + ** partial indices. Do not raise errors because that might break + ** legacy and because it does not hurt anything to just ignore the + ** database name. */ zDb = 0; }else{ for(i=0; inDb; i++){ @@ -320,7 +321,8 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; - assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */ + /* RIGHT JOIN not (yet) supported */ + assert( (pMatch->jointype & JT_RIGHT)==0 ); if( (pMatch->jointype & JT_LEFT)!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } @@ -641,7 +643,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->affinity = SQLITE_AFF_INTEGER; break; } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */ +#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + && !defined(SQLITE_OMIT_SUBQUERY) */ /* A lone identifier is the name of a column. */ @@ -706,19 +709,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ - sqlite3ErrorMsg(pParse, "second argument to likelihood() must be a " - "constant between 0.0 and 1.0"); + sqlite3ErrorMsg(pParse, + "second argument to likelihood() must be a " + "constant between 0.0 and 1.0"); pNC->nErr++; } }else{ - /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to - ** likelihood(X, 0.0625). - ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for - ** likelihood(X,0.0625). - ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for - ** likelihood(X,0.9375). - ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to - ** likelihood(X,0.9375). */ + /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is + ** equivalent to likelihood(X, 0.0625). + ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is + ** short-hand for likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand + ** for likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent + ** to likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } @@ -735,7 +739,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ return WRC_Prune; } #endif - if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); + if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){ + ExprSetProperty(pExpr,EP_ConstFunc); + } } if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); @@ -1046,7 +1052,8 @@ int sqlite3ResolveOrderGroupBy( resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); return 1; } - resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, zType,0); + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, + zType,0); } } return 0; diff --git a/src/select.c b/src/select.c index 8fbd4bb261..91b3d4345f 100644 --- a/src/select.c +++ b/src/select.c @@ -3194,7 +3194,10 @@ static void substSelect( ** ** (1) The subquery and the outer query do not both use aggregates. ** -** (2) The subquery is not an aggregate or the outer query is not a join. +** (2) The subquery is not an aggregate or (2a) the outer query is not a join +** and (2b) the outer query does not use subqueries other than the one +** FROM-clause subquery that is a candidate for flattening. (2b is +** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) ** ** (3) The subquery is not the right operand of a left outer join ** (Originally ticket #306. Strengthened by ticket #3300) @@ -3331,8 +3334,17 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ - if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ + if( subqueryIsAgg ){ + if( isAgg ) return 0; /* Restriction (1) */ + if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ + if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) + || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 + || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 + ){ + return 0; /* Restriction (2b) */ + } + } + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, @@ -4752,6 +4764,13 @@ int sqlite3Select( } isAgg = (p->selFlags & SF_Aggregate)!=0; assert( pEList!=0 ); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + /* Begin generating code. */ @@ -5497,9 +5516,9 @@ select_end: void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ int n = 0; pView = sqlite3TreeViewPush(pView, moreToFollow); - sqlite3TreeViewLine(pView, "SELECT%s%s", + sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p)", ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p ); if( p->pSrc && p->pSrc->nSrc ) n++; if( p->pWhere ) n++; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index cdf5a8c179..23681cc6f3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -970,6 +970,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_PHASETWO 22 #define SQLITE_FCNTL_WIN32_SET_HANDLE 23 #define SQLITE_FCNTL_ZIPVFS_PAGER 24 +#define SQLITE_FCNTL_OTA 25 /* ** CAPI3REF: Mutex Handle diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5fdbc6d65d..3f9a3a7cc7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1207,6 +1207,7 @@ struct sqlite3 { #define SQLITE_QueryOnly 0x02000000 /* Disable database changes */ #define SQLITE_VdbeEQP 0x04000000 /* Debug EXPLAIN QUERY PLAN */ + /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to @@ -2043,8 +2044,14 @@ struct Expr { #define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_Constant 0x080000 /* Node is a constant */ +#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */ #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ + +/* +** Combinations of two or more EP_* flags +*/ +#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ /* ** These macros can be used to test, set, or clear bits in the @@ -3152,6 +3159,7 @@ ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); void sqlite3ExprListDelete(sqlite3*, ExprList*); +u32 sqlite3ExprListFlags(const ExprList*); int sqlite3Init(sqlite3*, char**); int sqlite3InitCallback(void*, int, char**, char**); void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); @@ -3735,12 +3743,11 @@ void sqlite3MemJournalOpen(sqlite3_file *); int sqlite3MemJournalSize(void); int sqlite3IsMemJournal(sqlite3_file *); +void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 - void sqlite3ExprSetHeight(Parse *pParse, Expr *p); int sqlite3SelectExprHeight(Select *); int sqlite3ExprCheckHeight(Parse*, int); #else - #define sqlite3ExprSetHeight(x,y) #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif @@ -3818,7 +3825,6 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ - /* ** Threading interface */ diff --git a/src/test1.c b/src/test1.c index d3c3d314bf..a87fcd859d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6928,7 +6928,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif }; - static int bitmask_size = sizeof(Bitmask)*8; int i; extern int sqlite3_sync_count, sqlite3_fullsync_count; diff --git a/src/vdbeblob.c b/src/vdbeblob.c index cf1eb59054..ea01f5ce80 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -154,12 +154,17 @@ int sqlite3_blob_open( Incrblob *pBlob = 0; #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){ + if( ppBlob==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + *ppBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif flags = !!flags; /* flags = (flags ? 1 : 0); */ - *ppBlob = 0; sqlite3_mutex_enter(db->mutex); @@ -373,7 +378,7 @@ static int blobReadWrite( sqlite3_mutex_enter(db->mutex); v = (Vdbe*)p->pStmt; - if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ + if( n<0 || iOffset<0 || ((sqlite3_int64)iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; }else if( v==0 ){ diff --git a/src/wal.c b/src/wal.c index bf2ef44a46..fb51ac9fd2 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2920,9 +2920,7 @@ int sqlite3WalFrames( sqlite3Put4byte(&aWalHdr[4], WAL_MAX_VERSION); sqlite3Put4byte(&aWalHdr[8], szPage); sqlite3Put4byte(&aWalHdr[12], pWal->nCkpt); - if( pWal->nCkpt==0 ){ - sqlite3PagerWalSalt(pList->pPager, pWal->hdr.aSalt); - } + if( pWal->nCkpt==0 ) sqlite3_randomness(8, pWal->hdr.aSalt); memcpy(&aWalHdr[16], pWal->hdr.aSalt, 8); walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); diff --git a/test/collate8.test b/test/collate8.test index 60a89162dd..b06c87d2f8 100644 --- a/test/collate8.test +++ b/test/collate8.test @@ -13,7 +13,9 @@ # focus of this script is making sure collations pass through the # unary + operator. # -# $Id: collate8.test,v 1.2 2008/08/25 12:14:09 drh Exp $ +# 2015-02-09: Added tests to make sure COLLATE passes through function +# calls. Ticket [ca0d20b6cdddec5e81b8d66f89c46a5583b5f6f6]. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -122,4 +124,34 @@ do_test collate8-2.8 { } } {abc} +# Make sure the COLLATE operator perculates up through function calls +# and other Expr structures that use the Expr.x.pList field. +# +do_execsql_test collate8-3.1 { + SELECT 'abc'==('ABC'||'') COLLATE nocase; + SELECT 'abc'==('ABC'||'' COLLATE nocase); + SELECT 'abc'==('ABC'||('' COLLATE nocase)); + SELECT 'abc'==('ABC'||upper('' COLLATE nocase)); +} {1 1 1 1} +do_execsql_test collate8-3.2 { + SELECT 'abc'==('ABC'||max('' COLLATE nocase,'' COLLATE binary)); +} {1} + +# The COLLATE binary is on the left and so takes precedence +do_execsql_test collate8-3.3 { + SELECT 'abc'==('ABC'||max('' COLLATE binary,'' COLLATE nocase)); +} {0} + +do_execsql_test collate8-3.4 { + SELECT 'abc'==('ABC'||CASE WHEN 1-1=2 THEN '' COLLATE nocase + ELSE '' COLLATE binary END); + SELECT 'abc'==('ABC'||CASE WHEN 1+1=2 THEN '' COLLATE nocase + ELSE '' COLLATE binary END); +} {1 1} +do_execsql_test collate8-3.5 { + SELECT 'abc'==('ABC'||CASE WHEN 1=2 THEN '' COLLATE binary + ELSE '' COLLATE nocase END); +} {0} + + finish_test diff --git a/test/incrblob2.test b/test/incrblob2.test index a8f40f09dc..1a235f7d22 100644 --- a/test/incrblob2.test +++ b/test/incrblob2.test @@ -324,12 +324,34 @@ do_test incrblob2-6.2 { sqlite3_blob_read $rdHandle 0 2 } {AB} +do_test incrblob2-6.2b { + set rc [catch { + # Prior to 2015-02-07, the following caused a segfault due to + # integer overflow. + sqlite3_blob_read $rdHandle 2147483647 2147483647 + } errmsg] + lappend rc $errmsg +} {1 SQLITE_ERROR} + do_test incrblob2-6.3 { set wrHandle [db incrblob t1 data 1] sqlite3_blob_write $wrHandle 0 ZZZZZZZZZZ sqlite3_blob_read $rdHandle 2 4 } {ZZZZ} +do_test incrblob2-6.3b { + set rc [catch { + # Prior to 2015-02-07, the following caused a segfault due to + # integer overflow. + sqlite3_blob_write $wrHandle 2147483647 YYYYYYYYYYYYYYYYYY + } errmsg] + lappend rc $errmsg +} {1 SQLITE_ERROR} +do_test incrblob2-6.3c { + sqlite3_blob_read $rdHandle 2 4 +} {ZZZZ} + + do_test incrblob2-6.4 { close $wrHandle close $rdHandle diff --git a/test/select6.test b/test/select6.test index 64a8519d89..590512a6b0 100644 --- a/test/select6.test +++ b/test/select6.test @@ -557,5 +557,61 @@ do_catchsql_test 10.8 { ) } $err +# 2015-02-09 Ticket [2f7170d73bf9abf80339187aa3677dce3dbcd5ca] +# "misuse of aggregate" error if aggregate column from FROM +# subquery is used in correlated subquery +# +do_execsql_test 11.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(w INT, x INT); + INSERT INTO t1(w,x) + VALUES(1,10),(2,20),(3,30), + (2,21),(3,31), + (3,32); + CREATE INDEX t1wx ON t1(w,x); + + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(w INT, y VARCHAR(8)); + INSERT INTO t2(w,y) VALUES(1,'one'),(2,'two'),(3,'three'),(4,'four'); + CREATE INDEX t2wy ON t2(w,y); + + SELECT cnt, xyz, (SELECT y FROM t2 WHERE w=cnt), '|' + FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2) + ORDER BY cnt, xyz; +} {1 1 one | 2 2 two | 3 3 three |} +do_execsql_test 11.2 { + SELECT cnt, xyz, lower((SELECT y FROM t2 WHERE w=cnt)), '|' + FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2) + ORDER BY cnt, xyz; +} {1 1 one | 2 2 two | 3 3 three |} +do_execsql_test 11.3 { + SELECT cnt, xyz, '|' + FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2) + WHERE (SELECT y FROM t2 WHERE w=cnt)!='two' + ORDER BY cnt, xyz; +} {1 1 | 3 3 |} +do_execsql_test 11.4 { + SELECT cnt, xyz, '|' + FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2) + ORDER BY lower((SELECT y FROM t2 WHERE w=cnt)); +} {1 1 | 3 3 | 2 2 |} +do_execsql_test 11.5 { + SELECT cnt, xyz, + CASE WHEN (SELECT y FROM t2 WHERE w=cnt)=='two' + THEN 'aaa' ELSE 'bbb' + END, '|' + FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2) + ORDER BY +cnt; +} {1 1 bbb | 2 2 aaa | 3 3 bbb |} + +do_execsql_test 11.100 { + DROP TABLE t1; + DROP TABLE t2; + CREATE TABLE t1(x); + CREATE TABLE t2(y, z); + SELECT ( SELECT y FROM t2 WHERE z = cnt ) + FROM ( SELECT count(*) AS cnt FROM t1 ); +} {{}} + finish_test diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 5627a19d35..964946e788 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -315,9 +315,6 @@ set pragma_def { NAME: soft_heap_limit NAME: threads - - NAME: pager_ota_mode - IF: defined(SQLITE_ENABLE_OTA) } # Open the output file diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 4ab8b12b45..4034128d5b 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -112,8 +112,9 @@ foreach hdr { pcache.h pragma.h rtree.h - sqlite3ext.h sqlite3.h + sqlite3ext.h + sqlite3ota.h sqliteicu.h sqliteInt.h sqliteLimit.h @@ -334,6 +335,7 @@ foreach file { rtree.c icu.c fts3_icu.c + sqlite3ota.c } { copy_file tsrc/$file } diff --git a/tool/spaceanal.tcl b/tool/spaceanal.tcl index a227b85243..516d282445 100644 --- a/tool/spaceanal.tcl +++ b/tool/spaceanal.tcl @@ -4,6 +4,24 @@ # if {[catch { + +# Argument $tname is the name of a table within the database opened by +# database handle [db]. Return true if it is a WITHOUT ROWID table, or +# false otherwise. +# +proc is_without_rowid {tname} { + set t [string map {' ''} $tname] + db eval "PRAGMA index_list = '$t'" o { + if {$o(origin) == "pk"} { + set n $o(name) + if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} { + return 1 + } + } + } + return 0 +} + # Get the name of the database to analyze # proc usage {} { @@ -167,20 +185,21 @@ set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 } foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] { set is_index [expr {$name!=$tblname}] + set idx_btree [expr {$is_index || [is_without_rowid $name]}] db eval { SELECT sum(ncell) AS nentry, - sum(isleaf(pagetype, $is_index) * ncell) AS leaf_entries, + sum(isleaf(pagetype, $idx_btree) * ncell) AS leaf_entries, sum(payload) AS payload, - sum(isoverflow(pagetype, $is_index) * payload) AS ovfl_payload, + sum(isoverflow(pagetype, $idx_btree) * payload) AS ovfl_payload, sum(path LIKE '%+000000') AS ovfl_cnt, max(mx_payload) AS mx_payload, - sum(isinternal(pagetype, $is_index)) AS int_pages, - sum(isleaf(pagetype, $is_index)) AS leaf_pages, - sum(isoverflow(pagetype, $is_index)) AS ovfl_pages, - sum(isinternal(pagetype, $is_index) * unused) AS int_unused, - sum(isleaf(pagetype, $is_index) * unused) AS leaf_unused, - sum(isoverflow(pagetype, $is_index) * unused) AS ovfl_unused, + sum(isinternal(pagetype, $idx_btree)) AS int_pages, + sum(isleaf(pagetype, $idx_btree)) AS leaf_pages, + sum(isoverflow(pagetype, $idx_btree)) AS ovfl_pages, + sum(isinternal(pagetype, $idx_btree) * unused) AS int_unused, + sum(isleaf(pagetype, $idx_btree) * unused) AS leaf_unused, + sum(isoverflow(pagetype, $idx_btree) * unused) AS ovfl_unused, sum(pgsize) AS compressed_size FROM temp.dbstat WHERE name = $name } break