From 73b09b87d5a7c9e35af045b5b59cbd49756cb4df Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Aug 2022 20:45:43 +0000 Subject: [PATCH 01/42] Add new files for an extension to recover data from corrupted databases. FossilOrigin-Name: f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 --- ext/misc/dbdata.c | 31 +- ext/recover/recover1.test | 116 ++++++ ext/recover/recover_common.tcl | 5 + ext/recover/sqlite3recover.c | 728 +++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.h | 66 +++ ext/recover/test_recover.c | 185 +++++++++ main.mk | 3 + manifest | 26 +- manifest.uuid | 2 +- src/test_tclsh.c | 2 + 10 files changed, 1152 insertions(+), 12 deletions(-) create mode 100644 ext/recover/recover1.test create mode 100644 ext/recover/recover_common.tcl create mode 100644 ext/recover/sqlite3recover.c create mode 100644 ext/recover/sqlite3recover.h create mode 100644 ext/recover/test_recover.c diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 7405e7c890..b79eafce7c 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -660,6 +660,18 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +/* +** Return true if nul-terminated string zSchema ends in "()". Or false +** otherwise. +*/ +static int dbdataIsFunction(const char *zSchema){ + int n = strlen(zSchema); + if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){ + return n-2; + } + return 0; +} + /* ** Determine the size in pages of database zSchema (where zSchema is ** "main", "temp" or the name of an attached database) and set @@ -670,10 +682,16 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; int rc, rc2; + int nFunc = 0; sqlite3_stmt *pStmt = 0; - zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema); + }else{ + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + } if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); sqlite3_free(zSql); if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ @@ -711,9 +729,18 @@ static int dbdataFilter( } if( rc==SQLITE_OK ){ + int nFunc = 0; if( pTab->pStmt ){ pCsr->pStmt = pTab->pStmt; pTab->pStmt = 0; + }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){ + char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + } }else{ rc = sqlite3_prepare_v2(pTab->db, "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, @@ -732,7 +759,7 @@ static int dbdataFilter( return rc; } -/* +/* ** Return a column for the sqlite_dbdata or sqlite_dbptr table. */ static int dbdataColumn( diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test new file mode 100644 index 0000000000..c4348baea0 --- /dev/null +++ b/ext/recover/recover1.test @@ -0,0 +1,116 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recover1 + + + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn} { + forcedelete test.db2 + + uplevel [list do_test $tn.1 { + set R [sqlite3_recover_init db main test.db2] + $R step + $R finish + } {}] + + sqlite3 db2 test.db2 + uplevel [list do_test $tn.2 [list compare_dbs db db2] {}] + db2 close +} + + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b) WITHOUT ROWID; + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT i*2, hex(randomblob(250)) FROM s; + INSERT INTO t2 SELECT * FROM t1; +} + +do_recover_test 1 + +do_execsql_test 2.0 { + ALTER TABLE t1 ADD COLUMN c DEFAULT 'xyz' +} +do_recover_test 2 + +do_execsql_test 3.0 { + CREATE INDEX i1 ON t1(c); +} +do_recover_test 3 + +do_execsql_test 4.0 { + CREATE VIEW v1 AS SELECT * FROM t2; +} +do_recover_test 4 + +do_execsql_test 5.0 { + CREATE UNIQUE INDEX i2 ON t1(c, b); +} +do_recover_test 5 + +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1( + a INTEGER PRIMARY KEY, + b INT, + c TEXT, + d INT GENERATED ALWAYS AS (a*abs(b)) VIRTUAL, + e TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED, + f TEXT GENERATED ALWAYS AS (substr(c,b,b+1)) STORED + ); + + INSERT INTO t1 VALUES(1, 2, 'hello world'); +} +do_recover_test 6 + +do_execsql_test 7.0 { + CREATE TABLE t2(i, j GENERATED ALWAYS AS (i+1) STORED, k); + INSERT INTO t2 VALUES(10, 'ten'); +} +do_execsql_test 7.1 { + SELECT * FROM t2 +} {10 11 ten} + +do_recover_test 7.2 + +finish_test + diff --git a/ext/recover/recover_common.tcl b/ext/recover/recover_common.tcl new file mode 100644 index 0000000000..3f2ff2d6cc --- /dev/null +++ b/ext/recover/recover_common.tcl @@ -0,0 +1,5 @@ + + + + + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c new file mode 100644 index 0000000000..6c7828c007 --- /dev/null +++ b/ext/recover/sqlite3recover.c @@ -0,0 +1,728 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +#include "sqlite3recover.h" +#include +#include + +typedef unsigned int u32; +typedef sqlite3_int64 i64; + +typedef struct RecoverColumn RecoverColumn; +struct RecoverColumn { + char *zCol; + int eHidden; +}; + +#define RECOVER_EHIDDEN_NONE 0 +#define RECOVER_EHIDDEN_HIDDEN 1 +#define RECOVER_EHIDDEN_VIRTUAL 2 +#define RECOVER_EHIDDEN_STORED 3 + +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iPk; /* Index of IPK column, if bIntkey */ + + RecoverTable *pNext; +}; + +/* +** +*/ +#define RECOVERY_SCHEMA \ +" CREATE TABLE recovery.freelist(" \ +" pgno INTEGER PRIMARY KEY" \ +" );" \ +" CREATE TABLE recovery.dbptr(" \ +" pgno, child, PRIMARY KEY(child, pgno)" \ +" ) WITHOUT ROWID;" \ +" CREATE TABLE recovery.map(" \ +" pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" \ +" );" \ +" CREATE TABLE recovery.schema(" \ +" type, name, tbl_name, rootpage, sql" \ +" );" + + +struct sqlite3_recover { + sqlite3 *dbIn; + sqlite3 *dbOut; + + sqlite3_stmt *pGetPage; + + char *zDb; + char *zUri; + RecoverTable *pTblList; + + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + char *zStateDb; +}; + +/* +** Like strlen(). But handles NULL pointer arguments. +*/ +static int recoverStrlen(const char *zStr){ + int nRet = 0; + if( zStr ){ + while( zStr[nRet] ) nRet++; + } + return nRet; +} + +static void *recoverMalloc(sqlite3_recover *p, sqlite3_int64 nByte){ + void *pRet = 0; + assert( nByte>0 ); + if( p->errCode==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + }else{ + p->errCode = SQLITE_NOMEM; + } + } + return pRet; +} + +static int recoverError( + sqlite3_recover *p, + int errCode, + const char *zFmt, ... +){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + + sqlite3_free(p->zErrMsg); + p->zErrMsg = z; + p->errCode = errCode; + return errCode; +} + +static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ + return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); +} + +static sqlite3_stmt *recoverPrepare( + sqlite3_recover *p, + sqlite3 *db, + const char *zSql +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){ + recoverDbError(p, db); + } + } + return pStmt; +} + +/* +** Create a prepared statement using printf-style arguments for the SQL. +*/ +static sqlite3_stmt *recoverPreparePrintf( + sqlite3_recover *p, + sqlite3 *db, + const char *zFmt, ... +){ + sqlite3_stmt *pStmt = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + p->errCode = SQLITE_NOMEM; + }else{ + pStmt = recoverPrepare(p, db, z); + sqlite3_free(z); + } + } + return pStmt; +} + + +static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, sqlite3_db_handle(pStmt)); + } + return pStmt; +} + +static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ + recoverDbError(p, db); + } +} + +static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc ){ + recoverDbError(p, p->dbOut); + } + } + return p->errCode; +} + +/* +** The implementation of a user-defined SQL function invoked by the +** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages +** of the database being recovered. +** +** This function always takes a single integer argument. If the arguement +** is zero, then the value returned is the number of pages in the db being +** recovered. If the argument is greater than zero, it is a page number. +** The value returned in this case is an SQL blob containing the data for +** the identified page of the db being recovered. e.g. +** +** SELECT getpage(0); -- return number of pages in db +** SELECT getpage(4); -- return page 4 of db as a blob of data +*/ +static void recoverGetPage( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + + assert( nArg==1 ); + if( pgno==0 ){ + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + }else if( p->pGetPage==0 ){ + pStmt = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); + }else{ + pStmt = p->pGetPage; + } + + if( pStmt ){ + if( pgno ) sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + } + if( pgno ){ + p->pGetPage = recoverReset(p, pStmt); + }else{ + recoverFinalize(p, pStmt); + } + } + + if( p->errCode ){ + if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1); + sqlite3_result_error_code(pCtx, p->errCode); + } +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +static int recoverOpenOutput(sqlite3_recover *p){ + int rc = SQLITE_OK; + if( p->dbOut==0 ){ + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; + + assert( p->dbOut==0 ); + + rc = sqlite3_open_v2(p->zUri, &db, flags, 0); + if( rc==SQLITE_OK ){ + const char *zPath = p->zStateDb ? p->zStateDb : ":memory:"; + char *zSql = sqlite3_mprintf("ATTACH %Q AS recovery", zPath); + if( zSql==0 ){ + rc = p->errCode = SQLITE_NOMEM; + }else{ + rc = sqlite3_exec(db, zSql, 0, 0, 0); + } + sqlite3_free(zSql); + } + + if( rc==SQLITE_OK ){ + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db, "recovery"); + if( pBackup ){ + while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); + rc = sqlite3_backup_finish(pBackup); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, RECOVERY_SCHEMA, 0, 0, 0); + } + + if( rc==SQLITE_OK ){ + sqlite3_dbdata_init(db, 0, 0); + rc = sqlite3_create_function( + db, "getpage", 1, SQLITE_UTF8, (void*)p, recoverGetPage, 0, 0 + ); + } + + if( rc!=SQLITE_OK ){ + if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); + sqlite3_close(db); + }else{ + p->dbOut = db; + } + } + return rc; +} + +static int recoverCacheDbptr(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "INSERT INTO recovery.dbptr " + "SELECT pgno, child FROM sqlite_dbptr('getpage()')" + ); +} + +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM recovery.dbptr, pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ + sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, + "PRAGMA table_xinfo(%Q)", zName + ); + + if( pStmt ){ + RecoverTable *pNew = 0; + int nCol = 0; + int nName = recoverStrlen(zName); + int nByte = 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + nCol++; + nByte += (sqlite3_column_bytes(pStmt, 1)+1); + } + nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1; + recoverReset(p, pStmt); + + pNew = recoverMalloc(p, nByte); + if( pNew ){ + int i = 0; + char *csr = 0; + pNew->aCol = (RecoverColumn*)&pNew[1]; + pNew->zTab = csr = (char*)&pNew->aCol[nCol]; + pNew->nCol = nCol; + pNew->iRoot = iRoot; + pNew->iPk = -1; + memcpy(csr, zName, nName); + csr += nName+1; + + for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ + int bPk = sqlite3_column_int(pStmt, 5); + int n = sqlite3_column_bytes(pStmt, 1); + const char *z = (const char*)sqlite3_column_text(pStmt, 1); + int eHidden = sqlite3_column_int(pStmt, 6); + + if( bPk ) pNew->iPk = i; + pNew->aCol[i].zCol = csr; + pNew->aCol[i].eHidden = eHidden; + memcpy(csr, z, n); + csr += (n+1); + } + + pNew->pNext = p->pTblList; + p->pTblList = pNew; + } + + recoverFinalize(p, pStmt); + + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_info(%Q)", zName); + if( pStmt && sqlite3_step(pStmt)!=SQLITE_ROW ){ + pNew->bIntkey = 1; + }else{ + pNew->iPk = -1; + } + recoverFinalize(p, pStmt); + } +} + +/* +** +*/ +static int recoverWriteSchema1(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + sqlite3_stmt *pTblname = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "SELECT rootpage, sql, type='table' FROM recovery.schema " + " WHERE type='table' OR (type='index' AND sql LIKE '%unique%')" + ); + + pTblname = recoverPrepare(p, p->dbOut, + "SELECT name FROM sqlite_schema " + "WHERE type='table' ORDER BY rowid DESC LIMIT 1" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int bTable = sqlite3_column_int(pSelect, 2); + + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc==SQLITE_OK ){ + if( bTable ){ + if( SQLITE_ROW==sqlite3_step(pTblname) ){ + const char *zName = sqlite3_column_text(pTblname, 0); + recoverAddTable(p, zName, iRoot); + } + recoverReset(p, pTblname); + } + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + recoverFinalize(p, pTblname); + + return p->errCode; +} + +static int recoverWriteSchema2(sqlite3_recover *p){ + sqlite3_stmt *pSelect = 0; + + pSelect = recoverPrepare(p, p->dbOut, + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" + ); + + if( pSelect ){ + while( sqlite3_step(pSelect)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSelect, 0); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); + int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( rc!=SQLITE_OK && rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); + } + } + } + recoverFinalize(p, pSelect); + + return p->errCode; +} + + +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ + char *zRet = 0; + if( p->errCode==SQLITE_OK ){ + va_list ap; + char *z; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( zRet==0 ){ + p->errCode = SQLITE_NOMEM; + } + } + return zRet; +} + +static sqlite3_stmt *recoverInsertStmt( + sqlite3_recover *p, + RecoverTable *pTab, + int nField +){ + const char *zSep = ""; + char *zSql = 0; + char *zBind = 0; + int ii; + sqlite3_stmt *pRet = 0; + + assert( nField<=pTab->nCol ); + + zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + for(ii=0; iiaCol[ii].eHidden; + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); + zBind = recoverMPrintf(p, "%z%s?", zBind, zSep); + zSep = ", "; + } + } + zSql = recoverMPrintf(p, "%z) VALUES (%z)", zSql, zBind); + + pRet = recoverPrepare(p, p->dbOut, zSql); + sqlite3_free(zSql); + + return pRet; +} + + +static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ + RecoverTable *pRet = 0; + for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); + return pRet; +} + +static int recoverWriteData(sqlite3_recover *p){ + RecoverTable *pTbl; + int nMax = 0; + sqlite3_value **apVal = 0; + sqlite3_stmt *pSel = 0; + + /* Figure out the maximum number of columns for any table in the schema */ + for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ + if( pTbl->nCol>nMax ) nMax = pTbl->nCol; + } + + apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * nMax); + if( apVal==0 ) return p->errCode; + + pSel = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(root, page) AS (" + " SELECT rootpage, rootpage FROM recovery.schema" + " UNION" + " SELECT root, child FROM recovery.dbptr, pages WHERE pgno=page" + ") " + "SELECT root, page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + "UNION ALL " + "SELECT 0, 0, 0, 0, 0" + ); + if( pSel ){ + RecoverTable *pTab = 0; + sqlite3_stmt *pInsert = 0; + int nInsert = -1; + i64 iPrevRoot = -1; + i64 iPrevPage = -1; + int iPrevCell = -1; + i64 iRowid = 0; + int nVal = -1; + + while( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pSel, 0); + i64 iPage = sqlite3_column_int64(pSel, 1); + int iCell = sqlite3_column_int(pSel, 2); + int iField = sqlite3_column_int(pSel, 3); + sqlite3_value *pVal = sqlite3_column_value(pSel, 4); + + int bNewCell = (iPrevRoot!=iRoot || iPrevPage!=iPage || iPrevCell!=iCell); + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==nVal ); + + if( bNewCell ){ + if( nVal>=0 ){ + int ii; + + if( pTab ){ + int iVal = 0; + int iBind = 1; + + if( pInsert==0 || nVal!=nInsert ){ + recoverFinalize(p, pInsert); + pInsert = recoverInsertStmt(p, pTab, nVal); + nInsert = nVal; + } + + for(ii=0; iinCol && iValaCol[ii].eHidden; + switch( eHidden ){ + case RECOVER_EHIDDEN_NONE: + case RECOVER_EHIDDEN_HIDDEN: + if( ii==pTab->iPk ){ + sqlite3_bind_int64(pInsert, iBind, iRowid); + }else{ + sqlite3_bind_value(pInsert, iBind, apVal[iVal]); + } + iBind++; + iVal++; + break; + + case RECOVER_EHIDDEN_VIRTUAL: + break; + + case RECOVER_EHIDDEN_STORED: + iVal++; + break; + } + } + + sqlite3_step(pInsert); + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); + } + + for(ii=0; iierrCode; +} + +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + sqlite3_recover *pRet = 0; + int nDb = 0; + int nUri = 0; + int nByte = 0; + + if( zDb==0 ){ zDb = "main"; } + if( zUri==0 ){ zUri = ""; } + + nDb = recoverStrlen(zDb); + nUri = recoverStrlen(zUri); + + nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1; + pRet = (sqlite3_recover*)sqlite3_malloc(nByte); + if( pRet ){ + memset(pRet, 0, nByte); + pRet->dbIn = db; + pRet->zDb = (char*)&pRet[1]; + pRet->zUri = &pRet->zDb[nDb+1]; + memcpy(pRet->zDb, zDb, nDb); + memcpy(pRet->zUri, zUri, nUri); + } + + return pRet; +} + +const char *sqlite3_recover_errmsg(sqlite3_recover *p){ + return p ? p->zErrMsg : "not an error"; +} +int sqlite3_recover_errcode(sqlite3_recover *p){ + return p ? p->errCode : SQLITE_NOMEM; +} + +int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ + int rc = SQLITE_OK; + + switch( op ){ + case SQLITE_RECOVER_TESTDB: + sqlite3_free(p->zStateDb); + p->zStateDb = sqlite3_mprintf("%s", (char*)pArg); + break; + + default: + rc = SQLITE_NOTFOUND; + break; + } + + return rc; +} + +static void recoverStep(sqlite3_recover *p){ + + assert( p->errCode==SQLITE_OK ); + + if( p->dbOut==0 ){ + if( recoverOpenOutput(p) ) return; + if( recoverCacheDbptr(p) ) return; + if( recoverCacheSchema(p) ) return; + if( recoverWriteSchema1(p) ) return; + if( recoverWriteData(p) ) return; + if( recoverWriteSchema2(p) ) return; + } +} + +int sqlite3_recover_step(sqlite3_recover *p){ + if( p && p->errCode==SQLITE_OK ){ + recoverStep(p); + } + return p ? p->errCode : SQLITE_NOMEM; +} + +int sqlite3_recover_finish(sqlite3_recover *p){ + RecoverTable *pTab; + RecoverTable *pNext; + int rc; + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + + sqlite3_finalize(p->pGetPage); + rc = sqlite3_close(p->dbOut); + assert( rc==SQLITE_OK ); + p->pGetPage = 0; + rc = p->errCode; + + sqlite3_free(p); + return rc; +} + diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h new file mode 100644 index 0000000000..401f83ea28 --- /dev/null +++ b/ext/recover/sqlite3recover.h @@ -0,0 +1,66 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + + +#ifndef _SQLITE_RECOVER_H +#define _SQLITE_RECOVER_H + +#include "sqlite3.h" /* Required for error code definitions */ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct sqlite3_recover sqlite3_recover; + +/* Create an object to recover data from database zDb (e.g. "main") +** opened by handle db. Data will be recovered into the database +** identified by parameter zUri. Database zUri is clobbered if it +** already exists. +*/ +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +); + +/* Details TBD. */ +int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); + +#define SQLITE_RECOVER_TESTDB 789 + +/* Step the recovery object. Return SQLITE_DONE if recovery is complete, +** SQLITE_OK if recovery is not complete but no error has occurred, or +** an SQLite error code if an error has occurred. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +const char *sqlite3_recover_errmsg(sqlite3_recover*); + +int sqlite3_recover_errcode(sqlite3_recover*); + +/* Clean up a recovery object created by a call to sqlite3_recover_init(). +** This function returns SQLITE_DONE if the new database was created, +** SQLITE_OK if it processing was abandoned before it as finished or +** an SQLite error code (e.g. SQLITE_IOERR, SQLITE_NOMEM etc.) if an +** error occurred. */ +int sqlite3_recover_finish(sqlite3_recover*); + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE_RECOVER_H */ + diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c new file mode 100644 index 0000000000..912b8dec5c --- /dev/null +++ b/ext/recover/test_recover.c @@ -0,0 +1,185 @@ +/* +** 2022-08-27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + +#include "sqlite3recover.h" + +#include +#include + +typedef struct TestRecover TestRecover; +struct TestRecover { + sqlite3_recover *p; +}; + +static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ + Tcl_CmdInfo info; + if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ + Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0); + return TCL_ERROR; + } + *pDb = *(sqlite3 **)info.objClientData; + return TCL_OK; +} + +/* +** Implementation of the command created by [sqlite3_recover_init]: +** +** $cmd config OP ARG +** $cmd step +** $cmd errmsg +** $cmd errcode +** $cmd finalize +*/ +static int testRecoverCmd( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct RecoverSub { + const char *zSub; + int nArg; + const char *zMsg; + } aSub[] = { + { "config", 2, "REBASE-BLOB" }, /* 0 */ + { "step", 0, "" }, /* 1 */ + { "errmsg", 0, "" }, /* 2 */ + { "errcode", 0, "" }, /* 3 */ + { "finish", 0, "" }, /* 4 */ + { 0 } + }; + int rc = TCL_OK; + int iSub = 0; + TestRecover *pTest = (TestRecover*)clientData; + + if( objc<2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SUBCOMMAND ..."); + return TCL_ERROR; + } + rc = Tcl_GetIndexFromObjStruct(interp, + objv[1], aSub, sizeof(aSub[0]), "sub-command", 0, &iSub + ); + if( rc!=TCL_OK ) return rc; + if( (objc-2)!=aSub[iSub].nArg ){ + Tcl_WrongNumArgs(interp, 2, objv, aSub[iSub].zMsg); + return TCL_ERROR; + } + + switch( iSub ){ + case 0: assert( sqlite3_stricmp("config", aSub[iSub].zSub)==0 ); { + const char *aOp[] = { + "testdb", /* 0 */ + 0 + }; + int iOp = 0; + int res = 0; + if( Tcl_GetIndexFromObj(interp, objv[2], aOp, "option", 0, &iOp) ){ + return TCL_ERROR; + } + switch( iOp ){ + case 0: + res = sqlite3_recover_config( + pTest->p, SQLITE_RECOVER_TESTDB, (void*)Tcl_GetString(objv[3]) + ); + break; + } + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 1: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_step(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } + case 2: assert( sqlite3_stricmp("errmsg", aSub[iSub].zSub)==0 ); { + const char *zErr = sqlite3_recover_errmsg(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); + break; + } + case 3: assert( sqlite3_stricmp("errcode", aSub[iSub].zSub)==0 ); { + int errCode = sqlite3_recover_errcode(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(errCode)); + break; + } + case 4: assert( sqlite3_stricmp("finish", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_errcode(pTest->p); + int res2; + if( res!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(pTest->p); + char *zRes = sqlite3_mprintf("(%d) - %s", res, zErr); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes, -1)); + sqlite3_free(zRes); + } + res2 = sqlite3_recover_finish(pTest->p); + assert( res2==res ); + if( res ) return TCL_ERROR; + break; + } + } + + return TCL_OK; +} + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_recover_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static int iTestRecoverCmd = 1; + + TestRecover *pNew = 0; + sqlite3 *db = 0; + const char *zDb = 0; + const char *zUri = 0; + char zCmd[128]; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME URI"); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + zUri = Tcl_GetString(objv[3]); + + pNew = ckalloc(sizeof(TestRecover)); + pNew->p = sqlite3_recover_init(db, zDb, zUri); + + sprintf(zCmd, "sqlite_recover%d", iTestRecoverCmd++); + Tcl_CreateObjCommand(interp, zCmd, testRecoverCmd, (void*)pNew, 0); + + Tcl_SetObjResult(interp, Tcl_NewStringObj(zCmd, -1)); + return TCL_OK; +} + +int TestRecover_Init(Tcl_Interp *interp){ + struct Cmd { + const char *zCmd; + Tcl_ObjCmdProc *xProc; + } aCmd[] = { + { "sqlite3_recover_init", test_sqlite3_recover_init }, + }; + int i; + + for(i=0; izCmd, p->xProc, 0, 0); + } + + return TCL_OK; +} + diff --git a/main.mk b/main.mk index 3d8a07494d..5263546042 100644 --- a/main.mk +++ b/main.mk @@ -444,6 +444,9 @@ TESTSRC2 = \ $(TOP)/ext/misc/stmt.c \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ fts5.c # Header files used by all library source files. diff --git a/manifest b/manifest index 746bbe3c38..0fab53a46f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sb-tree\spage\ssorting\scode\sto\sensure\sthat\ssqlite3PagerRekey()\snever\noverloads\sa\spage\snumber\sand\suses\sonly\sthe\sPENDING_BYTE\spage\sfor\stemporary\nstorage. -D 2022-08-31T15:04:42.204 +C Add\snew\sfiles\sfor\san\sextension\sto\srecover\sdata\sfrom\scorrupted\sdatabases. +D 2022-08-31T20:45:43.730 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c e316fba936571584e55abd5b974a32a191727a6b746053a0c9d439bd2cf93940 +F ext/misc/dbdata.c f317980cea788e67932828b94a16ee8a8b859e3c2d62859d09ba3d5ca85f87cb F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -387,6 +387,11 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a +F ext/recover/recover1.test 861ad5140566102a8c5a3d1f936a7d6da569f34c86597c274de695f597031bac +F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c +F ext/recover/sqlite3recover.c 594fb45777a14f0b88b944b9fb2ccb3e85a29ef5b17522b8dac3e3944c4c27ea +F ext/recover/sqlite3recover.h 3255f6491007e57be310aedb72a848c88f79fc14e7222bda4b8d4dab1a2450c3 +F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -509,7 +514,7 @@ F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e4 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 20801eed419dc58936ff9449b04041edbbbc0488a9fc683e72471dded050e0bb +F main.mk 8c9965c408aaa8b93d0dd52e83445894835e1a42dc360c77435393f80f8d8d1d F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -641,7 +646,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c 540feaea7280cd5f926168aee9deb1065ae136d0bbbe7361e2ef3541783e187a F src/test_superlock.c 4839644b9201da822f181c5bc406c0b2385f672e F src/test_syscall.c 1073306ba2e9bfc886771871a13d3de281ed3939 -F src/test_tclsh.c c4065ced25126e25c40122c5ff62dc89902ea617d72cdd27765151cdd7fcc477 +F src/test_tclsh.c 7dd98be675a1dc0d1fd302b8247bab992c909db384df054381a2279ad76f9b0e F src/test_tclvar.c 33ff42149494a39c5fbb0df3d25d6fafb2f668888e41c0688d07273dcb268dfc F src/test_thread.c 269ea9e1fa5828dba550eb26f619aa18aedbc29fd92f8a5f6b93521fbb74a61c F src/test_vdbecov.c f60c6f135ec42c0de013a1d5136777aa328a776d33277f92abac648930453d43 @@ -1999,8 +2004,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dd017bb1b3e31c7692d29dc4865d6bda871e429978c8738a39160d0114e5bf9b -R 98122ff0aaf5ca87deda59c5c8a25251 -U drh -Z 8d73d18db9ab73a94a9689d17f937c1d +P 5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 +R 563b8320bf923831e4768bc403655fc2 +T *branch * recover-extension +T *sym-recover-extension * +T -sym-trunk * +U dan +Z 1c7612740eb933f84d589533d182c6df # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 00cb077051..16703e1b2f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 \ No newline at end of file +f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 \ No newline at end of file diff --git a/src/test_tclsh.c b/src/test_tclsh.c index 707c16812c..c133deca25 100644 --- a/src/test_tclsh.c +++ b/src/test_tclsh.c @@ -108,6 +108,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ extern int TestExpert_Init(Tcl_Interp*); extern int Sqlitetest_window_Init(Tcl_Interp *); extern int Sqlitetestvdbecov_Init(Tcl_Interp *); + extern int TestRecover_Init(Tcl_Interp*); Tcl_CmdInfo cmdInfo; @@ -175,6 +176,7 @@ const char *sqlite3TestInit(Tcl_Interp *interp){ TestExpert_Init(interp); Sqlitetest_window_Init(interp); Sqlitetestvdbecov_Init(interp); + TestRecover_Init(interp); Tcl_CreateObjCommand( interp, "load_testfixture_extensions", load_testfixture_extensions,0,0 From 6c86783f9a212781f6e69dbf102c1048e1c066f4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 Sep 2022 21:00:39 +0000 Subject: [PATCH 02/42] Further work on making recovery extension compatible with the shell tool ".recover" code. FossilOrigin-Name: 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 --- ext/recover/recover1.test | 2 + ext/recover/sqlite3recover.c | 177 +++++++++++++++++++++++++++++++---- ext/recover/sqlite3recover.h | 3 +- manifest | 19 ++-- manifest.uuid | 2 +- 5 files changed, 171 insertions(+), 32 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index c4348baea0..73832c8a12 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -40,9 +40,11 @@ proc compare_dbs {db1 db2} { proc do_recover_test {tn} { forcedelete test.db2 + forcedelete rstate.db uplevel [list do_test $tn.1 { set R [sqlite3_recover_init db main test.db2] + $R config testdb rstate.db $R step $R finish } {}] diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 6c7828c007..d8b151616b 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -48,6 +48,12 @@ struct RecoverTable { RecoverTable *pNext; }; +typedef struct RecoverBitmap RecoverBitmap; +struct RecoverBitmap { + i64 nPg; /* Size of bitmap */ + u32 aElem[0]; /* Array of 32-bit bitmasks */ +}; + /* ** */ @@ -59,7 +65,7 @@ struct RecoverTable { " pgno, child, PRIMARY KEY(child, pgno)" \ " ) WITHOUT ROWID;" \ " CREATE TABLE recovery.map(" \ -" pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" \ +" pgno INTEGER PRIMARY KEY, parent INT" \ " );" \ " CREATE TABLE recovery.schema(" \ " type, name, tbl_name, rootpage, sql" \ @@ -75,11 +81,14 @@ struct sqlite3_recover { char *zDb; char *zUri; RecoverTable *pTblList; + RecoverBitmap *pUsed; /* Used by recoverWriteLostAndFound() */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ char *zStateDb; + int bLostAndFound; + }; /* @@ -124,6 +133,41 @@ static int recoverError( return errCode; } + +static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ + int nElem = (nPg+1+31) / 32; + int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); + RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte); + + if( pRet ){ + pRet->nPg = nPg; + } + return pRet; +} + +static void recoverBitmapFree(RecoverBitmap *pMap){ + sqlite3_free(pMap); +} + +static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + pMap->aElem[iElem] |= (((u32)1) << iBit); + } +} + +static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ + int ret = 1; + if( iPg<=pMap->nPg ){ + int iElem = (iPg / 32); + int iBit = (iPg % 32); + ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; + } + return ret; +} + + static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); } @@ -194,6 +238,24 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +/* +** Execute "PRAGMA page_count" against the input database. If successful, +** return the integer result. Or, if an error occurs, leave an error code +** and error message in the sqlite3_recover handle. +*/ +static i64 recoverPageCount(sqlite3_recover *p){ + i64 nPg = 0; + if( p->errCode==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); + if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + nPg = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(p, pStmt); + } + return nPg; +} + /* ** The implementation of a user-defined SQL function invoked by the ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages @@ -219,24 +281,23 @@ static void recoverGetPage( assert( nArg==1 ); if( pgno==0 ){ - pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); - }else if( p->pGetPage==0 ){ - pStmt = recoverPreparePrintf( - p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb - ); + sqlite3_result_int64(pCtx, recoverPageCount(p)); + return; }else{ - pStmt = p->pGetPage; - } - - if( pStmt ){ - if( pgno ) sqlite3_bind_int64(pStmt, 1, pgno); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); - } - if( pgno ){ - p->pGetPage = recoverReset(p, pStmt); + if( p->pGetPage==0 ){ + pStmt = recoverPreparePrintf( + p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb + ); }else{ - recoverFinalize(p, pStmt); + pStmt = p->pGetPage; + } + + if( pStmt ){ + sqlite3_bind_int64(pStmt, 1, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + } + p->pGetPage = recoverReset(p, pStmt); } } @@ -311,7 +372,7 @@ static int recoverCacheSchema(sqlite3_recover *p){ "WITH RECURSIVE pages(p) AS (" " SELECT 1" " UNION" - " SELECT child FROM recovery.dbptr, pages WHERE pgno=p" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" ")" "INSERT INTO recovery.schema SELECT" " max(CASE WHEN field=0 THEN value ELSE NULL END)," @@ -504,6 +565,77 @@ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ return pRet; } +static int recoverWriteLostAndFound(sqlite3_recover *p){ + i64 nPg = 0; + RecoverBitmap *pMap = 0; + + nPg = recoverPageCount(p); + pMap = p->pUsed = recoverBitmapAlloc(p, nPg); + if( pMap ){ + sqlite3_stmt *pStmt = 0; + char *zField = 0; + const char *zSep = 0; + int ii; + + sqlite3_stmt *pStmt = recoverPrepare( + p, p->dbOut, + "WITH RECURSIVE used(page) AS (" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + ); + while( pStmt && sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(pStmt); + + pStmt = recoverPreparePrintf( + p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "INSERT INTO recover.map(pgno) " + " SELECT ii FROM seq WHERE !page_is_used(ii)" + ); + sqlite3_step(pStmt); + recoverFinalize(pStmt); + + pStmt = recoverPrepare( + p, p->dbOut, + "UPDATE recover.map SET parent = ptr.pgno " + " FROM sqlite_dbptr('getpage()') WHERE recover.map.pgno=ptr.child" + ); + sqlite3_step(pStmt); + recoverFinalize(pStmt); + + pStmt = recoverPrepare( + p, p->dbOut, + "SELECT max(field) FROM sqlite_dbdata('getpage') WHERE pgno IN (" + " SELECT pgno FROM recover.map" + ")" + ); + if( pStmt && sqlite3_step(pStmt) ){ + nMaxField = sqlite3_column_int64(pStmt, 0); + } + recoverFinalize(pStmt); + + if( nMaxField==0 || p->errCode!=SQLITE_OK ) return p->errCode; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; iierrCode = SQLITE_NOMEM; + } + } + } +} + static int recoverWriteData(sqlite3_recover *p){ RecoverTable *pTbl; int nMax = 0; @@ -522,7 +654,8 @@ static int recoverWriteData(sqlite3_recover *p){ "WITH RECURSIVE pages(root, page) AS (" " SELECT rootpage, rootpage FROM recovery.schema" " UNION" - " SELECT root, child FROM recovery.dbptr, pages WHERE pgno=page" + " SELECT root, child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" ") " "SELECT root, page, cell, field, value " "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " @@ -677,6 +810,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->zStateDb = sqlite3_mprintf("%s", (char*)pArg); break; + case SQLITE_RECOVER_LOST_AND_FOUND: + p->bLostAndFound = (pArg ? 1 : 0); + break; + default: rc = SQLITE_NOTFOUND; break; @@ -695,6 +832,7 @@ static void recoverStep(sqlite3_recover *p){ if( recoverCacheSchema(p) ) return; if( recoverWriteSchema1(p) ) return; if( recoverWriteData(p) ) return; + if( p->bLostAndFound && recoverWriteLostAndFound(p) ) return; if( recoverWriteSchema2(p) ) return; } } @@ -722,6 +860,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ p->pGetPage = 0; rc = p->errCode; + sqlite3_free(p->zStateDb); sqlite3_free(p); return rc; } diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 401f83ea28..0c83f8dea5 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -38,7 +38,8 @@ sqlite3_recover *sqlite3_recover_init( /* Details TBD. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); -#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_LOST_AND_FOUND 790 /* Step the recovery object. Return SQLITE_DONE if recovery is complete, ** SQLITE_OK if recovery is not complete but no error has occurred, or diff --git a/manifest b/manifest index 0fab53a46f..7f8f333f25 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sfiles\sfor\san\sextension\sto\srecover\sdata\sfrom\scorrupted\sdatabases. -D 2022-08-31T20:45:43.730 +C Further\swork\son\smaking\srecovery\sextension\scompatible\swith\sthe\sshell\stool\s".recover"\scode. +D 2022-09-01T21:00:39.747 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,10 +387,10 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 861ad5140566102a8c5a3d1f936a7d6da569f34c86597c274de695f597031bac +F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/sqlite3recover.c 594fb45777a14f0b88b944b9fb2ccb3e85a29ef5b17522b8dac3e3944c4c27ea -F ext/recover/sqlite3recover.h 3255f6491007e57be310aedb72a848c88f79fc14e7222bda4b8d4dab1a2450c3 +F ext/recover/sqlite3recover.c d81b430f968d838035ebf5ca168b43ae8a0bec1e7c2c950b74ec4fd5e16ca47b +F ext/recover/sqlite3recover.h 94e277a9b314a03df46b5e94cc44b70ed6c6893d2776d09c7ea0b55c969ad854 F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 @@ -2004,11 +2004,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5007742886bd20de20be3973737cf46b010359911615eb3da69cd262bd9a2435 -R 563b8320bf923831e4768bc403655fc2 -T *branch * recover-extension -T *sym-recover-extension * -T -sym-trunk * +P f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 +R 185cdcc7bc591773e93e55f9de4bfb4c U dan -Z 1c7612740eb933f84d589533d182c6df +Z 5d96e275b1b3f6a2d23b93e2a033e9c9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 16703e1b2f..5c5b60080f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 \ No newline at end of file +8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 \ No newline at end of file From f2f8a3a348c59d1faabaf9690422fab09002ae32 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2022 20:07:39 +0000 Subject: [PATCH 03/42] Further work on making the recover extension compatible with the .recover command. FossilOrigin-Name: f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 --- ext/misc/dbdata.c | 1 + ext/recover/recoverold.test | 139 ++++++++++++ ext/recover/sqlite3recover.c | 396 +++++++++++++++++++++++++++++------ ext/recover/sqlite3recover.h | 12 ++ ext/recover/test_recover.c | 12 +- manifest | 21 +- manifest.uuid | 2 +- test/permutations.test | 1 + 8 files changed, 503 insertions(+), 81 deletions(-) create mode 100644 ext/recover/recoverold.test diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index b79eafce7c..a18304b96b 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -477,6 +477,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); if( rc!=SQLITE_OK ) return rc; if( pCsr->aPage ) break; + if( pCsr->bOnePage ) return SQLITE_OK; pCsr->iPgno++; } pCsr->iCell = pTab->bPtr ? -2 : 0; diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test new file mode 100644 index 0000000000..61f09397b8 --- /dev/null +++ b/ext/recover/recoverold.test @@ -0,0 +1,139 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoverold + +ifcapable !vtab { + finish_test; return +} + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "sql: $sql" + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn {tsql {}} {res {}}} { + forcedelete test.db2 + forcedelete rstate.db + + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R config testdb rstate.db + $R step + $R finish + + sqlite3 db2 test.db2 + + if {$tsql==""} { + uplevel [list do_test $tn [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn $tsql $res] + } + db2 close +} + +set doc { + hello + world +} +do_execsql_test 1.1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 4, X'1234567800'); + INSERT INTO t1 VALUES(2, 'test', 8.1); + INSERT INTO t1 VALUES(3, $doc, 8.4); +} +do_recover_test 1.1.2 + +do_execsql_test 1.2.1 " + DELETE FROM t1; + INSERT INTO t1 VALUES(13, 'hello\r\nworld', 13); +" +do_recover_test 1.2.2 + +do_execsql_test 1.3.1 " + CREATE TABLE t2(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t2 VALUES(NULL, 1, 2); + INSERT INTO t2 VALUES(NULL, 3, 4); + INSERT INTO t2 VALUES(NULL, 5, 6); + CREATE TABLE t3(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t3 VALUES(NULL, 1, 2); + INSERT INTO t3 VALUES(NULL, 3, 4); + INSERT INTO t3 VALUES(NULL, 5, 6); + DELETE FROM t2; +" +do_recover_test 1.3.2 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1.0 { + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} + +do_recover_test 2.1.1 + +do_execsql_test 2.2.0 { + PRAGMA writable_schema = 1; + DELETE FROM sqlite_master WHERE name='t1'; +} +do_recover_test 2.2.1 { + SELECT name FROM sqlite_master +} {lost_and_found} + +do_execsql_test 2.3.0 { + CREATE TABLE lost_and_found(a, b, c); +} +do_recover_test 2.3.1 { + SELECT name FROM sqlite_master +} {lost_and_found lost_and_found_0} + +do_execsql_test 2.4.0 { + CREATE TABLE lost_and_found_0(a, b, c); +} +do_recover_test 2.4.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + +#------------------------------------------------------------------------- +breakpoint +reset_db +do_recover_test 3.0 + +finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index d8b151616b..ec333f0d05 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -22,6 +22,9 @@ typedef sqlite3_int64 i64; typedef struct RecoverColumn RecoverColumn; struct RecoverColumn { + int iField; /* Field in record on disk */ + int iBind; /* Binding to use in INSERT */ + int bIPK; /* True for IPK column */ char *zCol; int eHidden; }; @@ -35,6 +38,9 @@ struct RecoverColumn { ** When running the ".recover" command, each output table, and the special ** orphaned row table if it is required, is represented by an instance ** of the following struct. +** +** aCol[]: +** Array of nCol columns. In the order in which they appear in the table. */ typedef struct RecoverTable RecoverTable; struct RecoverTable { @@ -43,7 +49,6 @@ struct RecoverTable { int nCol; /* Number of columns in table */ RecoverColumn *aCol; /* Array of columns */ int bIntkey; /* True for intkey, false for without rowid */ - int iPk; /* Index of IPK column, if bIntkey */ RecoverTable *pNext; }; @@ -81,13 +86,13 @@ struct sqlite3_recover { char *zDb; char *zUri; RecoverTable *pTblList; - RecoverBitmap *pUsed; /* Used by recoverWriteLostAndFound() */ + RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ char *zStateDb; - int bLostAndFound; + char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ }; @@ -238,6 +243,21 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +static char *recoverPrintf(sqlite3_recover *p, const char *zFmt, ...){ + va_list ap; + char *z; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( p->errCode==SQLITE_OK ){ + if( z==0 ) p->errCode = SQLITE_NOMEM; + }else{ + sqlite3_free(z); + z = 0; + } + return z; +} + /* ** Execute "PRAGMA page_count" against the input database. If successful, ** return the integer result. Or, if an error occurs, leave an error code @@ -256,6 +276,24 @@ static i64 recoverPageCount(sqlite3_recover *p){ return nPg; } +/* +** SELECT page_is_used(pgno); +*/ +static void recoverPageIsUsed( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); + sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + sqlite3_stmt *pStmt = 0; + int bRet; + + assert( nArg==1 ); + bRet = recoverBitmapQuery(p->pUsed, pgno); + sqlite3_result_int(pCtx, bRet); +} + /* ** The implementation of a user-defined SQL function invoked by the ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages @@ -281,13 +319,14 @@ static void recoverGetPage( assert( nArg==1 ); if( pgno==0 ){ - sqlite3_result_int64(pCtx, recoverPageCount(p)); + i64 nPg = recoverPageCount(p); + sqlite3_result_int64(pCtx, nPg); return; }else{ if( p->pGetPage==0 ){ pStmt = recoverPreparePrintf( p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb - ); + ); }else{ pStmt = p->pGetPage; } @@ -321,6 +360,9 @@ static int recoverOpenOutput(sqlite3_recover *p){ assert( p->dbOut==0 ); rc = sqlite3_open_v2(p->zUri, &db, flags, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "PRAGMA writable_schema = 1", 0, 0, 0); + } if( rc==SQLITE_OK ){ const char *zPath = p->zStateDb ? p->zStateDb : ":memory:"; char *zSql = sqlite3_mprintf("ATTACH %Q AS recovery", zPath); @@ -332,6 +374,7 @@ static int recoverOpenOutput(sqlite3_recover *p){ sqlite3_free(zSql); } + if( rc==SQLITE_OK ){ sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db, "recovery"); if( pBackup ){ @@ -349,6 +392,11 @@ static int recoverOpenOutput(sqlite3_recover *p){ db, "getpage", 1, SQLITE_UTF8, (void*)p, recoverGetPage, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "page_is_used", 1, SQLITE_UTF8, (void*)p, recoverPageIsUsed, 0, 0 + ); + } if( rc!=SQLITE_OK ){ if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); @@ -392,6 +440,7 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ ); if( pStmt ){ + int iPk = -1; RecoverTable *pNew = 0; int nCol = 0; int nName = recoverStrlen(zName); @@ -406,24 +455,37 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ pNew = recoverMalloc(p, nByte); if( pNew ){ int i = 0; + int iField = 0; + int iBind = 1; char *csr = 0; pNew->aCol = (RecoverColumn*)&pNew[1]; pNew->zTab = csr = (char*)&pNew->aCol[nCol]; pNew->nCol = nCol; pNew->iRoot = iRoot; - pNew->iPk = -1; memcpy(csr, zName, nName); csr += nName+1; for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){ - int bPk = sqlite3_column_int(pStmt, 5); + int iPKF = sqlite3_column_int(pStmt, 5); int n = sqlite3_column_bytes(pStmt, 1); const char *z = (const char*)sqlite3_column_text(pStmt, 1); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); int eHidden = sqlite3_column_int(pStmt, 6); - if( bPk ) pNew->iPk = i; + if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i; + if( iPKF>1 ) iPk = -2; pNew->aCol[i].zCol = csr; pNew->aCol[i].eHidden = eHidden; + if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){ + pNew->aCol[i].iField = -1; + }else{ + pNew->aCol[i].iField = iField++; + } + if( eHidden!=RECOVER_EHIDDEN_VIRTUAL + && eHidden!=RECOVER_EHIDDEN_STORED + ){ + pNew->aCol[i].iBind = iBind++; + } memcpy(csr, z, n); csr += (n+1); } @@ -434,18 +496,26 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ recoverFinalize(p, pStmt); - pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_info(%Q)", zName); - if( pStmt && sqlite3_step(pStmt)!=SQLITE_ROW ){ - pNew->bIntkey = 1; - }else{ - pNew->iPk = -1; + pNew->bIntkey = 1; + pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); + while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + int iField = sqlite3_column_int(pStmt, 0); + int iCol = sqlite3_column_int(pStmt, 1); + + assert( iFieldnCol && iColnCol ); + pNew->aCol[iCol].iField = iField; + + pNew->bIntkey = 0; + iPk = -2; } recoverFinalize(p, pStmt); + + if( iPk>=0 ) pNew->aCol[iPk].bIPK = 1; } } /* -** +** */ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; @@ -453,7 +523,8 @@ static int recoverWriteSchema1(sqlite3_recover *p){ pSelect = recoverPrepare(p, p->dbOut, "SELECT rootpage, sql, type='table' FROM recovery.schema " - " WHERE type='table' OR (type='index' AND sql LIKE '%unique%')" + " WHERE type='table' OR (type='index' AND sql LIKE '%unique%') " + " ORDER BY type!='table', name!='sqlite_sequence'" ); pTblname = recoverPrepare(p, p->dbOut, @@ -545,8 +616,9 @@ static sqlite3_stmt *recoverInsertStmt( if( eHidden!=RECOVER_EHIDDEN_VIRTUAL && eHidden!=RECOVER_EHIDDEN_STORED ){ + assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); - zBind = recoverMPrintf(p, "%z%s?", zBind, zSep); + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); zSep = ", "; } } @@ -565,74 +637,268 @@ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ return pRet; } -static int recoverWriteLostAndFound(sqlite3_recover *p){ +/* +** This function attempts to create a lost and found table within the +** output db. If successful, it returns a pointer to a buffer containing +** the name of the new table. It is the responsibility of the caller to +** eventually free this buffer using sqlite3_free(). +** +** If an error occurs, NULL is returned and an error code and error +** message left in the recover handle. +*/ +static char *recoverLostAndFoundCreate( + sqlite3_recover *p, /* Recover object */ + int nField /* Number of column fields in new table */ +){ + char *zTbl = 0; + sqlite3_stmt *pProbe = 0; + int ii = 0; + + pProbe = recoverPrepare(p, p->dbOut, + "SELECT 1 FROM sqlite_schema WHERE name=?" + ); + for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ + int bFail = 0; + if( ii<0 ){ + zTbl = recoverPrintf(p, "%s", p->zLostAndFound); + }else{ + zTbl = recoverPrintf(p, "%s_%d", p->zLostAndFound, ii); + } + + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pProbe) ){ + bFail = 1; + } + recoverReset(p, pProbe); + } + + if( bFail ){ + sqlite3_clear_bindings(pProbe); + sqlite3_free(zTbl); + zTbl = 0; + } + } + recoverFinalize(p, pProbe); + + if( zTbl ){ + const char *zSep = 0; + char *zField = 0; + char *zSql = 0; + + zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; + for(ii=0; p->errCode==SQLITE_OK && iidbOut, zSql); + sqlite3_free(zSql); + }else if( p->errCode==SQLITE_OK ){ + recoverError( + p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound + ); + } + + return zTbl; +} + +/* +** Synthesize and prepare an INSERT statement to write to the lost_and_found +** table in the output database. The name of the table is zTab, and it has +** nField c* fields. +*/ +static sqlite3_stmt *recoverLostAndFoundInsert( + sqlite3_recover *p, + const char *zTab, + int nField +){ + int nTotal = nField + 4; + int ii; + char *zBind = 0; + const char *zSep = ""; + sqlite3_stmt *pRet = 0; + + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + sqlite3_free(zBind); + return pRet; +} + +static void recoverLostAndFoundPopulate( + sqlite3_recover *p, + sqlite3_stmt *pInsert, + int nField +){ + sqlite3_stmt *pStmt = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE pages(root, page) AS (" + " SELECT pgno, pgno FROM recovery.map WHERE parent IS NULL" + " UNION" + " SELECT root, child FROM sqlite_dbptr('getpage()'), pages " + " WHERE pgno=page" + ") " + "SELECT root, page, cell, field, value " + "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " + " AND NOT page_is_used(page) " + "UNION ALL " + "SELECT 0, 0, 0, 0, 0" + ); + + sqlite3_value **apVal = 0; + int nVal = -1; + i64 iRowid = 0; + int bHaveRowid = 0; + int ii; + + i64 iPrevRoot = -1; + i64 iPrevPage = -1; + int iPrevCell = -1; + + apVal = (sqlite3_value**)recoverMalloc(p, nField*sizeof(sqlite3_value*)); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iRoot = sqlite3_column_int64(pStmt, 0); + i64 iPage = sqlite3_column_int64(pStmt, 1); + int iCell = sqlite3_column_int64(pStmt, 2); + int iField = sqlite3_column_int64(pStmt, 3); + + if( iPrevRoot>0 && ( + iPrevRoot!=iRoot || iPrevPage!=iPage || iPrevCell!=iCell + )){ + /* Insert the new row */ + sqlite3_bind_int64(pInsert, 1, iPrevRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPrevPage); /* pgno */ + sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ + if( bHaveRowid ){ + sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ + } + for(ii=0; iipUsed = recoverBitmapAlloc(p, nPg); if( pMap ){ - sqlite3_stmt *pStmt = 0; - char *zField = 0; - const char *zSep = 0; - int ii; + char *zTab = 0; /* Name of lost_and_found table */ + sqlite3_stmt *pInsert = 0; /* INSERT INTO lost_and_found ... */ + int nField = 0; + /* Add all pages that are part of any tree in the recoverable part of + ** the input database schema to the bitmap. */ sqlite3_stmt *pStmt = recoverPrepare( p, p->dbOut, - "WITH RECURSIVE used(page) AS (" + "WITH roots(r) AS (" + " SELECT 1 UNION ALL" " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" " UNION" " SELECT child FROM sqlite_dbptr('getpage()'), used " " WHERE pgno=page" ") " "SELECT page FROM used" ); - while( pStmt && sqlite3_step(pStmt) ){ - i64 iPg = sqlite3_column_int64(pStmt); + while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt, 0); recoverBitmapSet(pMap, iPg); } - recoverFinalize(pStmt); + recoverFinalize(p, pStmt); + /* Add an entry for each page not already added to the bitmap to + ** the recovery.map table. This loop leaves the "parent" column + ** of each recovery.map row set to NULL - to be filled in below. */ pStmt = recoverPreparePrintf( p, p->dbOut, "WITH RECURSIVE seq(ii) AS (" " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" ")" - "INSERT INTO recover.map(pgno) " - " SELECT ii FROM seq WHERE !page_is_used(ii)" + "INSERT INTO recovery.map(pgno) " + " SELECT ii FROM seq WHERE NOT page_is_used(ii)", nPg ); sqlite3_step(pStmt); - recoverFinalize(pStmt); + recoverFinalize(p, pStmt); + /* Set the "parent" column for each row of the recovery.map table */ pStmt = recoverPrepare( p, p->dbOut, - "UPDATE recover.map SET parent = ptr.pgno " - " FROM sqlite_dbptr('getpage()') WHERE recover.map.pgno=ptr.child" + "UPDATE recovery.map SET parent = ptr.pgno " + " FROM sqlite_dbptr('getpage()') AS ptr " + " WHERE recovery.map.pgno=ptr.child" ); sqlite3_step(pStmt); - recoverFinalize(pStmt); + recoverFinalize(p, pStmt); + /* Figure out the number of fields in the longest record that will be + ** recovered into the lost_and_found table. Set nField to this value. */ pStmt = recoverPrepare( p, p->dbOut, - "SELECT max(field) FROM sqlite_dbdata('getpage') WHERE pgno IN (" - " SELECT pgno FROM recover.map" + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno IN (" + " SELECT pgno FROM recovery.map" ")" ); - if( pStmt && sqlite3_step(pStmt) ){ - nMaxField = sqlite3_column_int64(pStmt, 0); + if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + nField = sqlite3_column_int64(pStmt, 0); } - recoverFinalize(pStmt); + recoverFinalize(p, pStmt); - if( nMaxField==0 || p->errCode!=SQLITE_OK ) return p->errCode; - - zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, "; - for(ii=0; iierrCode = SQLITE_NOMEM; - } + if( nField>0 ){ + zTab = recoverLostAndFoundCreate(p, nField); + pInsert = recoverLostAndFoundInsert(p, zTab, nField); + recoverLostAndFoundPopulate(p, pInsert, nField); + recoverFinalize(p, pInsert); + sqlite3_free(zTab); } + + recoverBitmapFree(pMap); + p->pUsed = 0; } } @@ -647,7 +913,7 @@ static int recoverWriteData(sqlite3_recover *p){ if( pTbl->nCol>nMax ) nMax = pTbl->nCol; } - apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * nMax); + apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * (nMax+1)); if( apVal==0 ) return p->errCode; pSel = recoverPrepare(p, p->dbOut, @@ -697,26 +963,15 @@ static int recoverWriteData(sqlite3_recover *p){ nInsert = nVal; } - for(ii=0; iinCol && iValaCol[ii].eHidden; - switch( eHidden ){ - case RECOVER_EHIDDEN_NONE: - case RECOVER_EHIDDEN_HIDDEN: - if( ii==pTab->iPk ){ - sqlite3_bind_int64(pInsert, iBind, iRowid); - }else{ - sqlite3_bind_value(pInsert, iBind, apVal[iVal]); - } - iBind++; - iVal++; - break; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; - case RECOVER_EHIDDEN_VIRTUAL: - break; - - case RECOVER_EHIDDEN_STORED: - iVal++; - break; + if( pCol->iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, pCol->iBind, iRowid); + }else if( pCol->iFieldiBind, apVal[pCol->iField]); + } } } @@ -811,7 +1066,13 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ break; case SQLITE_RECOVER_LOST_AND_FOUND: - p->bLostAndFound = (pArg ? 1 : 0); + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } break; default: @@ -832,7 +1093,7 @@ static void recoverStep(sqlite3_recover *p){ if( recoverCacheSchema(p) ) return; if( recoverWriteSchema1(p) ) return; if( recoverWriteData(p) ) return; - if( p->bLostAndFound && recoverWriteLostAndFound(p) ) return; + if( p->zLostAndFound && recoverLostAndFound(p) ) return; if( recoverWriteSchema2(p) ) return; } } @@ -861,6 +1122,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ rc = p->errCode; sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); sqlite3_free(p); return rc; } diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 0c83f8dea5..8306c67a12 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -38,6 +38,18 @@ sqlite3_recover *sqlite3_recover_init( /* Details TBD. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); +/* +** SQLITE_RECOVER_TESTDB: +** +** +** SQLITE_RECOVER_LOST_AND_FOUND: +** The pArg argument points to a string buffer containing the name +** of a "lost-and-found" table in the output database, or NULL. If +** the argument is non-NULL and the database contains seemingly +** valid pages that cannot be associated with any table in the +** recovered part of the schema, data is extracted from these +** pages to add to the lost-and-found table. +*/ #define SQLITE_RECOVER_TESTDB 789 #define SQLITE_RECOVER_LOST_AND_FOUND 790 diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 912b8dec5c..cdd5d090fa 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -79,7 +79,8 @@ static int testRecoverCmd( switch( iSub ){ case 0: assert( sqlite3_stricmp("config", aSub[iSub].zSub)==0 ); { const char *aOp[] = { - "testdb", /* 0 */ + "testdb", /* 0 */ + "lostandfound", /* 1 */ 0 }; int iOp = 0; @@ -89,8 +90,13 @@ static int testRecoverCmd( } switch( iOp ){ case 0: - res = sqlite3_recover_config( - pTest->p, SQLITE_RECOVER_TESTDB, (void*)Tcl_GetString(objv[3]) + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_TESTDB, (void*)Tcl_GetString(objv[3]) + ); + break; + case 1: + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_LOST_AND_FOUND, (void*)Tcl_GetString(objv[3]) ); break; } diff --git a/manifest b/manifest index 7f8f333f25..10281dc64e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\swork\son\smaking\srecovery\sextension\scompatible\swith\sthe\sshell\stool\s".recover"\scode. -D 2022-09-01T21:00:39.747 +C Further\swork\son\smaking\sthe\srecover\sextension\scompatible\swith\sthe\s.recover\scommand. +D 2022-09-03T20:07:39.011 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c f317980cea788e67932828b94a16ee8a8b859e3c2d62859d09ba3d5ca85f87cb +F ext/misc/dbdata.c 9bb3666519bd8a54cce4934076a557fe6441c5bafce7e9c24d8b5ced148e8154 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -389,9 +389,10 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/sqlite3recover.c d81b430f968d838035ebf5ca168b43ae8a0bec1e7c2c950b74ec4fd5e16ca47b -F ext/recover/sqlite3recover.h 94e277a9b314a03df46b5e94cc44b70ed6c6893d2776d09c7ea0b55c969ad854 -F ext/recover/test_recover.c 919f61df54776598b350250057fd2d3ea9cc2cef1aeac0dbb760958d26fe1afb +F ext/recover/recoverold.test 33ccbe2393af0e82f292c135b725e3eca1e803960681cf6da41fc00d28bd8683 +F ext/recover/sqlite3recover.c 8d93b9aa056c3fae9a5e2736a4ffa71414bdb502863ef879e55bec7b37030266 +F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b +F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -1341,7 +1342,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 909c84575ac50f6d30fa125a109a01986e08c26b9ea38d28501a0711b50b5627 +F test/permutations.test 847df2d81f0172ab7032e55145f0f3da460dd65759ac2b02864e385add1947d5 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test cae534c12a033a5c319ccc94f50b32811acdef9f67bf19a82ff42697caccd69f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -2004,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f8298eeba01cb5b02ac4d642c06f3801331ca90edea533ea898a3283981a9e49 -R 185cdcc7bc591773e93e55f9de4bfb4c +P 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 +R 93788fe8081a2011cde8fac3142d518f U dan -Z 5d96e275b1b3f6a2d23b93e2a033e9c9 +Z 21514150d144c868f0bbea27057fe308 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5c5b60080f..093f196c7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 \ No newline at end of file +f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index c5044e1a6d..8ab9dc8755 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -91,6 +91,7 @@ foreach f [glob -nocomplain \ $testdir/../ext/fts5/test/*.test \ $testdir/../ext/expert/*.test \ $testdir/../ext/lsm1/test/*.test \ + $testdir/../ext/recover/*.test \ ] { lappend alltests $f } From be2e212cf93f6bae68dc75ed0aa9f2b07988cd3f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2022 20:31:36 +0000 Subject: [PATCH 04/42] Take the freelist into account when recovering data that is not linked in to any tree associated with a schema entry. FossilOrigin-Name: dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a --- ext/recover/recoverold.test | 17 ++++++++++ ext/recover/sqlite3recover.c | 64 ++++++++++++++++++++++++++++++++++-- manifest | 14 ++++---- manifest.uuid | 2 +- 4 files changed, 86 insertions(+), 11 deletions(-) diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 61f09397b8..001426b88d 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -131,6 +131,23 @@ do_recover_test 2.4.1 { 2 2 3 {} 8 9 7 } +do_execsql_test 2.5 { + CREATE TABLE x1(a, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO x1 SELECT i, i, hex(randomblob(500)) FROM s; + DROP TABLE x1; +} +do_recover_test 2.5.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + #------------------------------------------------------------------------- breakpoint reset_db diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index ec333f0d05..34cc0cdd50 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -107,7 +107,7 @@ static int recoverStrlen(const char *zStr){ return nRet; } -static void *recoverMalloc(sqlite3_recover *p, sqlite3_int64 nByte){ +static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ void *pRet = 0; assert( nByte>0 ); if( p->errCode==SQLITE_OK ){ @@ -276,6 +276,36 @@ static i64 recoverPageCount(sqlite3_recover *p){ return nPg; } +/* +** Scalar function "read_i32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void recoverReadI32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + assert( argc==2 ); + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + i64 iVal = ((i64)a[0]<<24) + + ((i64)a[1]<<16) + + ((i64)a[2]<< 8) + + ((i64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + /* ** SELECT page_is_used(pgno); */ @@ -285,7 +315,7 @@ static void recoverPageIsUsed( sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + i64 pgno = sqlite3_value_int64(apArg[0]); sqlite3_stmt *pStmt = 0; int bRet; @@ -314,7 +344,7 @@ static void recoverGetPage( sqlite3_value **apArg ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); - sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]); + i64 pgno = sqlite3_value_int64(apArg[0]); sqlite3_stmt *pStmt = 0; assert( nArg==1 ); @@ -397,6 +427,11 @@ static int recoverOpenOutput(sqlite3_recover *p){ db, "page_is_used", 1, SQLITE_UTF8, (void*)p, recoverPageIsUsed, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "read_i32", 2, SQLITE_UTF8, (void*)p, recoverReadI32, 0, 0 + ); + } if( rc!=SQLITE_OK ){ if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); @@ -852,6 +887,29 @@ static int recoverLostAndFound(sqlite3_recover *p){ } recoverFinalize(p, pStmt); + /* Add all pages that appear to be part of the freelist to the bitmap. */ + pStmt = recoverPrepare(p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")" + "SELECT freepgno FROM freelist" + ); + while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt, 0); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(p, pStmt); + /* Add an entry for each page not already added to the bitmap to ** the recovery.map table. This loop leaves the "parent" column ** of each recovery.map row set to NULL - to be filled in below. */ diff --git a/manifest b/manifest index 10281dc64e..00fb244d2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\swork\son\smaking\sthe\srecover\sextension\scompatible\swith\sthe\s.recover\scommand. -D 2022-09-03T20:07:39.011 +C Take\sthe\sfreelist\sinto\saccount\swhen\srecovering\sdata\sthat\sis\snot\slinked\sin\sto\sany\stree\sassociated\swith\sa\sschema\sentry. +D 2022-09-03T20:31:36.832 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,8 +389,8 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverold.test 33ccbe2393af0e82f292c135b725e3eca1e803960681cf6da41fc00d28bd8683 -F ext/recover/sqlite3recover.c 8d93b9aa056c3fae9a5e2736a4ffa71414bdb502863ef879e55bec7b37030266 +F ext/recover/recoverold.test 7578e9b938db15dc469a4af247e15866226f366bde0cbe09a40b0aef4a0506c8 +F ext/recover/sqlite3recover.c 6c9cbc993a970060f9fb881d78f6c7e182ec988a5e48acbf15bb4a5f05ce2902 F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759 -R 93788fe8081a2011cde8fac3142d518f +P f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 +R 471b646b541e0fcab850e84cb036ac46 U dan -Z 21514150d144c868f0bbea27057fe308 +Z 9f579b130a06a2078244b88aebbd5365 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 093f196c7a..f01f3f35e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 \ No newline at end of file +dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a \ No newline at end of file From 7302079dbe01c7b136ccc0853816bd7cc4319dde Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Sep 2022 21:08:38 +0000 Subject: [PATCH 05/42] Add an option to assume the freelist is corrupt when recovering data. FossilOrigin-Name: 253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c --- ext/recover/recoverold.test | 13 +++++++++- ext/recover/sqlite3recover.c | 48 ++++++++++++++++++++---------------- ext/recover/sqlite3recover.h | 16 +++++++++--- ext/recover/test_recover.c | 9 +++++++ manifest | 18 +++++++------- manifest.uuid | 2 +- 6 files changed, 71 insertions(+), 35 deletions(-) diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 001426b88d..8f70a7473d 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -47,7 +47,6 @@ proc do_recover_test {tn {tsql {}} {res {}}} { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found - $R config testdb rstate.db $R step $R finish @@ -148,6 +147,18 @@ do_recover_test 2.5.1 { 2 2 3 {} 8 9 7 } +do_test 2.6 { + forcedelete test.db2 + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R config freelistcorrupt 1 + $R step + $R finish + sqlite3 db2 test.db2 + execsql { SELECT count(*) FROM lost_and_found_1; } db2 +} {103} +db2 close + #------------------------------------------------------------------------- breakpoint reset_db diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 34cc0cdd50..14cc29e476 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -93,7 +93,7 @@ struct sqlite3_recover { char *zStateDb; char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ - + int bFreelistCorrupt; }; /* @@ -888,27 +888,29 @@ static int recoverLostAndFound(sqlite3_recover *p){ recoverFinalize(p, pStmt); /* Add all pages that appear to be part of the freelist to the bitmap. */ - pStmt = recoverPrepare(p, p->dbOut, - "WITH trunk(pgno) AS (" - " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" - " UNION" - " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" - ")," - "trunkdata(pgno, data) AS (" - " SELECT pgno, getpage(pgno) FROM trunk" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" - " UNION ALL" - " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" - ")" - "SELECT freepgno FROM freelist" - ); - while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iPg = sqlite3_column_int64(pStmt, 0); - recoverBitmapSet(pMap, iPg); + if( p->bFreelistCorrupt==0 ){ + pStmt = recoverPrepare(p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")" + "SELECT freepgno FROM freelist" + ); + while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + i64 iPg = sqlite3_column_int64(pStmt, 0); + recoverBitmapSet(pMap, iPg); + } + recoverFinalize(p, pStmt); } - recoverFinalize(p, pStmt); /* Add an entry for each page not already added to the bitmap to ** the recovery.map table. This loop leaves the "parent" column @@ -1133,6 +1135,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ } break; + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = (pArg ? 1 : 0); + break; + default: rc = SQLITE_NOTFOUND; break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 8306c67a12..638a8b1c92 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -41,7 +41,6 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); /* ** SQLITE_RECOVER_TESTDB: ** -** ** SQLITE_RECOVER_LOST_AND_FOUND: ** The pArg argument points to a string buffer containing the name ** of a "lost-and-found" table in the output database, or NULL. If @@ -49,9 +48,20 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** valid pages that cannot be associated with any table in the ** recovered part of the schema, data is extracted from these ** pages to add to the lost-and-found table. +** +** SQLITE_RECOVER_FREELIST_CORRUPT: +** The pArg value must actually be integer (type "int") value 0 or 1 +** cast as a (void*). If this option is set (argument is 1) and +** a lost-and-found table has been configured using +** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is +** corrupt and an attempt is made to recover records from pages that +** appear to be linked into the freelist. Otherwise, pages on the freelist +** are ignored. Setting this option can recover more data from the +** database, but often ends up "recovering" deleted records. */ -#define SQLITE_RECOVER_TESTDB 789 -#define SQLITE_RECOVER_LOST_AND_FOUND 790 +#define SQLITE_RECOVER_TESTDB 789 +#define SQLITE_RECOVER_LOST_AND_FOUND 790 +#define SQLITE_RECOVER_FREELIST_CORRUPT 791 /* Step the recovery object. Return SQLITE_DONE if recovery is complete, ** SQLITE_OK if recovery is not complete but no error has occurred, or diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index cdd5d090fa..835a203f66 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -81,6 +81,7 @@ static int testRecoverCmd( const char *aOp[] = { "testdb", /* 0 */ "lostandfound", /* 1 */ + "freelistcorrupt", /* 2 */ 0 }; int iOp = 0; @@ -99,6 +100,14 @@ static int testRecoverCmd( SQLITE_RECOVER_LOST_AND_FOUND, (void*)Tcl_GetString(objv[3]) ); break; + case 2: { + int iVal = 0; + if( Tcl_GetIntFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_FREELIST_CORRUPT, (void*)iVal + ); + break; + } } Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; diff --git a/manifest b/manifest index 00fb244d2e..04d150cbd7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\sthe\sfreelist\sinto\saccount\swhen\srecovering\sdata\sthat\sis\snot\slinked\sin\sto\sany\stree\sassociated\swith\sa\sschema\sentry. -D 2022-09-03T20:31:36.832 +C Add\san\soption\sto\sassume\sthe\sfreelist\sis\scorrupt\swhen\srecovering\sdata. +D 2022-09-03T21:08:38.958 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,10 +389,10 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverold.test 7578e9b938db15dc469a4af247e15866226f366bde0cbe09a40b0aef4a0506c8 -F ext/recover/sqlite3recover.c 6c9cbc993a970060f9fb881d78f6c7e182ec988a5e48acbf15bb4a5f05ce2902 -F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b -F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2 +F ext/recover/recoverold.test e7e00c78ec35b60488369ddf99e36a3b30e686566571969b05781e5063bdffe8 +F ext/recover/sqlite3recover.c 395c9f623cf84bd8c2e651ec112898d81e1908bbda66fe5f0efcfaa85ad2b262 +F ext/recover/sqlite3recover.h 35aacde3b3834d8ceefb20a2cf0ba221cbb5d802efc11a0529aafc018c462e13 +F ext/recover/test_recover.c 112c580e7cd765a20bbc94998f8b43b629db47fa6bfd696484ca722e418f4172 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2 -R 471b646b541e0fcab850e84cb036ac46 +P dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a +R f1692b1a741d9ca9573411dcac80d98d U dan -Z 9f579b130a06a2078244b88aebbd5365 +Z 9a22d599f360a7b8179802a74accdf31 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f01f3f35e4..9852bbbed1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a \ No newline at end of file +253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c \ No newline at end of file From a768b67dcc5880f7078de0135838e0d0a0e7334f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Sep 2022 15:56:09 +0000 Subject: [PATCH 06/42] Add the SQLITE_RECOVER_ROWIDS option. To specify that rowid values that are not also explicit INTEGER PRIMARY KEY values should be preserved. FossilOrigin-Name: 69cc9aba56a196bbd159bd24868aa5ccc60bed0dc612d09ed8a3ae898f156809 --- ext/recover/sqlite3recover.c | 27 +++++++++++++++++++++++++-- ext/recover/sqlite3recover.h | 3 +++ ext/recover/test_recover.c | 11 ++++++++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 14cc29e476..e0eda09d82 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -49,6 +49,7 @@ struct RecoverTable { int nCol; /* Number of columns in table */ RecoverColumn *aCol; /* Array of columns */ int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ RecoverTable *pNext; }; @@ -94,6 +95,7 @@ struct sqlite3_recover { char *zStateDb; char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; + int bRecoverRowid; }; /* @@ -476,6 +478,7 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ if( pStmt ){ int iPk = -1; + int iBind = 1; RecoverTable *pNew = 0; int nCol = 0; int nName = recoverStrlen(zName); @@ -491,7 +494,6 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ if( pNew ){ int i = 0; int iField = 0; - int iBind = 1; char *csr = 0; pNew->aCol = (RecoverColumn*)&pNew[1]; pNew->zTab = csr = (char*)&pNew->aCol[nCol]; @@ -545,7 +547,11 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ } recoverFinalize(p, pStmt); - if( iPk>=0 ) pNew->aCol[iPk].bIPK = 1; + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } } } @@ -646,6 +652,13 @@ static sqlite3_stmt *recoverInsertStmt( assert( nField<=pTab->nCol ); zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab); + + if( pTab->iRowidBind ){ + assert( pTab->bIntkey ); + zSql = recoverMPrintf(p, "%z_rowid_", zSql); + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + zSep = ", "; + } for(ii=0; iiaCol[ii].eHidden; if( eHidden!=RECOVER_EHIDDEN_VIRTUAL @@ -995,6 +1008,7 @@ static int recoverWriteData(sqlite3_recover *p){ i64 iPrevRoot = -1; i64 iPrevPage = -1; int iPrevCell = -1; + int bHaveRowid = 0; /* True if iRowid is valid */ i64 iRowid = 0; int nVal = -1; @@ -1034,6 +1048,9 @@ static int recoverWriteData(sqlite3_recover *p){ } } } + if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); + } sqlite3_step(pInsert); recoverReset(p, pInsert); @@ -1046,6 +1063,7 @@ static int recoverWriteData(sqlite3_recover *p){ apVal[ii] = 0; } nVal = -1; + bHaveRowid = 0; } if( iRoot==0 ) continue; @@ -1061,6 +1079,7 @@ static int recoverWriteData(sqlite3_recover *p){ iRowid = sqlite3_column_int64(pSel, 4); assert( nVal==-1 ); nVal = 0; + bHaveRowid = 1; }else if( iFieldbFreelistCorrupt = (pArg ? 1 : 0); break; + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = (pArg ? 1 : 0); + break; + default: rc = SQLITE_NOTFOUND; break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 638a8b1c92..0294baddaa 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -58,10 +58,13 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** appear to be linked into the freelist. Otherwise, pages on the freelist ** are ignored. Setting this option can recover more data from the ** database, but often ends up "recovering" deleted records. +** +** SQLITE_RECOVER_ROWIDS: */ #define SQLITE_RECOVER_TESTDB 789 #define SQLITE_RECOVER_LOST_AND_FOUND 790 #define SQLITE_RECOVER_FREELIST_CORRUPT 791 +#define SQLITE_RECOVER_ROWIDS 792 /* Step the recovery object. Return SQLITE_DONE if recovery is complete, ** SQLITE_OK if recovery is not complete but no error has occurred, or diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 835a203f66..0646ff78b8 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -82,6 +82,7 @@ static int testRecoverCmd( "testdb", /* 0 */ "lostandfound", /* 1 */ "freelistcorrupt", /* 2 */ + "rowids", /* 3 */ 0 }; int iOp = 0; @@ -102,12 +103,20 @@ static int testRecoverCmd( break; case 2: { int iVal = 0; - if( Tcl_GetIntFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; res = sqlite3_recover_config(pTest->p, SQLITE_RECOVER_FREELIST_CORRUPT, (void*)iVal ); break; } + case 3: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_ROWIDS, (void*)iVal + ); + break; + } } Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; diff --git a/manifest b/manifest index 04d150cbd7..9f01e7c919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\soption\sto\sassume\sthe\sfreelist\sis\scorrupt\swhen\srecovering\sdata. -D 2022-09-03T21:08:38.958 +C Add\sthe\sSQLITE_RECOVER_ROWIDS\soption.\sTo\sspecify\sthat\srowid\svalues\sthat\sare\snot\salso\sexplicit\sINTEGER\sPRIMARY\sKEY\svalues\sshould\sbe\spreserved. +D 2022-09-05T15:56:09.391 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,9 +390,9 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282 F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverold.test e7e00c78ec35b60488369ddf99e36a3b30e686566571969b05781e5063bdffe8 -F ext/recover/sqlite3recover.c 395c9f623cf84bd8c2e651ec112898d81e1908bbda66fe5f0efcfaa85ad2b262 -F ext/recover/sqlite3recover.h 35aacde3b3834d8ceefb20a2cf0ba221cbb5d802efc11a0529aafc018c462e13 -F ext/recover/test_recover.c 112c580e7cd765a20bbc94998f8b43b629db47fa6bfd696484ca722e418f4172 +F ext/recover/sqlite3recover.c 13e78c8a9d5521e06ebe5ac992a90169155e685f5c4b3cebc632c50b41e061c9 +F ext/recover/sqlite3recover.h ed34bc96befdf581a7de039d99772caabf0b338eb2a841d869acd9f7893ffce7 +F ext/recover/test_recover.c 9b8144ac94e6a2f3aabfcd24db6416179afdd32a3d654d1ef603c570e0384b2f F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a -R f1692b1a741d9ca9573411dcac80d98d +P 253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c +R e5ada01eb624b55d2f64c4994830b06d U dan -Z 9a22d599f360a7b8179802a74accdf31 +Z a7e3c99975f0369d94d22f82581262dd # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9852bbbed1..f34597368d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c \ No newline at end of file +69cc9aba56a196bbd159bd24868aa5ccc60bed0dc612d09ed8a3ae898f156809 \ No newline at end of file From d5e3fe50f8f7d3aa01cf06ad7815dbf5bec1548f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Sep 2022 21:00:22 +0000 Subject: [PATCH 07/42] Add a mode to output SQL statements instead of populating a database to the recover extension. FossilOrigin-Name: 73058416e7da6581000898b7988a7010e2ce6632246f4c12b4398700c7744b83 --- ext/recover/recover1.test | 20 ++++++++ ext/recover/recoverold.test | 28 ++++++++++- ext/recover/sqlite3recover.c | 96 ++++++++++++++++++++++++++++-------- ext/recover/sqlite3recover.h | 12 ++++- ext/recover/test_recover.c | 44 +++++++++++++++-- manifest | 20 ++++---- manifest.uuid | 2 +- 7 files changed, 182 insertions(+), 40 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 73832c8a12..167b2796bd 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -52,8 +52,27 @@ proc do_recover_test {tn} { sqlite3 db2 test.db2 uplevel [list do_test $tn.2 [list compare_dbs db db2] {}] db2 close + + forcedelete test.db2 + forcedelete rstate.db + + uplevel [list do_test $tn.3 { + set ::sqlhook [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config testdb rstate.db + $R step + $R finish + } {}] + + sqlite3 db2 test.db2 + execsql [join $::sqlhook ";"] db2 + uplevel [list do_test $tn.4 [list compare_dbs db db2] {}] + db2 close } +proc my_sql_hook {sql} { + lappend ::sqlhook $sql +} do_execsql_test 1.0 { CREATE TABLE t1(a INTEGER PRIMARY KEY, b); @@ -112,6 +131,7 @@ do_execsql_test 7.1 { SELECT * FROM t2 } {10 11 ten} +breakpoint do_recover_test 7.2 finish_test diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 8f70a7473d..83f210aebc 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -53,13 +53,37 @@ proc do_recover_test {tn {tsql {}} {res {}}} { sqlite3 db2 test.db2 if {$tsql==""} { - uplevel [list do_test $tn [list compare_dbs db db2] {}] + uplevel [list do_test $tn.1 [list compare_dbs db db2] {}] } else { - uplevel [list do_execsql_test -db db2 $tn $tsql $res] + uplevel [list do_execsql_test -db db2 $tn.1 $tsql $res] + } + db2 close + + forcedelete test.db2 + forcedelete rstate.db + + set ::sqlhook [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config lostandfound lost_and_found + $R step + $R finish + + sqlite3 db2 test.db2 + db2 eval [join $::sqlhook ";"] + + if {$tsql==""} { + uplevel [list do_test $tn.sql [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn.sql $tsql $res] } db2 close } +proc my_sql_hook {sql} { + lappend ::sqlhook $sql +} + + set doc { hello world diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index e0eda09d82..1014e576a4 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -64,12 +64,6 @@ struct RecoverBitmap { ** */ #define RECOVERY_SCHEMA \ -" CREATE TABLE recovery.freelist(" \ -" pgno INTEGER PRIMARY KEY" \ -" );" \ -" CREATE TABLE recovery.dbptr(" \ -" pgno, child, PRIMARY KEY(child, pgno)" \ -" ) WITHOUT ROWID;" \ " CREATE TABLE recovery.map(" \ " pgno INTEGER PRIMARY KEY, parent INT" \ " );" \ @@ -96,6 +90,9 @@ struct sqlite3_recover { char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; int bRecoverRowid; + + void *pSqlCtx; + int (*xSql)(void*,const char*); }; /* @@ -445,13 +442,6 @@ static int recoverOpenOutput(sqlite3_recover *p){ return rc; } -static int recoverCacheDbptr(sqlite3_recover *p){ - return recoverExec(p, p->dbOut, - "INSERT INTO recovery.dbptr " - "SELECT pgno, child FROM sqlite_dbptr('getpage()')" - ); -} - static int recoverCacheSchema(sqlite3_recover *p){ return recoverExec(p, p->dbOut, "WITH RECURSIVE pages(p) AS (" @@ -555,6 +545,15 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ } } +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); + } + } +} + /* ** */ @@ -581,6 +580,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); if( bTable ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zName = sqlite3_column_text(pTblname, 0); @@ -614,6 +614,8 @@ static int recoverWriteSchema2(sqlite3_recover *p){ int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc!=SQLITE_OK && rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); + }else if( rc==SQLITE_OK ){ + recoverSqlCallback(p, zSql); } } } @@ -644,9 +646,12 @@ static sqlite3_stmt *recoverInsertStmt( int nField ){ const char *zSep = ""; + const char *zSqlSep = ""; char *zSql = 0; + char *zFinal = 0; char *zBind = 0; int ii; + int bSql = p->xSql ? 1 : 0; sqlite3_stmt *pRet = 0; assert( nField<=pTab->nCol ); @@ -656,9 +661,16 @@ static sqlite3_stmt *recoverInsertStmt( if( pTab->iRowidBind ){ assert( pTab->bIntkey ); zSql = recoverMPrintf(p, "%z_rowid_", zSql); - zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + if( bSql ){ + zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind); + }else{ + zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind); + } + zSqlSep = "||', '||"; zSep = ", "; } + + for(ii=0; iiaCol[ii].eHidden; if( eHidden!=RECOVER_EHIDDEN_VIRTUAL @@ -666,14 +678,31 @@ static sqlite3_stmt *recoverInsertStmt( ){ assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 ); zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); - zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + + if( bSql ){ + zBind = recoverMPrintf( + p, "%z%squote(?%d)", zBind, zSqlSep, pTab->aCol[ii].iBind + ); + zSqlSep = "||', '||"; + }else{ + zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind); + } zSep = ", "; } } - zSql = recoverMPrintf(p, "%z) VALUES (%z)", zSql, zBind); - pRet = recoverPrepare(p, p->dbOut, zSql); + if( bSql ){ + zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", + zSql, zBind + ); + }else{ + zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind); + } + + pRet = recoverPrepare(p, p->dbOut, zFinal); sqlite3_free(zSql); + sqlite3_free(zBind); + sqlite3_free(zFinal); return pRet; } @@ -744,6 +773,7 @@ static char *recoverLostAndFoundCreate( sqlite3_free(zField); recoverExec(p, p->dbOut, zSql); + recoverSqlCallback(p, zSql); sqlite3_free(zSql); }else if( p->errCode==SQLITE_OK ){ recoverError( @@ -838,6 +868,7 @@ static void recoverLostAndFoundPopulate( sqlite3_value_free(apVal[ii]); apVal[ii] = 0; } + sqlite3_clear_bindings(pInsert); bHaveRowid = 0; nVal = -1; } @@ -1052,7 +1083,10 @@ static int recoverWriteData(sqlite3_recover *p){ sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); } - sqlite3_step(pInsert); + if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ + const char *zSql = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, zSql); + } recoverReset(p, pInsert); assert( p->errCode || pInsert ); if( pInsert ) sqlite3_clear_bindings(pInsert); @@ -1098,10 +1132,12 @@ static int recoverWriteData(sqlite3_recover *p){ return p->errCode; } -sqlite3_recover *sqlite3_recover_init( +sqlite3_recover *recoverInit( sqlite3* db, const char *zDb, - const char *zUri + const char *zUri, + int (*xSql)(void*, const char*), + void *pSqlCtx ){ sqlite3_recover *pRet = 0; int nDb = 0; @@ -1123,11 +1159,30 @@ sqlite3_recover *sqlite3_recover_init( pRet->zUri = &pRet->zDb[nDb+1]; memcpy(pRet->zDb, zDb, nDb); memcpy(pRet->zUri, zUri, nUri); + pRet->xSql = xSql; + pRet->pSqlCtx = pSqlCtx; } return pRet; } +sqlite3_recover *sqlite3_recover_init( + sqlite3* db, + const char *zDb, + const char *zUri +){ + return recoverInit(db, zDb, zUri, 0, 0); +} + +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pSqlCtx +){ + return recoverInit(db, zDb, "", xSql, pSqlCtx); +} + const char *sqlite3_recover_errmsg(sqlite3_recover *p){ return p ? p->zErrMsg : "not an error"; } @@ -1176,7 +1231,6 @@ static void recoverStep(sqlite3_recover *p){ if( p->dbOut==0 ){ if( recoverOpenOutput(p) ) return; - if( recoverCacheDbptr(p) ) return; if( recoverCacheSchema(p) ) return; if( recoverWriteSchema1(p) ) return; if( recoverWriteData(p) ) return; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 0294baddaa..5dc2a56d30 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -24,7 +24,8 @@ extern "C" { typedef struct sqlite3_recover sqlite3_recover; -/* Create an object to recover data from database zDb (e.g. "main") +/* +** Create an object to recover data from database zDb (e.g. "main") ** opened by handle db. Data will be recovered into the database ** identified by parameter zUri. Database zUri is clobbered if it ** already exists. @@ -35,6 +36,13 @@ sqlite3_recover *sqlite3_recover_init( const char *zUri ); +sqlite3_recover *sqlite3_recover_init_sql( + sqlite3* db, + const char *zDb, + int (*xSql)(void*, const char*), + void *pCtx +); + /* Details TBD. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); @@ -60,6 +68,8 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** database, but often ends up "recovering" deleted records. ** ** SQLITE_RECOVER_ROWIDS: +** +** SQLITE_RECOVER_SQLHOOK: */ #define SQLITE_RECOVER_TESTDB 789 #define SQLITE_RECOVER_LOST_AND_FOUND 790 diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 0646ff78b8..acfcf8c7a3 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -20,8 +20,31 @@ typedef struct TestRecover TestRecover; struct TestRecover { sqlite3_recover *p; + Tcl_Interp *interp; + Tcl_Obj *pScript; }; +static int xSqlCallback(void *pSqlArg, const char *zSql){ + TestRecover *p = (TestRecover*)pSqlArg; + Tcl_Obj *pEval = 0; + int res = 0; + + pEval = Tcl_DuplicateObj(p->pScript); + Tcl_IncrRefCount(pEval); + + res = Tcl_ListObjAppendElement(p->interp, pEval, Tcl_NewStringObj(zSql, -1)); + if( res==TCL_OK ){ + res = Tcl_EvalObjEx(p->interp, pEval, 0); + } + + Tcl_DecrRefCount(pEval); + if( res ){ + Tcl_BackgroundError(p->interp); + return TCL_ERROR; + } + return SQLITE_OK; +} + static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ Tcl_CmdInfo info; if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){ @@ -171,17 +194,26 @@ static int test_sqlite3_recover_init( const char *zDb = 0; const char *zUri = 0; char zCmd[128]; + int bSql = clientData ? 1 : 0; if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME URI"); + const char *zErr = (bSql ? "DB DBNAME SCRIPT" : "DB DBNAME URI"); + Tcl_WrongNumArgs(interp, 1, objv, zErr); return TCL_ERROR; } if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; zDb = Tcl_GetString(objv[2]); - zUri = Tcl_GetString(objv[3]); pNew = ckalloc(sizeof(TestRecover)); - pNew->p = sqlite3_recover_init(db, zDb, zUri); + if( bSql==0 ){ + zUri = Tcl_GetString(objv[3]); + pNew->p = sqlite3_recover_init(db, zDb, zUri); + }else{ + pNew->interp = interp; + pNew->pScript = objv[3]; + Tcl_IncrRefCount(pNew->pScript); + pNew->p = sqlite3_recover_init_sql(db, zDb, xSqlCallback, (void*)pNew); + } sprintf(zCmd, "sqlite_recover%d", iTestRecoverCmd++); Tcl_CreateObjCommand(interp, zCmd, testRecoverCmd, (void*)pNew, 0); @@ -194,14 +226,16 @@ int TestRecover_Init(Tcl_Interp *interp){ struct Cmd { const char *zCmd; Tcl_ObjCmdProc *xProc; + void *pArg; } aCmd[] = { - { "sqlite3_recover_init", test_sqlite3_recover_init }, + { "sqlite3_recover_init", test_sqlite3_recover_init, 0 }, + { "sqlite3_recover_init_sql", test_sqlite3_recover_init, (void*)1 }, }; int i; for(i=0; izCmd, p->xProc, 0, 0); + Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, p->pArg, 0); } return TCL_OK; diff --git a/manifest b/manifest index 9f01e7c919..9244609811 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_RECOVER_ROWIDS\soption.\sTo\sspecify\sthat\srowid\svalues\sthat\sare\snot\salso\sexplicit\sINTEGER\sPRIMARY\sKEY\svalues\sshould\sbe\spreserved. -D 2022-09-05T15:56:09.391 +C Add\sa\smode\sto\soutput\sSQL\sstatements\sinstead\sof\spopulating\sa\sdatabase\sto\sthe\srecover\sextension. +D 2022-09-05T21:00:22.268 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,12 +387,12 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea +F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e56d636e51e93 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverold.test e7e00c78ec35b60488369ddf99e36a3b30e686566571969b05781e5063bdffe8 -F ext/recover/sqlite3recover.c 13e78c8a9d5521e06ebe5ac992a90169155e685f5c4b3cebc632c50b41e061c9 -F ext/recover/sqlite3recover.h ed34bc96befdf581a7de039d99772caabf0b338eb2a841d869acd9f7893ffce7 -F ext/recover/test_recover.c 9b8144ac94e6a2f3aabfcd24db6416179afdd32a3d654d1ef603c570e0384b2f +F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 +F ext/recover/sqlite3recover.c 47767b52f09fb1bba47009236285f09bcf68b6b5d1ec0de96b1a30b508e536a4 +F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 +F ext/recover/test_recover.c be0d74f0eba44fe7964e22d287dba0f3fa2baf197f630d51f0f9066af9b5eb2a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 253e498f5200b8b3e2bc309587af108dd1cec8a884b3d2a49d5406525c9a4b4c -R e5ada01eb624b55d2f64c4994830b06d +P 69cc9aba56a196bbd159bd24868aa5ccc60bed0dc612d09ed8a3ae898f156809 +R fe94b03379801a454a2e4362e6c6eea7 U dan -Z a7e3c99975f0369d94d22f82581262dd +Z bbf845b5205dd27415c5eec6b9356a28 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f34597368d..89f3b16ef4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69cc9aba56a196bbd159bd24868aa5ccc60bed0dc612d09ed8a3ae898f156809 \ No newline at end of file +73058416e7da6581000898b7988a7010e2ce6632246f4c12b4398700c7744b83 \ No newline at end of file From 108c51b6ae731d76b5c77f4c4689d64c89401230 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Sep 2022 21:22:35 +0000 Subject: [PATCH 08/42] Fix a problem with script mode and lost-and-found tables. FossilOrigin-Name: 09ec588d2fe24dd321e88318fe90a9ae912cbc73c8a2d59a10c821625dd12d9d --- ext/recover/sqlite3recover.c | 29 +++++++++++++++++++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 1014e576a4..f4bf5c0e4f 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -800,13 +800,24 @@ static sqlite3_stmt *recoverLostAndFoundInsert( const char *zSep = ""; sqlite3_stmt *pRet = 0; - for(ii=0; iixSql==0 ){ + for(ii=0; iidbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind + ); + }else{ + const char *zSep = ""; + for(ii=0; iidbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind + ); } - pRet = recoverPreparePrintf( - p, p->dbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind - ); sqlite3_free(zBind); return pRet; } @@ -860,7 +871,9 @@ static void recoverLostAndFoundPopulate( for(ii=0; iixSql ){ + recoverSqlCallback(p, sqlite3_column_text(pInsert, 0)); + } recoverReset(p, pInsert); /* Discard the accumulated row data */ @@ -1229,6 +1242,8 @@ static void recoverStep(sqlite3_recover *p){ assert( p->errCode==SQLITE_OK ); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + if( p->dbOut==0 ){ if( recoverOpenOutput(p) ) return; if( recoverCacheSchema(p) ) return; @@ -1237,6 +1252,8 @@ static void recoverStep(sqlite3_recover *p){ if( p->zLostAndFound && recoverLostAndFound(p) ) return; if( recoverWriteSchema2(p) ) return; } + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); } int sqlite3_recover_step(sqlite3_recover *p){ diff --git a/manifest b/manifest index 9244609811..d0030bd28b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smode\sto\soutput\sSQL\sstatements\sinstead\sof\spopulating\sa\sdatabase\sto\sthe\srecover\sextension. -D 2022-09-05T21:00:22.268 +C Fix\sa\sproblem\swith\sscript\smode\sand\slost-and-found\stables. +D 2022-09-05T21:22:35.859 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,7 +390,7 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282 F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e56d636e51e93 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 -F ext/recover/sqlite3recover.c 47767b52f09fb1bba47009236285f09bcf68b6b5d1ec0de96b1a30b508e536a4 +F ext/recover/sqlite3recover.c 5dca1b3904cb028bef7348a5e0726a253c48f7fcd5da5ec687a44e4e665a24bf F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 F ext/recover/test_recover.c be0d74f0eba44fe7964e22d287dba0f3fa2baf197f630d51f0f9066af9b5eb2a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 69cc9aba56a196bbd159bd24868aa5ccc60bed0dc612d09ed8a3ae898f156809 -R fe94b03379801a454a2e4362e6c6eea7 +P 73058416e7da6581000898b7988a7010e2ce6632246f4c12b4398700c7744b83 +R 9dcaa53e272447eefaba97715f5d61ae U dan -Z bbf845b5205dd27415c5eec6b9356a28 +Z 82e767f579594364114308f585b07fed # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 89f3b16ef4..8ad6e97f6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73058416e7da6581000898b7988a7010e2ce6632246f4c12b4398700c7744b83 \ No newline at end of file +09ec588d2fe24dd321e88318fe90a9ae912cbc73c8a2d59a10c821625dd12d9d \ No newline at end of file From 497b3e6a1152e58be842fa17dee247360aaf8888 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Sep 2022 19:38:06 +0000 Subject: [PATCH 09/42] Tests and a fix for the SQLITE_RECOVER_ROWIDS option. FossilOrigin-Name: 1d5000f5718004110776ff58284d824854a93fe1cd1f6d8a4e8b8b0c2b2c3076 --- ext/recover/recoverrowid.test | 58 +++++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 18 ++++++++--- manifest | 13 ++++---- manifest.uuid | 2 +- 4 files changed, 79 insertions(+), 12 deletions(-) create mode 100644 ext/recover/recoverrowid.test diff --git a/ext/recover/recoverrowid.test b/ext/recover/recoverrowid.test new file mode 100644 index 0000000000..3dfafbcc7f --- /dev/null +++ b/ext/recover/recoverrowid.test @@ -0,0 +1,58 @@ +# 2022 September 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_ROWIDS option. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoverrowid + +ifcapable !vtab { + finish_test; return +} + +proc recover {db bRowids output} { + forcedelete $output + + set R [sqlite3_recover_init db main test.db2] + $R config rowids $bRowids + $R step + $R finish +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); + DELETE FROM t1 WHERE a IN (1, 3); +} + +do_test 1.1 { + recover db 0 test.db2 + sqlite3 db2 test.db2 + execsql { SELECT rowid, a, b FROM t1 ORDER BY rowid} db2 +} {1 2 2 2 4 4} + +do_test 1.2 { + db2 close + recover db 1 test.db2 + sqlite3 db2 test.db2 + execsql { SELECT rowid, a, b FROM t1 ORDER BY rowid} db2 +} {2 2 2 4 4 4} +db2 close + + + + +finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index f4bf5c0e4f..df6ab36b98 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -403,14 +403,22 @@ static int recoverOpenOutput(sqlite3_recover *p){ sqlite3_free(zSql); } - + /* Truncate the output database. This is done by opening a new, empty, + ** temp db, then using the backup API to clobber any existing output db + ** with a copy of it. */ if( rc==SQLITE_OK ){ - sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db, "recovery"); - if( pBackup ){ - while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); - rc = sqlite3_backup_finish(pBackup); + sqlite3 *db2 = 0; + rc = sqlite3_open("", &db2); + if( rc==SQLITE_OK ){ + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); + rc = sqlite3_backup_finish(pBackup); + } } + sqlite3_close(db2); } + if( rc==SQLITE_OK ){ rc = sqlite3_exec(db, RECOVERY_SCHEMA, 0, 0, 0); } diff --git a/manifest b/manifest index d0030bd28b..a19ce57ac1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sscript\smode\sand\slost-and-found\stables. -D 2022-09-05T21:22:35.859 +C Tests\sand\sa\sfix\sfor\sthe\sSQLITE_RECOVER_ROWIDS\soption. +D 2022-09-06T19:38:06.927 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,7 +390,8 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282 F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e56d636e51e93 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 -F ext/recover/sqlite3recover.c 5dca1b3904cb028bef7348a5e0726a253c48f7fcd5da5ec687a44e4e665a24bf +F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 +F ext/recover/sqlite3recover.c 297fdef9da8523ef7fa3f88adb31356340826dac56c1fb13db2dd3e167806efe F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 F ext/recover/test_recover.c be0d74f0eba44fe7964e22d287dba0f3fa2baf197f630d51f0f9066af9b5eb2a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2005,8 +2006,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73058416e7da6581000898b7988a7010e2ce6632246f4c12b4398700c7744b83 -R 9dcaa53e272447eefaba97715f5d61ae +P 09ec588d2fe24dd321e88318fe90a9ae912cbc73c8a2d59a10c821625dd12d9d +R 2900e793f9b609ad9d5a7b230af64f99 U dan -Z 82e767f579594364114308f585b07fed +Z a6081cceac978cdb593b5cd251ae6a63 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8ad6e97f6b..c0fbb47278 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09ec588d2fe24dd321e88318fe90a9ae912cbc73c8a2d59a10c821625dd12d9d \ No newline at end of file +1d5000f5718004110776ff58284d824854a93fe1cd1f6d8a4e8b8b0c2b2c3076 \ No newline at end of file From abb28667f159796be159292b8c0c6a68b56f73d9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Sep 2022 16:41:33 +0000 Subject: [PATCH 10/42] Ensure that the recover extension properly escapes CR and NL characters in text mode. Also that it holds transactions open on both input and output databases for the duration of a recovery operation. FossilOrigin-Name: 6cca8913e703635ad89415a60fc84000ac188d9df43f45594b8ad87facb91d54 --- ext/recover/sqlite3recover.c | 331 +++++++++++++++++++++++------------ ext/recover/test_recover.c | 5 +- manifest | 14 +- manifest.uuid | 2 +- 4 files changed, 229 insertions(+), 123 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index df6ab36b98..de3dc6ea1a 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -234,15 +234,15 @@ static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ if( p->errCode==SQLITE_OK ){ - int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + int rc = sqlite3_exec(db, zSql, 0, 0, 0); if( rc ){ - recoverDbError(p, p->dbOut); + recoverDbError(p, db); } } return p->errCode; } -static char *recoverPrintf(sqlite3_recover *p, const char *zFmt, ...){ +static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ va_list ap; char *z; va_start(ap, zFmt); @@ -375,79 +375,187 @@ static void recoverGetPage( } } +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *unused_string( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + + +/* +** Scalar function "escape_crnl". The argument passed to this function is the +** output of built-in function quote(). If the first character of the input is +** "'", indicating that the value passed to quote() was a text value, then this +** function searches the input for "\n" and "\r" characters and adds a wrapper +** similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void recoverEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText && zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + + #ifdef _WIN32 __declspec(dllexport) #endif int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); static int recoverOpenOutput(sqlite3_recover *p){ - int rc = SQLITE_OK; - if( p->dbOut==0 ){ - const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; - sqlite3 *db = 0; + struct Func { + const char *zName; + int nArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value **); + } aFunc[] = { + { "getpage", 1, recoverGetPage }, + { "page_is_used", 1, recoverPageIsUsed }, + { "read_i32", 2, recoverReadI32 }, + { "escape_crnl", 1, recoverEscapeCrnl }, + }; - assert( p->dbOut==0 ); + const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + sqlite3 *db = 0; /* New database handle */ + int ii; /* For iterating through aFunc[] */ - rc = sqlite3_open_v2(p->zUri, &db, flags, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(db, "PRAGMA writable_schema = 1", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - const char *zPath = p->zStateDb ? p->zStateDb : ":memory:"; - char *zSql = sqlite3_mprintf("ATTACH %Q AS recovery", zPath); - if( zSql==0 ){ - rc = p->errCode = SQLITE_NOMEM; - }else{ - rc = sqlite3_exec(db, zSql, 0, 0, 0); - } - sqlite3_free(zSql); - } + assert( p->dbOut==0 ); - /* Truncate the output database. This is done by opening a new, empty, - ** temp db, then using the backup API to clobber any existing output db - ** with a copy of it. */ - if( rc==SQLITE_OK ){ - sqlite3 *db2 = 0; - rc = sqlite3_open("", &db2); - if( rc==SQLITE_OK ){ - sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); - if( pBackup ){ - while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); - rc = sqlite3_backup_finish(pBackup); - } - } - sqlite3_close(db2); - } - - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(db, RECOVERY_SCHEMA, 0, 0, 0); - } - - if( rc==SQLITE_OK ){ - sqlite3_dbdata_init(db, 0, 0); - rc = sqlite3_create_function( - db, "getpage", 1, SQLITE_UTF8, (void*)p, recoverGetPage, 0, 0 - ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - db, "page_is_used", 1, SQLITE_UTF8, (void*)p, recoverPageIsUsed, 0, 0 - ); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_create_function( - db, "read_i32", 2, SQLITE_UTF8, (void*)p, recoverReadI32, 0, 0 - ); - } - - if( rc!=SQLITE_OK ){ - if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db); - sqlite3_close(db); - }else{ - p->dbOut = db; - } + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); + }else{ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, db, zSql); + recoverExec(p, db, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); } - return rc; + + /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. + ** These two are registered with the output database handle - this + ** module depends on the input handle supporting the sqlite_dbpage + ** virtual table only. */ + if( p->errCode==SQLITE_OK ){ + p->errCode = sqlite3_dbdata_init(db, 0, 0); + } + + /* Register the custom user-functions with the output handle. */ + for(ii=0; p->errCode==SQLITE_OK && iierrCode = sqlite3_create_function(db, aFunc[ii].zName, + aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0 + ); + } + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc==SQLITE_OK ){ + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); + p->errCode = sqlite3_backup_finish(pBackup); + } + } + sqlite3_close(db2); + } + + p->dbOut = db; + return p->errCode; } static int recoverCacheSchema(sqlite3_recover *p){ @@ -632,22 +740,6 @@ static int recoverWriteSchema2(sqlite3_recover *p){ return p->errCode; } - -static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ - char *zRet = 0; - if( p->errCode==SQLITE_OK ){ - va_list ap; - char *z; - va_start(ap, zFmt); - zRet = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( zRet==0 ){ - p->errCode = SQLITE_NOMEM; - } - } - return zRet; -} - static sqlite3_stmt *recoverInsertStmt( sqlite3_recover *p, RecoverTable *pTab, @@ -688,8 +780,8 @@ static sqlite3_stmt *recoverInsertStmt( zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol); if( bSql ){ - zBind = recoverMPrintf( - p, "%z%squote(?%d)", zBind, zSqlSep, pTab->aCol[ii].iBind + zBind = recoverMPrintf(p, + "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind ); zSqlSep = "||', '||"; }else{ @@ -745,9 +837,9 @@ static char *recoverLostAndFoundCreate( for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){ int bFail = 0; if( ii<0 ){ - zTbl = recoverPrintf(p, "%s", p->zLostAndFound); + zTbl = recoverMPrintf(p, "%s", p->zLostAndFound); }else{ - zTbl = recoverPrintf(p, "%s_%d", p->zLostAndFound, ii); + zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii); } if( p->errCode==SQLITE_OK ){ @@ -777,7 +869,7 @@ static char *recoverLostAndFoundCreate( zSep = ", "; } - zSql = recoverPrintf(p, "CREATE TABLE %s(%s)", zTbl, zField); + zSql = recoverMPrintf(p, "CREATE TABLE %s(%s)", zTbl, zField); sqlite3_free(zField); recoverExec(p, p->dbOut, zSql); @@ -1217,14 +1309,14 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ switch( op ){ case SQLITE_RECOVER_TESTDB: sqlite3_free(p->zStateDb); - p->zStateDb = sqlite3_mprintf("%s", (char*)pArg); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); break; case SQLITE_RECOVER_LOST_AND_FOUND: const char *zArg = (const char*)pArg; sqlite3_free(p->zLostAndFound); if( zArg ){ - p->zLostAndFound = recoverPrintf(p, "%s", zArg); + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); }else{ p->zLostAndFound = 0; } @@ -1247,21 +1339,47 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ } static void recoverStep(sqlite3_recover *p){ - + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + int rc = SQLITE_OK; assert( p->errCode==SQLITE_OK ); + recoverSqlCallback(p, "BEGIN"); recoverSqlCallback(p, "PRAGMA writable_schema = on"); - if( p->dbOut==0 ){ - if( recoverOpenOutput(p) ) return; - if( recoverCacheSchema(p) ) return; - if( recoverWriteSchema1(p) ) return; - if( recoverWriteData(p) ) return; - if( p->zLostAndFound && recoverLostAndFound(p) ) return; - if( recoverWriteSchema2(p) ) return; - } + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + recoverExec(p, p->dbIn, "BEGIN"); + recoverExec(p, p->dbOut, "BEGIN"); + + recoverCacheSchema(p); + recoverWriteSchema1(p); + recoverWriteData(p); + if( p->zLostAndFound ) recoverLostAndFound(p); + recoverWriteSchema2(p); + + /* If no error has occurred, commit the write transaction on the output + ** database. Then end the read transaction on the input database, regardless + ** of whether or not prior errors have occurred. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + + sqlite3_finalize(p->pGetPage); + sqlite3_close(p->dbOut); + p->pGetPage = 0; } int sqlite3_recover_step(sqlite3_recover *p){ @@ -1272,21 +1390,8 @@ int sqlite3_recover_step(sqlite3_recover *p){ } int sqlite3_recover_finish(sqlite3_recover *p){ - RecoverTable *pTab; - RecoverTable *pNext; - int rc; - - for(pTab=p->pTblList; pTab; pTab=pNext){ - pNext = pTab->pNext; - sqlite3_free(pTab); - } - - sqlite3_finalize(p->pGetPage); - rc = sqlite3_close(p->dbOut); - assert( rc==SQLITE_OK ); - p->pGetPage = 0; - rc = p->errCode; - + int rc = p->errCode; + sqlite3_free(p->zErrMsg); sqlite3_free(p->zStateDb); sqlite3_free(p->zLostAndFound); sqlite3_free(p); diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index acfcf8c7a3..1de99f4bd6 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -13,6 +13,7 @@ */ #include "sqlite3recover.h" +#include "sqliteInt.h" #include #include @@ -128,7 +129,7 @@ static int testRecoverCmd( int iVal = 0; if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_FREELIST_CORRUPT, (void*)iVal + SQLITE_RECOVER_FREELIST_CORRUPT, SQLITE_INT_TO_PTR(iVal) ); break; } @@ -136,7 +137,7 @@ static int testRecoverCmd( int iVal = 0; if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_ROWIDS, (void*)iVal + SQLITE_RECOVER_ROWIDS, SQLITE_INT_TO_PTR(iVal) ); break; } diff --git a/manifest b/manifest index a19ce57ac1..9e036d2201 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tests\sand\sa\sfix\sfor\sthe\sSQLITE_RECOVER_ROWIDS\soption. -D 2022-09-06T19:38:06.927 +C Ensure\sthat\sthe\srecover\sextension\sproperly\sescapes\sCR\sand\sNL\scharacters\sin\stext\smode.\sAlso\sthat\sit\sholds\stransactions\sopen\son\sboth\sinput\sand\soutput\sdatabases\sfor\sthe\sduration\sof\sa\srecovery\soperation. +D 2022-09-07T16:41:33.248 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -391,9 +391,9 @@ F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 297fdef9da8523ef7fa3f88adb31356340826dac56c1fb13db2dd3e167806efe +F ext/recover/sqlite3recover.c 9724f913fd457f655e2873552bc6600a6aaff7104b9113ccb38fea18b6a71f03 F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 -F ext/recover/test_recover.c be0d74f0eba44fe7964e22d287dba0f3fa2baf197f630d51f0f9066af9b5eb2a +F ext/recover/test_recover.c 7aa268d3431d630eaa82ce14974ae04be50fe7feba660ffaea009cd581916d27 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2006,8 +2006,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09ec588d2fe24dd321e88318fe90a9ae912cbc73c8a2d59a10c821625dd12d9d -R 2900e793f9b609ad9d5a7b230af64f99 +P 1d5000f5718004110776ff58284d824854a93fe1cd1f6d8a4e8b8b0c2b2c3076 +R 3bf4abb672d881722cf6509f544c5091 U dan -Z a6081cceac978cdb593b5cd251ae6a63 +Z c71f206aa1a37ae4579417497726bb42 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c0fbb47278..85ba19965b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d5000f5718004110776ff58284d824854a93fe1cd1f6d8a4e8b8b0c2b2c3076 \ No newline at end of file +6cca8913e703635ad89415a60fc84000ac188d9df43f45594b8ad87facb91d54 \ No newline at end of file From 4a08e793c6125c5ef6dc2f3c7b403c90ee6a196a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Sep 2022 11:04:23 +0000 Subject: [PATCH 11/42] Add new test file recoverclobber.test. FossilOrigin-Name: cb4e950c472bd24a79a8505a7f8e4c3a0f7821648297d05cc760738b777d5149 --- ext/recover/recoverclobber.test | 58 +++++++++++++++++++++++++++++++++ manifest | 11 ++++--- manifest.uuid | 2 +- 3 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 ext/recover/recoverclobber.test diff --git a/ext/recover/recoverclobber.test b/ext/recover/recoverclobber.test new file mode 100644 index 0000000000..e6967ec9be --- /dev/null +++ b/ext/recover/recoverclobber.test @@ -0,0 +1,58 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_ROWIDS option. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoverclobber + +ifcapable !vtab { + finish_test; return +} + +proc recover {db output} { + set R [sqlite3_recover_init db main test.db2] + $R step + $R finish +} + +forcedelete test.db2 +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.x1(x, one); + INSERT INTO x1 VALUES(1, 'one'), (2, 'two'), (3, 'three'); + + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); + + DETACH aux; +} + +breakpoint +do_test 1.1 { + recover db test.db2 +} {} + +do_execsql_test 1.2 { + ATTACH 'test.db2' AS aux; + SELECT * FROM aux.t1; +} {1 1 2 2 3 3 4 4} + +do_catchsql_test 1.3 { + SELECT * FROM aux.x1; +} {1 {no such table: aux.x1}} + +finish_test diff --git a/manifest b/manifest index 9e036d2201..421372fcab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\srecover\sextension\sproperly\sescapes\sCR\sand\sNL\scharacters\sin\stext\smode.\sAlso\sthat\sit\sholds\stransactions\sopen\son\sboth\sinput\sand\soutput\sdatabases\sfor\sthe\sduration\sof\sa\srecovery\soperation. -D 2022-09-07T16:41:33.248 +C Add\snew\stest\sfile\srecoverclobber.test. +D 2022-09-08T11:04:23.161 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,6 +389,7 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e56d636e51e93 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c +F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 F ext/recover/sqlite3recover.c 9724f913fd457f655e2873552bc6600a6aaff7104b9113ccb38fea18b6a71f03 @@ -2006,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1d5000f5718004110776ff58284d824854a93fe1cd1f6d8a4e8b8b0c2b2c3076 -R 3bf4abb672d881722cf6509f544c5091 +P 6cca8913e703635ad89415a60fc84000ac188d9df43f45594b8ad87facb91d54 +R 102ea2ce4694a6be6ee34f8e72c4fbd9 U dan -Z c71f206aa1a37ae4579417497726bb42 +Z e86817dfd35f39cea8c450fbde6bc553 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 85ba19965b..2f04809195 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6cca8913e703635ad89415a60fc84000ac188d9df43f45594b8ad87facb91d54 \ No newline at end of file +cb4e950c472bd24a79a8505a7f8e4c3a0f7821648297d05cc760738b777d5149 \ No newline at end of file From 3887ffe82a146aaee393d1d5560b9bce8058a75a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Sep 2022 17:42:33 +0000 Subject: [PATCH 12/42] Fix problems with recovering the sqlite_sequence table. FossilOrigin-Name: 356d2209ea5f6b69ce15b62027c63419c2d039e52f01c74a3810a6317abf4fb0 --- ext/recover/recover1.test | 26 ++++- ext/recover/sqlite3recover.c | 183 ++++++++++++++++++----------------- manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 128 insertions(+), 97 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 167b2796bd..26e7c60d26 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -45,6 +45,7 @@ proc do_recover_test {tn} { uplevel [list do_test $tn.1 { set R [sqlite3_recover_init db main test.db2] $R config testdb rstate.db + $R config rowids 1 $R step $R finish } {}] @@ -60,12 +61,14 @@ proc do_recover_test {tn} { set ::sqlhook [list] set R [sqlite3_recover_init_sql db main my_sql_hook] $R config testdb rstate.db + $R config rowids 1 $R step $R finish } {}] sqlite3 db2 test.db2 execsql [join $::sqlhook ";"] db2 + # puts [join $::sqlhook ";\n"] uplevel [list do_test $tn.4 [list compare_dbs db db2] {}] db2 close } @@ -82,6 +85,7 @@ do_execsql_test 1.0 { ) INSERT INTO t1 SELECT i*2, hex(randomblob(250)) FROM s; INSERT INTO t2 SELECT * FROM t1; + } do_recover_test 1 @@ -131,8 +135,28 @@ do_execsql_test 7.1 { SELECT * FROM t2 } {10 11 ten} -breakpoint do_recover_test 7.2 +#-------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<2 + ) + INSERT INTO x1(b, c) SELECT hex(randomblob(100)), hex(randomblob(100)) FROM s; + + CREATE INDEX x1b ON x1(b); + CREATE INDEX x1cb ON x1(c, b); + DELETE FROM x1 WHERE a>50; + + ANALYZE; +} + +do_recover_test 8 + + + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index de3dc6ea1a..1ab4f5d6d5 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -60,17 +60,6 @@ struct RecoverBitmap { u32 aElem[0]; /* Array of 32-bit bitmasks */ }; -/* -** -*/ -#define RECOVERY_SCHEMA \ -" CREATE TABLE recovery.map(" \ -" pgno INTEGER PRIMARY KEY, parent INT" \ -" );" \ -" CREATE TABLE recovery.schema(" \ -" type, name, tbl_name, rootpage, sql" \ -" );" - struct sqlite3_recover { sqlite3 *dbIn; @@ -1123,6 +1112,8 @@ static int recoverWriteData(sqlite3_recover *p){ RecoverTable *pTbl; int nMax = 0; sqlite3_value **apVal = 0; + + sqlite3_stmt *pTbls = 0; sqlite3_stmt *pSel = 0; /* Figure out the maximum number of columns for any table in the schema */ @@ -1133,114 +1124,130 @@ static int recoverWriteData(sqlite3_recover *p){ apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * (nMax+1)); if( apVal==0 ) return p->errCode; + pTbls = recoverPrepare(p, p->dbOut, + "SELECT rootpage FROM recovery.schema WHERE type='table'" + " ORDER BY (tbl_name='sqlite_sequence') ASC" + ); + pSel = recoverPrepare(p, p->dbOut, - "WITH RECURSIVE pages(root, page) AS (" - " SELECT rootpage, rootpage FROM recovery.schema" + "WITH RECURSIVE pages(page) AS (" + " SELECT ?1" " UNION" - " SELECT root, child FROM sqlite_dbptr('getpage()'), pages " + " SELECT child FROM sqlite_dbptr('getpage()'), pages " " WHERE pgno=page" ") " - "SELECT root, page, cell, field, value " + "SELECT page, cell, field, value " "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " "UNION ALL " - "SELECT 0, 0, 0, 0, 0" + "SELECT 0, 0, 0, 0" ); if( pSel ){ - RecoverTable *pTab = 0; - sqlite3_stmt *pInsert = 0; - int nInsert = -1; - i64 iPrevRoot = -1; - i64 iPrevPage = -1; - int iPrevCell = -1; - int bHaveRowid = 0; /* True if iRowid is valid */ - i64 iRowid = 0; - int nVal = -1; - while( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ - i64 iRoot = sqlite3_column_int64(pSel, 0); - i64 iPage = sqlite3_column_int64(pSel, 1); - int iCell = sqlite3_column_int(pSel, 2); - int iField = sqlite3_column_int(pSel, 3); - sqlite3_value *pVal = sqlite3_column_value(pSel, 4); + /* The outer loop runs once for each table to recover. */ + while( sqlite3_step(pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(pTbls, 0); + RecoverTable *pTab = recoverFindTable(p, iRoot); + if( pTab ){ + int ii; + sqlite3_stmt *pInsert = 0; + int nInsert = -1; + i64 iPrevPage = -1; + int iPrevCell = -1; + int bHaveRowid = 0; /* True if iRowid is valid */ + i64 iRowid = 0; + int nVal = -1; - int bNewCell = (iPrevRoot!=iRoot || iPrevPage!=iPage || iPrevCell!=iCell); - assert( bNewCell==0 || (iField==-1 || iField==0) ); - assert( bNewCell || iField==nVal ); + if( sqlite3_stricmp("sqlite_sequence", pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } - if( bNewCell ){ - if( nVal>=0 ){ - int ii; + sqlite3_bind_int64(pSel, 1, iRoot); + while( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); - if( pTab ){ - int iVal = 0; - int iBind = 1; + int bNewCell = (iPrevPage!=iPage || iPrevCell!=iCell); + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==nVal ); - if( pInsert==0 || nVal!=nInsert ){ - recoverFinalize(p, pInsert); - pInsert = recoverInsertStmt(p, pTab, nVal); - nInsert = nVal; - } + if( bNewCell ){ + if( nVal>=0 ){ + int ii; + int iVal = 0; + int iBind = 1; - for(ii=0; iinCol; ii++){ - RecoverColumn *pCol = &pTab->aCol[ii]; + if( pInsert==0 || nVal!=nInsert ){ + recoverFinalize(p, pInsert); + pInsert = recoverInsertStmt(p, pTab, nVal); + nInsert = nVal; + } - if( pCol->iBind>0 ){ - if( pCol->bIPK ){ - sqlite3_bind_int64(pInsert, pCol->iBind, iRowid); - }else if( pCol->iFieldiBind, apVal[pCol->iField]); + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + + if( pCol->iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, pCol->iBind, iRowid); + }else if( pCol->iFieldiBind,apVal[pCol->iField]); + } } } - } - if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ - sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); + if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); + } + + if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ + const char *zSql = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, zSql); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); } - if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ - const char *zSql = (const char*)sqlite3_column_text(pInsert, 0); - recoverSqlCallback(p, zSql); + for(ii=0; iierrCode || pInsert ); - if( pInsert ) sqlite3_clear_bindings(pInsert); + nVal = -1; + bHaveRowid = 0; } - for(ii=0; iierrCode; } diff --git a/manifest b/manifest index 421372fcab..f5344ad585 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\srecoverclobber.test. -D 2022-09-08T11:04:23.161 +C Fix\sproblems\swith\srecovering\sthe\ssqlite_sequence\stable. +D 2022-09-08T17:42:33.679 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,12 +387,12 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test ae8ce9828210aa6c466bf88e23b0933849d5bb43091abe48cf2e56d636e51e93 +F ext/recover/recover1.test ddc322148170eafe1dabbea91ac175a72f7e7d2777619a6434696a310beff9a3 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 9724f913fd457f655e2873552bc6600a6aaff7104b9113ccb38fea18b6a71f03 +F ext/recover/sqlite3recover.c 2c45ab8cce41dcb578ef739652e65675d161751fe0d979b806d947a02de7fd32 F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 F ext/recover/test_recover.c 7aa268d3431d630eaa82ce14974ae04be50fe7feba660ffaea009cd581916d27 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6cca8913e703635ad89415a60fc84000ac188d9df43f45594b8ad87facb91d54 -R 102ea2ce4694a6be6ee34f8e72c4fbd9 +P cb4e950c472bd24a79a8505a7f8e4c3a0f7821648297d05cc760738b777d5149 +R e45081ba4805e1d831caf7a65d0c00b9 U dan -Z e86817dfd35f39cea8c450fbde6bc553 +Z 8c44bc5cd3a46bb8aa19fae7f571d64e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f04809195..34b201c3c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb4e950c472bd24a79a8505a7f8e4c3a0f7821648297d05cc760738b777d5149 \ No newline at end of file +356d2209ea5f6b69ce15b62027c63419c2d039e52f01c74a3810a6317abf4fb0 \ No newline at end of file From 9a27d65044d64cf4ad49b618244f3de022a39d9e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Sep 2022 19:22:29 +0000 Subject: [PATCH 13/42] Update the shell to use the recover extension for the .recover command. FossilOrigin-Name: ae832e77084eddd696c80cb926d070a5db9d45dce34156a02522b3140e8f5e8b --- ext/recover/recover1.test | 2 - ext/recover/sqlite3recover.c | 16 +- ext/recover/sqlite3recover.h | 8 +- ext/recover/test_recover.c | 4 +- manifest | 20 +- manifest.uuid | 2 +- src/shell.c.in | 658 ++--------------------------------- 7 files changed, 54 insertions(+), 656 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 26e7c60d26..9afe43209f 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -45,7 +45,6 @@ proc do_recover_test {tn} { uplevel [list do_test $tn.1 { set R [sqlite3_recover_init db main test.db2] $R config testdb rstate.db - $R config rowids 1 $R step $R finish } {}] @@ -68,7 +67,6 @@ proc do_recover_test {tn} { sqlite3 db2 test.db2 execsql [join $::sqlhook ";"] db2 - # puts [join $::sqlhook ";\n"] uplevel [list do_test $tn.4 [list compare_dbs db db2] {}] db2 close } diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 1ab4f5d6d5..cef2fb9da7 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -84,6 +84,11 @@ struct sqlite3_recover { int (*xSql)(void*,const char*); }; +/* +** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). +*/ +#define RECOVER_ROWID_DEFAULT 1 + /* ** Like strlen(). But handles NULL pointer arguments. */ @@ -371,7 +376,7 @@ static void recoverGetPage( ** Try to use zA and zB first. If both of those are already found in z[] ** then make up some string and store it in the buffer zBuf. */ -static const char *unused_string( +static const char *recoverUnusedString( const char *z, /* Result must not appear anywhere in z */ const char *zA, const char *zB, /* Try these first */ char *zBuf /* Space to store a generated string */ @@ -416,11 +421,11 @@ static void recoverEscapeCrnl( for(i=0; zText[i]; i++){ if( zNL==0 && zText[i]=='\n' ){ - zNL = unused_string(zText, "\\n", "\\012", zBuf1); + zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1); nNL = (int)strlen(zNL); } if( zCR==0 && zText[i]=='\r' ){ - zCR = unused_string(zText, "\\r", "\\015", zBuf2); + zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2); nCR = (int)strlen(zCR); } } @@ -1281,6 +1286,7 @@ sqlite3_recover *recoverInit( memcpy(pRet->zUri, zUri, nUri); pRet->xSql = xSql; pRet->pSqlCtx = pSqlCtx; + pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; } return pRet; @@ -1330,11 +1336,11 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ break; case SQLITE_RECOVER_FREELIST_CORRUPT: - p->bFreelistCorrupt = (pArg ? 1 : 0); + p->bFreelistCorrupt = *(int*)pArg; break; case SQLITE_RECOVER_ROWIDS: - p->bRecoverRowid = (pArg ? 1 : 0); + p->bRecoverRowid = *(int*)pArg; break; default: diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 5dc2a56d30..1ef707e13c 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -16,7 +16,7 @@ #ifndef _SQLITE_RECOVER_H #define _SQLITE_RECOVER_H -#include "sqlite3.h" /* Required for error code definitions */ +#include "sqlite3.h" #ifdef __cplusplus extern "C" { @@ -58,9 +58,9 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** pages to add to the lost-and-found table. ** ** SQLITE_RECOVER_FREELIST_CORRUPT: -** The pArg value must actually be integer (type "int") value 0 or 1 -** cast as a (void*). If this option is set (argument is 1) and -** a lost-and-found table has been configured using +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1) and a lost-and-found table has been configured using ** SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is ** corrupt and an attempt is made to recover records from pages that ** appear to be linked into the freelist. Otherwise, pages on the freelist diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 1de99f4bd6..83166e994c 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -129,7 +129,7 @@ static int testRecoverCmd( int iVal = 0; if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_FREELIST_CORRUPT, SQLITE_INT_TO_PTR(iVal) + SQLITE_RECOVER_FREELIST_CORRUPT, (void*)&iVal ); break; } @@ -137,7 +137,7 @@ static int testRecoverCmd( int iVal = 0; if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_ROWIDS, SQLITE_INT_TO_PTR(iVal) + SQLITE_RECOVER_ROWIDS, (void*)&iVal ); break; } diff --git a/manifest b/manifest index f5344ad585..013ce9cd0f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\swith\srecovering\sthe\ssqlite_sequence\stable. -D 2022-09-08T17:42:33.679 +C Update\sthe\sshell\sto\suse\sthe\srecover\sextension\sfor\sthe\s.recover\scommand. +D 2022-09-08T19:22:29.395 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,14 +387,14 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test ddc322148170eafe1dabbea91ac175a72f7e7d2777619a6434696a310beff9a3 +F ext/recover/recover1.test aa9f7f46b7209cae6d52321052d4440bc8f82b93991e693c4bad20a6f05a53e5 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 2c45ab8cce41dcb578ef739652e65675d161751fe0d979b806d947a02de7fd32 -F ext/recover/sqlite3recover.h 32f89b66f0235c0d94dfee0f1c3e9ed1ad726b3c4f3716ef0262b31cc33e8301 -F ext/recover/test_recover.c 7aa268d3431d630eaa82ce14974ae04be50fe7feba660ffaea009cd581916d27 +F ext/recover/sqlite3recover.c b44241ca3b0cca2b50cfa7f715f79f4eddbe87d56e3baadd4e7bafe4c5872550 +F ext/recover/sqlite3recover.h 218cd9ba8c5c66c3841ca5014910982dc956cba5274257a0ecefb889db879133 +F ext/recover/test_recover.c 68b095ad396d8b1d9242ea663a4be1ad7585a46b1fc03483e9a692c8a87d2674 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -597,7 +597,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ccce37e7fbe71089cf6aec91e7134c9c0c1d4840cff9f02587bbc71240d914a5 -F src/shell.c.in e7e7c2c69ae86c5ee9e8ad66227203d46ff6dce8700a1b1dababff01c71d33df +F src/shell.c.in b36581c005ebaa59b1eeb143bd3ad5a4b273bf15380ccac63ac8cdf4f0c4d3c9 F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P cb4e950c472bd24a79a8505a7f8e4c3a0f7821648297d05cc760738b777d5149 -R e45081ba4805e1d831caf7a65d0c00b9 +P 356d2209ea5f6b69ce15b62027c63419c2d039e52f01c74a3810a6317abf4fb0 +R 9479c02b6838b39b62d6729211ce276e U dan -Z 8c44bc5cd3a46bb8aa19fae7f571d64e +Z df9c400431a19b9f6f7750b759f567b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 34b201c3c5..8cbff6b606 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -356d2209ea5f6b69ce15b62027c63419c2d039e52f01c74a3810a6317abf4fb0 \ No newline at end of file +ae832e77084eddd696c80cb926d070a5db9d45dce34156a02522b3140e8f5e8b \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index d6f3a0aeb6..2a2aa81463 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1039,6 +1039,8 @@ INCLUDE ../ext/expert/sqlite3expert.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) INCLUDE ../ext/misc/dbdata.c +INCLUDE ../ext/recover/sqlite3recover.h +INCLUDE ../ext/recover/sqlite3recover.c #endif #if defined(SQLITE_ENABLE_SESSION) @@ -7252,363 +7254,15 @@ end_ar_command: #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -/* -** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, the SQL statement or statements in zSql are executed using -** database connection db and the error code written to *pRc before -** this function returns. -*/ -static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ - int rc = *pRc; - if( rc==SQLITE_OK ){ - char *zErr = 0; - rc = sqlite3_exec(db, zSql, 0, 0, &zErr); - if( rc!=SQLITE_OK ){ - raw_printf(stderr, "SQL error: %s\n", zErr); - } - sqlite3_free(zErr); - *pRc = rc; - } -} /* -** Like shellExec(), except that zFmt is a printf() style format string. +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. */ -static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - shellExec(db, pRc, z); - } - sqlite3_free(z); - } -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, an attempt is made to allocate, zero and return a pointer -** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set -** to SQLITE_NOMEM and NULL returned. -*/ -static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ - void *pRet = 0; - if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc64(nByte); - if( pRet==0 ){ - *pRc = SQLITE_NOMEM; - }else{ - memset(pRet, 0, nByte); - } - } - return pRet; -} - -/* -** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** Otherwise, zFmt is treated as a printf() style string. The result of -** formatting it along with any trailing arguments is written into a -** buffer obtained from sqlite3_malloc(), and pointer to which is returned. -** It is the responsibility of the caller to eventually free this buffer -** using a call to sqlite3_free(). -** -** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL -** pointer returned. -*/ -static char *shellMPrintf(int *pRc, const char *zFmt, ...){ - char *z = 0; - if( *pRc==SQLITE_OK ){ - va_list ap; - va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - if( z==0 ){ - *pRc = SQLITE_NOMEM; - } - } - return z; -} - - -/* -** When running the ".recover" command, each output table, and the special -** orphaned row table if it is required, is represented by an instance -** of the following struct. -*/ -typedef struct RecoverTable RecoverTable; -struct RecoverTable { - char *zQuoted; /* Quoted version of table name */ - int nCol; /* Number of columns in table */ - char **azlCol; /* Array of column lists */ - int iPk; /* Index of IPK column */ -}; - -/* -** Free a RecoverTable object allocated by recoverFindTable() or -** recoverOrphanTable(). -*/ -static void recoverFreeTable(RecoverTable *pTab){ - if( pTab ){ - sqlite3_free(pTab->zQuoted); - if( pTab->azlCol ){ - int i; - for(i=0; i<=pTab->nCol; i++){ - sqlite3_free(pTab->azlCol[i]); - } - sqlite3_free(pTab->azlCol); - } - sqlite3_free(pTab); - } -} - -/* -** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. -** Otherwise, it allocates and returns a RecoverTable object based on the -** final four arguments passed to this function. It is the responsibility -** of the caller to eventually free the returned object using -** recoverFreeTable(). -*/ -static RecoverTable *recoverNewTable( - int *pRc, /* IN/OUT: Error code */ - const char *zName, /* Name of table */ - const char *zSql, /* CREATE TABLE statement */ - int bIntkey, - int nCol -){ - sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ - int rc = *pRc; - RecoverTable *pTab = 0; - - pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); - if( rc==SQLITE_OK ){ - int nSqlCol = 0; - int bSqlIntkey = 0; - sqlite3_stmt *pStmt = 0; - - rc = sqlite3_open("", &dbtmp); - if( rc==SQLITE_OK ){ - sqlite3_create_function(dbtmp, "shell_idquote", 1, SQLITE_UTF8, 0, - shellIdQuote, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); - if( rc==SQLITE_ERROR ){ - rc = SQLITE_OK; - goto finished; - } - } - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT count(*) FROM pragma_table_info(%Q)", zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - nSqlCol = sqlite3_column_int(pStmt, 0); - } - shellFinalize(&rc, pStmt); - - if( rc!=SQLITE_OK || nSqlColiPk to the index - ** of the column, where columns are 0-numbered from left to right. - ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, - ** leave zPk as "_rowid_" and pTab->iPk at -2. */ - pTab->iPk = -2; - if( bIntkey ){ - shellPreparePrintf(dbtmp, &rc, &pPkFinder, - "SELECT cid, name FROM pragma_table_info(%Q) " - " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" - , zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); - if( zPk==0 ){ zPk = "_"; /* Defensive. Should never happen */ } - } - } - - pTab->zQuoted = shellMPrintf(&rc, "\"%w\"", zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); - pTab->nCol = nSqlCol; - - if( bIntkey ){ - pTab->azlCol[0] = shellMPrintf(&rc, "\"%w\"", zPk); - }else{ - pTab->azlCol[0] = shellMPrintf(&rc, ""); - } - i = 1; - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT %Q || group_concat(shell_idquote(name), ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " - "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, - bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", - zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zText = (const char*)sqlite3_column_text(pStmt, 0); - pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); - i++; - } - shellFinalize(&rc, pStmt); - - shellFinalize(&rc, pPkFinder); - } - } - - finished: - sqlite3_close(dbtmp); - *pRc = rc; - if( rc!=SQLITE_OK || (pTab && pTab->zQuoted==0) ){ - recoverFreeTable(pTab); - pTab = 0; - } - return pTab; -} - -/* -** This function is called to search the schema recovered from the -** sqlite_schema table of the (possibly) corrupt database as part -** of a ".recover" command. Specifically, for a table with root page -** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the -** table must be a WITHOUT ROWID table, or if non-zero, not one of -** those. -** -** If a table is found, a (RecoverTable*) object is returned. Or, if -** no such table is found, but bIntkey is false and iRoot is the -** root page of an index in the recovered schema, then (*pbNoop) is -** set to true and NULL returned. Or, if there is no such table or -** index, NULL is returned and (*pbNoop) set to 0, indicating that -** the caller should write data to the orphans table. -*/ -static RecoverTable *recoverFindTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - int iRoot, /* Root page of table */ - int bIntkey, /* True for an intkey table */ - int nCol, /* Number of columns in table */ - int *pbNoop /* OUT: True if iRoot is root of index */ -){ - sqlite3_stmt *pStmt = 0; - RecoverTable *pRet = 0; - int bNoop = 0; - const char *zSql = 0; - const char *zName = 0; - - /* Search the recovered schema for an object with root page iRoot. */ - shellPreparePrintf(pState->db, pRc, &pStmt, - "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot - ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zType = (const char*)sqlite3_column_text(pStmt, 0); - if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ - bNoop = 1; - break; - } - if( sqlite3_stricmp(zType, "table")==0 ){ - zName = (const char*)sqlite3_column_text(pStmt, 1); - zSql = (const char*)sqlite3_column_text(pStmt, 2); - if( zName!=0 && zSql!=0 ){ - pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); - break; - } - } - } - - shellFinalize(pRc, pStmt); - *pbNoop = bNoop; - return pRet; -} - -/* -** Return a RecoverTable object representing the orphans table. -*/ -static RecoverTable *recoverOrphanTable( - ShellState *pState, /* Shell state object */ - int *pRc, /* IN/OUT: Error code */ - const char *zLostAndFound, /* Base name for orphans table */ - int nCol /* Number of user data columns */ -){ - RecoverTable *pTab = 0; - if( nCol>=0 && *pRc==SQLITE_OK ){ - int i; - - /* This block determines the name of the orphan table. The prefered - ** name is zLostAndFound. But if that clashes with another name - ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 - ** and so on until a non-clashing name is found. */ - int iTab = 0; - char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); - sqlite3_stmt *pTest = 0; - shellPrepare(pState->db, pRc, - "SELECT 1 FROM recovery.schema WHERE name=?", &pTest - ); - if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ - shellReset(pRc, pTest); - sqlite3_free(zTab); - zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); - sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); - } - shellFinalize(pRc, pTest); - - pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pTab ){ - pTab->zQuoted = shellMPrintf(pRc, "\"%w\"", zTab); - pTab->nCol = nCol; - pTab->iPk = -2; - if( nCol>0 ){ - pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); - if( pTab->azlCol ){ - pTab->azlCol[nCol] = shellMPrintf(pRc, ""); - for(i=nCol-1; i>=0; i--){ - pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); - } - } - } - - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; - }else{ - raw_printf(pState->out, - "CREATE TABLE %s(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted - ); - for(i=0; iout, ", c%d", i); - } - raw_printf(pState->out, ");\n"); - } - } - sqlite3_free(zTab); - } - return pTab; +static int recoverSqlCb(void *pCtx, const char *zSql){ + ShellState *pState = (ShellState*)pCtx; + raw_printf(stdout, "%s;\n", zSql); + return SQLITE_OK; } /* @@ -7618,17 +7272,13 @@ static RecoverTable *recoverOrphanTable( */ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; - sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ - sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ - sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ const char *zRecoveryDb = ""; /* Name of "recovery" database */ - const char *zLostAndFound = "lost_and_found"; - int i; - int nOrphan = -1; - RecoverTable *pOrphan = 0; - + const char *zLAF = "lost_and_found"; int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ int bRowids = 1; /* 0 if --no-rowids */ + sqlite3_recover *p = 0; + int i = 0; + for(i=1; idb, &rc, - /* Attach an in-memory database named 'recovery'. Create an indexed - ** cache of the sqlite_dbptr virtual table. */ - "PRAGMA writable_schema = on;" - "ATTACH %Q AS recovery;" - "DROP TABLE IF EXISTS recovery.dbptr;" - "DROP TABLE IF EXISTS recovery.freelist;" - "DROP TABLE IF EXISTS recovery.map;" - "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + p = sqlite3_recover_init_sql( + pState->db, "main", recoverSqlCb, (void*)pState ); - if( bFreelist ){ - shellExec(pState->db, &rc, - "WITH trunk(pgno) AS (" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " - " WHERE x>0" - " UNION" - " SELECT shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " - " FROM trunk WHERE x>0" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, shell_int32(data, 1)-1), t.pgno " - " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" - " UNION ALL" - " SELECT data, n-1, shell_int32(data, 2+n) " - " FROM freelist WHERE n>=0" - ")" - "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" - ); + sqlite3_recover_config(p, SQLITE_RECOVER_TESTDB, (void*)zRecoveryDb); + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + + sqlite3_recover_step(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + raw_printf(stderr, "sql error: %s (%d)\n", zErr, errCode); } - - /* If this is an auto-vacuum database, add all pointer-map pages to - ** the freelist table. Do this regardless of whether or not - ** --freelist-corrupt was specified. */ - shellExec(pState->db, &rc, - "WITH ptrmap(pgno) AS (" - " SELECT 2 WHERE shell_int32(" - " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 13" - " )" - " UNION ALL " - " SELECT pgno+1+(SELECT page_size FROM pragma_page_size)/5 AS pp " - " FROM ptrmap WHERE pp<=(SELECT page_count FROM pragma_page_count)" - ")" - "REPLACE INTO recovery.freelist SELECT pgno FROM ptrmap" - ); - - shellExec(pState->db, &rc, - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr" - " WHERE pgno NOT IN freelist AND child NOT IN freelist;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - - /* Create the "map" table that will (eventually) contain instructions - ** for dealing with each page in the db that contains one or more - ** records. */ - "CREATE TABLE recovery.map(" - "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" - ");" - - /* Populate table [map]. If there are circular loops of pages in the - ** database, the following adds all pages in such a loop to the map - ** as individual root pages. This could be handled better. */ - "WITH pages(i, maxlen) AS (" - " SELECT page_count, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" - " ) FROM pragma_page_count WHERE page_count>0" - " UNION ALL" - " SELECT i-1, (" - " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" - " ) FROM pages WHERE i>=2" - ")" - "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " - " SELECT i, maxlen, NULL, (" - " WITH p(orig, pgno, parent) AS (" - " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" - " UNION " - " SELECT i, p.parent, " - " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" - " )" - " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" - ") " - "FROM pages WHERE maxlen IS NOT NULL AND i NOT IN freelist;" - "UPDATE recovery.map AS o SET intkey = (" - " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" - ");" - - /* Extract data from page 1 and any linked pages into table - ** recovery.schema. With the same schema as an sqlite_schema table. */ - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - "INSERT INTO recovery.schema SELECT " - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=1" - ")" - "GROUP BY pgno, cell;" - "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" - ); - - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" - ** CREATE TABLE statements that extracted from the existing schema. */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - /* ".recover" might output content in an order which causes immediate - ** foreign key constraints to be violated. So disable foreign-key - ** constraint enforcement to prevent problems when running the output - ** script. */ - raw_printf(pState->out, "PRAGMA foreign_keys=OFF;\n"); - raw_printf(pState->out, "BEGIN;\n"); - raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); - shellPrepare(pState->db, &rc, - "SELECT sql FROM recovery.schema " - "WHERE type='table' AND sql LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", - &zCreateTable[12] - ); - } - shellFinalize(&rc, pStmt); - } - - /* Figure out if an orphan table will be required. And if so, how many - ** user columns it should contain */ - shellPrepare(pState->db, &rc, - "SELECT coalesce(max(maxlen), -2) FROM recovery.map WHERE root>1" - , &pLoop - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - nOrphan = sqlite3_column_int(pLoop, 0); - } - shellFinalize(&rc, pLoop); - pLoop = 0; - - shellPrepare(pState->db, &rc, - "SELECT pgno FROM recovery.map WHERE root=?", &pPages - ); - - shellPrepare(pState->db, &rc, - "SELECT max(field), group_concat(shell_escape_crnl(quote" - "(case when (? AND field<0) then NULL else value end)" - "), ', ')" - ", min(field) " - "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" - "GROUP BY cell", &pCells - ); - - /* Loop through each root page. */ - shellPrepare(pState->db, &rc, - "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" - " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" - ")", &pLoop - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - int iRoot = sqlite3_column_int(pLoop, 0); - int bIntkey = sqlite3_column_int(pLoop, 1); - int nCol = sqlite3_column_int(pLoop, 2); - int bNoop = 0; - RecoverTable *pTab; - - assert( bIntkey==0 || bIntkey==1 ); - pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); - if( bNoop || rc ) continue; - if( pTab==0 ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab = pOrphan; - if( pTab==0 ) break; - } - - if( 0==sqlite3_stricmp(pTab->zQuoted, "\"sqlite_sequence\"") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); - } - sqlite3_bind_int(pPages, 1, iRoot); - if( bRowids==0 && pTab->iPk<0 ){ - sqlite3_bind_int(pCells, 1, 1); - }else{ - sqlite3_bind_int(pCells, 1, 0); - } - sqlite3_bind_int(pCells, 3, pTab->iPk); - - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - int iPgno = sqlite3_column_int(pPages, 0); - sqlite3_bind_int(pCells, 2, iPgno); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int nField = sqlite3_column_int(pCells, 0); - int iMin = sqlite3_column_int(pCells, 2); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); - - RecoverTable *pTab2 = pTab; - if( pTab!=pOrphan && (iMin<0)!=bIntkey ){ - if( pOrphan==0 ){ - pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); - } - pTab2 = pOrphan; - if( pTab2==0 ) break; - } - - nField = nField+1; - if( pTab2==pOrphan ){ - raw_printf(pState->out, - "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", - pTab2->zQuoted, iRoot, iPgno, nField, - iMin<0 ? "" : "NULL, ", zVal, pTab2->azlCol[nField] - ); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab2->zQuoted, pTab2->azlCol[nField], zVal - ); - } - } - shellReset(&rc, pCells); - } - shellReset(&rc, pPages); - if( pTab!=pOrphan ) recoverFreeTable(pTab); - } - shellFinalize(&rc, pLoop); - shellFinalize(&rc, pPages); - shellFinalize(&rc, pCells); - recoverFreeTable(pOrphan); - - /* The rest of the schema */ - if( rc==SQLITE_OK ){ - sqlite3_stmt *pStmt = 0; - shellPrepare(pState->db, &rc, - "SELECT sql, name FROM recovery.schema " - "WHERE sql NOT LIKE 'create table%'", &pStmt - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); - if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ - const char *zName = (const char*)sqlite3_column_text(pStmt, 1); - char *zPrint = shellMPrintf(&rc, - "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", - zName, zName, zSql - ); - raw_printf(pState->out, "%s;\n", zPrint); - sqlite3_free(zPrint); - }else{ - raw_printf(pState->out, "%s;\n", zSql); - } - } - shellFinalize(&rc, pStmt); - } - - if( rc==SQLITE_OK ){ - raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); - raw_printf(pState->out, "COMMIT;\n"); - } - sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + rc = sqlite3_recover_finish(p); return rc; } #endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ From 46d439844043cef65b11e32317d54caf798cc7dd Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Sep 2022 21:43:18 +0000 Subject: [PATCH 14/42] Update comments in sqlite3recover.h. FossilOrigin-Name: 14164047c43e3ca43aa010c77ea00dfa85400e15645ee0f5b90a677898b6a836 --- ext/recover/sqlite3recover.c | 2 +- ext/recover/sqlite3recover.h | 93 ++++++++++++++++++++++++++++++------ ext/recover/test_recover.c | 2 +- manifest | 18 +++---- manifest.uuid | 2 +- src/shell.c.in | 2 +- 6 files changed, 91 insertions(+), 28 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index cef2fb9da7..3e797e4bb3 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1395,7 +1395,7 @@ static void recoverStep(sqlite3_recover *p){ p->pGetPage = 0; } -int sqlite3_recover_step(sqlite3_recover *p){ +int sqlite3_recover_run(sqlite3_recover *p){ if( p && p->errCode==SQLITE_OK ){ recoverStep(p); } diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 1ef707e13c..ffe016b4f3 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -10,6 +10,31 @@ ** ************************************************************************* ** +** This file contains the public interface to the "recover" extension - +** an SQLite extension designed to recover data from corrupted database +** files. +*/ + +/* +** OVERVIEW: +** +** To use the API to recover data from a corrupted database, an +** application: +** +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). +** +** 3) Executes the recovery by calling sqlite3_recover_run() on the handle. +** +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. +** +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). */ @@ -22,20 +47,46 @@ extern "C" { #endif +/* +** Opaque handle type. +*/ typedef struct sqlite3_recover sqlite3_recover; /* -** Create an object to recover data from database zDb (e.g. "main") -** opened by handle db. Data will be recovered into the database -** identified by parameter zUri. Database zUri is clobbered if it -** already exists. +** These two APIs attempt to create and return a new sqlite3_recover object. +** In both cases the first two arguments identify the (possibly +** corrupt) database to recover data from. The first argument is an open +** database handle and the second the name of a database attached to that +** handle (i.e. "main", "temp" or the name of an attached database). +** +** If sqlite3_recover_init() is used to create the new sqlite3_recover +** handle, then data is recovered into a new database, identified by +** string parameter zUri. zUri may be an absolute or relative file path, +** or may be an SQLite URI. If the identified database file already exists, +** it is overwritten. +** +** If sqlite3_recover_init_sql() is invoked, then any recovered data will +** be returned to the user as a series of SQL statements. Executing these +** SQL statements results in the same database as would have been created +** had sqlite3_recover_init() been used. For each SQL statement in the +** output, the callback function passed as the third argument (xSql) is +** invoked once. The first parameter is a passed a copy of the fourth argument +** to this function (pCtx) as its first parameter, and a pointer to a +** nul-terminated buffer containing the SQL statement formated as UTF-8 as +** the second. If the xSql callback returns any value other than SQLITE_OK, +** then processing is immediately abandoned and the value returned used as +** the recover handle error code (see below). +** +** If an out-of-memory error occurs, NULL may be returned instead of +** a valid handle. In all other cases, it is the responsibility of the +** application to avoid resource leaks by ensuring that +** sqlite3_recover_finish() is called on all allocated handles. */ sqlite3_recover *sqlite3_recover_init( sqlite3* db, const char *zDb, const char *zUri ); - sqlite3_recover *sqlite3_recover_init_sql( sqlite3* db, const char *zDb, @@ -43,7 +94,9 @@ sqlite3_recover *sqlite3_recover_init_sql( void *pCtx ); -/* Details TBD. */ +/* +** +*/ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); /* @@ -76,21 +129,31 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); #define SQLITE_RECOVER_FREELIST_CORRUPT 791 #define SQLITE_RECOVER_ROWIDS 792 -/* Step the recovery object. Return SQLITE_DONE if recovery is complete, -** SQLITE_OK if recovery is not complete but no error has occurred, or -** an SQLite error code if an error has occurred. +/* +** Run the recovery. Return an SQLite error code if an error occurs, or +** SQLITE_OK otherwise. */ -int sqlite3_recover_step(sqlite3_recover*); +int sqlite3_recover_run(sqlite3_recover*); +/* +** Return a pointer to a buffer containing the English language error +** message stored in the sqlite3_recover handle. If no error message +** is available (including in the case where no error has occurred), +** NULL is returned. +*/ const char *sqlite3_recover_errmsg(sqlite3_recover*); +/* +** Return the recover handle error code. SQLITE_OK is returned if no error +** has occurred. +*/ int sqlite3_recover_errcode(sqlite3_recover*); -/* Clean up a recovery object created by a call to sqlite3_recover_init(). -** This function returns SQLITE_DONE if the new database was created, -** SQLITE_OK if it processing was abandoned before it as finished or -** an SQLite error code (e.g. SQLITE_IOERR, SQLITE_NOMEM etc.) if an -** error occurred. */ +/* +** Clean up a recovery object created by a call to sqlite3_recover_init(). +** This function returns SQLITE_OK if no error occurred, or else a copy +** of the recover handle error code. +*/ int sqlite3_recover_finish(sqlite3_recover*); diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 83166e994c..e7fef42fd7 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -146,7 +146,7 @@ static int testRecoverCmd( break; } case 1: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { - int res = sqlite3_recover_step(pTest->p); + int res = sqlite3_recover_run(pTest->p); Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; } diff --git a/manifest b/manifest index 013ce9cd0f..f0dfaa0cd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sshell\sto\suse\sthe\srecover\sextension\sfor\sthe\s.recover\scommand. -D 2022-09-08T19:22:29.395 +C Update\scomments\sin\ssqlite3recover.h. +D 2022-09-08T21:43:18.230 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -392,9 +392,9 @@ F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007acef F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c b44241ca3b0cca2b50cfa7f715f79f4eddbe87d56e3baadd4e7bafe4c5872550 -F ext/recover/sqlite3recover.h 218cd9ba8c5c66c3841ca5014910982dc956cba5274257a0ecefb889db879133 -F ext/recover/test_recover.c 68b095ad396d8b1d9242ea663a4be1ad7585a46b1fc03483e9a692c8a87d2674 +F ext/recover/sqlite3recover.c aaab721cf1ade0703162e95f1d154b811eef962be42a861b59ef015514bc3b65 +F ext/recover/sqlite3recover.h 73648878cea3492427b9a8facebdad0464ed15b161ed1cd70f349a02456e8879 +F ext/recover/test_recover.c b973526785e145e7d9d7920ccde72112025ee5204e22b332db9d86796be73f00 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -597,7 +597,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ccce37e7fbe71089cf6aec91e7134c9c0c1d4840cff9f02587bbc71240d914a5 -F src/shell.c.in b36581c005ebaa59b1eeb143bd3ad5a4b273bf15380ccac63ac8cdf4f0c4d3c9 +F src/shell.c.in a805e05f3ed90e0236eb4eb5bdec327bf735d12e754b83ac037cbc22e45ecc09 F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 356d2209ea5f6b69ce15b62027c63419c2d039e52f01c74a3810a6317abf4fb0 -R 9479c02b6838b39b62d6729211ce276e +P ae832e77084eddd696c80cb926d070a5db9d45dce34156a02522b3140e8f5e8b +R 260b6b87f29011e60d0280e7bdf9107f U dan -Z df9c400431a19b9f6f7750b759f567b5 +Z c0005f6f4d932e314a4bcb73861ec3e0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8cbff6b606..e003488dbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae832e77084eddd696c80cb926d070a5db9d45dce34156a02522b3140e8f5e8b \ No newline at end of file +14164047c43e3ca43aa010c77ea00dfa85400e15645ee0f5b90a677898b6a836 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2a2aa81463..632d9633fd 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7314,7 +7314,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); - sqlite3_recover_step(p); + sqlite3_recover_run(p); if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(p); int errCode = sqlite3_recover_errcode(p); From 7920162093da6dd205cc8902e3e93fdb9592991b Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 Sep 2022 16:25:19 +0000 Subject: [PATCH 15/42] Handle recovery of virtual tables by recovering each shadow table individually, then writing the CREATE VIRTUAL TABLE statement directly into the sqlite_schema table. FossilOrigin-Name: 5f2d5ccd56c06c3468377126acfd4be39b79b05bb6fb09b674b2e185df143aa3 --- ext/recover/recover1.test | 13 +++++++++++++ ext/recover/sqlite3recover.c | 36 ++++++++++++++++++++++++++++-------- main.mk | 2 ++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/e_wal.test | 1 + 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 9afe43209f..5f1bb2b0fe 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -67,6 +67,8 @@ proc do_recover_test {tn} { sqlite3 db2 test.db2 execsql [join $::sqlhook ";"] db2 + db2 close + sqlite3 db2 test.db2 uplevel [list do_test $tn.4 [list compare_dbs db db2] {}] db2 close } @@ -154,6 +156,17 @@ do_execsql_test 8.0 { do_recover_test 8 +#------------------------------------------------------------------------- +reset_db +ifcapable fts5 { + do_execsql_test 9.1 { + CREATE VIRTUAL TABLE ft5 USING fts5(a, b); + INSERT INTO ft5 VALUES('hello', 'world'); + } + do_recover_test 9 +} + + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 3e797e4bb3..7d05ec2b3e 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -672,9 +672,17 @@ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pTblname = 0; pSelect = recoverPrepare(p, p->dbOut, - "SELECT rootpage, sql, type='table' FROM recovery.schema " - " WHERE type='table' OR (type='index' AND sql LIKE '%unique%') " - " ORDER BY type!='table', name!='sqlite_sequence'" + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isUnique) AS (" + " SELECT rootpage, name, sql, " + " type='table', " + " sql LIKE 'create virtual%'," + " (type='index' AND sql LIKE '%unique%')" + " FROM recovery.schema" + ")" + "SELECT rootpage, tbl, isVirtual, name, sql" + " FROM dbschema " + " WHERE tbl OR isUnique" + " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" ); pTblname = recoverPrepare(p, p->dbOut, @@ -685,13 +693,23 @@ static int recoverWriteSchema1(sqlite3_recover *p){ if( pSelect ){ while( sqlite3_step(pSelect)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(pSelect, 0); - const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); - int bTable = sqlite3_column_int(pSelect, 2); + int bTable = sqlite3_column_int(pSelect, 1); + int bVirtual = sqlite3_column_int(pSelect, 2); + const char *zName = (const char*)sqlite3_column_text(pSelect, 3); + const char *zSql = (const char*)sqlite3_column_text(pSelect, 4); + char *zFree = 0; + int rc = SQLITE_OK; - int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); + if( bVirtual ){ + zSql = (const char*)(zFree = recoverMPrintf(p, + "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + )); + } + rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); - if( bTable ){ + if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ const char *zName = sqlite3_column_text(pTblname, 0); recoverAddTable(p, zName, iRoot); @@ -701,6 +719,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ }else if( rc!=SQLITE_ERROR ){ recoverDbError(p, p->dbOut); } + sqlite3_free(zFree); } } recoverFinalize(p, pSelect); @@ -1130,7 +1149,8 @@ static int recoverWriteData(sqlite3_recover *p){ if( apVal==0 ) return p->errCode; pTbls = recoverPrepare(p, p->dbOut, - "SELECT rootpage FROM recovery.schema WHERE type='table'" + "SELECT rootpage FROM recovery.schema " + " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" " ORDER BY (tbl_name='sqlite_sequence') ASC" ); diff --git a/main.mk b/main.mk index 5263546042..123c7ce757 100644 --- a/main.mk +++ b/main.mk @@ -763,6 +763,8 @@ SHELL_SRC = \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/manifest b/manifest index f0dfaa0cd4..3da6b77b47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\scomments\sin\ssqlite3recover.h. -D 2022-09-08T21:43:18.230 +C Handle\srecovery\sof\svirtual\stables\sby\srecovering\seach\sshadow\stable\sindividually,\sthen\swriting\sthe\sCREATE\sVIRTUAL\sTABLE\sstatement\sdirectly\sinto\sthe\ssqlite_schema\stable. +D 2022-09-09T16:25:19.064 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,12 +387,12 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test aa9f7f46b7209cae6d52321052d4440bc8f82b93991e693c4bad20a6f05a53e5 +F ext/recover/recover1.test 942016356f9098ca36933536b194b5878827a3a749e0bf41a83d83530c0d0ea8 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c aaab721cf1ade0703162e95f1d154b811eef962be42a861b59ef015514bc3b65 +F ext/recover/sqlite3recover.c 7ed9a573d081435a16982b972c6bb963d273a0a457ec69fa16b0918165991e9d F ext/recover/sqlite3recover.h 73648878cea3492427b9a8facebdad0464ed15b161ed1cd70f349a02456e8879 F ext/recover/test_recover.c b973526785e145e7d9d7920ccde72112025ee5204e22b332db9d86796be73f00 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -517,7 +517,7 @@ F ext/wasm/testing2.js d37433c601f88ed275712c1cfc92d3fb36c7c22e1ed8c7396fb2359e4 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 8c9965c408aaa8b93d0dd52e83445894835e1a42dc360c77435393f80f8d8d1d +F main.mk 3e2a50cc103bd3386bbac9c96bf3650f91174b26e092701164aa42875bff24c9 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -931,7 +931,7 @@ F test/e_totalchanges.test c927f7499dc3aa28b9b556b7d6d115a2f0fe41f012b128d16bf1f F test/e_update.test f46c2554d915c9197548681e8d8c33a267e84528 F test/e_uri.test 86564382132d9c453845eeb5293c7e375487b625900ab56c181a0464908417d8 F test/e_vacuum.test 89fc48e8beee2f9dfd6de1fbb2edea6542dae9121dc0fbe6313764169e742104 -F test/e_wal.test ae9a593207a77d711443ee69ffe081fda9243625 +F test/e_wal.test db7c33642711cf3c7959714b5f012aca08cacfa78da0382f95e849eb3ba66aa4 F test/e_walauto.test 248af31e73c98df23476a22bdb815524c9dc3ba8 F test/e_walckpt.test 28c371a6bb5e5fe7f31679c1df1763a19d19e8a0 F test/e_walhook.test 01b494287ba9e60b70f6ebf3c6c62e0ffe01788e344a4846b08e5de0b344cb66 @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae832e77084eddd696c80cb926d070a5db9d45dce34156a02522b3140e8f5e8b -R 260b6b87f29011e60d0280e7bdf9107f +P 14164047c43e3ca43aa010c77ea00dfa85400e15645ee0f5b90a677898b6a836 +R e52e610730a5083b962d51491c2e90a5 U dan -Z c0005f6f4d932e314a4bcb73861ec3e0 +Z 0661ae93944ad96dd9dbcc27159d9d3c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e003488dbc..c6ebbcda0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14164047c43e3ca43aa010c77ea00dfa85400e15645ee0f5b90a677898b6a836 \ No newline at end of file +5f2d5ccd56c06c3468377126acfd4be39b79b05bb6fb09b674b2e185df143aa3 \ No newline at end of file diff --git a/test/e_wal.test b/test/e_wal.test index 77ac83a0ae..c9c5e9643f 100644 --- a/test/e_wal.test +++ b/test/e_wal.test @@ -15,6 +15,7 @@ source $testdir/tester.tcl set testprefix e_wal db close +forcedelete test.db-shm testvfs oldvfs -iversion 1 From bc2e7fc228f2a8c41bb1702aff1a9f450f8bfcda Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 Sep 2022 20:44:56 +0000 Subject: [PATCH 16/42] Further fixes to comments in sqlite3recover.h. Also rework some data structures in sqlite3recover.c. FossilOrigin-Name: 599d1f8ec2f9e24924a6f9e66c85664360c7b95531b07a4efe1dd8c096b3fc99 --- ext/recover/sqlite3recover.c | 140 ++++++++++++++++++++++++++--------- ext/recover/sqlite3recover.h | 82 +++++++++++++++----- ext/recover/test_recover.c | 2 +- manifest | 18 ++--- manifest.uuid | 2 +- test/savepoint.test | 2 + 6 files changed, 181 insertions(+), 65 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 7d05ec2b3e..e6975b8771 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -20,7 +20,76 @@ typedef unsigned int u32; typedef sqlite3_int64 i64; +typedef struct RecoverTable RecoverTable; typedef struct RecoverColumn RecoverColumn; + +/* +** When recovering rows of data that can be associated with table +** definitions recovered from the sqlite_schema table, each table is +** represented by an instance of the following object. +** +** iRoot: +** The root page in the original database. Not necessarily (and usually +** not) the same in the recovered database. +** +** zTab: +** Name of the table. +** +** nCol/aCol[]: +** aCol[] is an array of nCol columns. In the order in which they appear +** in the table. +** +** bIntkey: +** Set to true for intkey tables, false for WITHOUT ROWID. +** +** iRowidBind: +** Each column in the aCol[] array has associated with it the index of +** the bind parameter its values will be bound to in the INSERT statement +** used to construct the output database. If the table does has a rowid +** but not an INTEGER PRIMARY KEY column, then iRowidBind contains the +** index of the bind paramater to which the rowid value should be bound. +** Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY +** KEY column, then the rowid value should be bound to the index associated +** with the column. +** +** pNext: +** All RecoverTable objects used by the recovery operation are allocated +** and populated as part of creating the recovered database schema in +** the output database, before any non-schema data are recovered. They +** are then stored in a singly-linked list linked by this variable beginning +** at sqlite3_recover.pTblList. +*/ +struct RecoverTable { + u32 iRoot; /* Root page in original database */ + char *zTab; /* Name of table */ + int nCol; /* Number of columns in table */ + RecoverColumn *aCol; /* Array of columns */ + int bIntkey; /* True for intkey, false for without rowid */ + int iRowidBind; /* If >0, bind rowid to INSERT here */ + RecoverTable *pNext; +}; + +/* +** Each database column is represented by an instance of the following object +** stored in the RecoverTable.aCol[] array of the associated table. +** +** iField: +** The index of the associated field within database records. Or -1 if +** there is no associated field (e.g. for virtual generated columns). +** +** iBind: +** The bind index of the INSERT statement to bind this columns values +** to. Or 0 if there is no such index (iff (iField<0)). +** +** bIPK: +** True if this is the INTEGER PRIMARY KEY column. +** +** zCol: +** Name of column. +** +** eHidden: +** A RECOVER_EHIDDEN_* constant value (see below for interpretation of each). +*/ struct RecoverColumn { int iField; /* Field in record on disk */ int iBind; /* Binding to use in INSERT */ @@ -29,59 +98,62 @@ struct RecoverColumn { int eHidden; }; -#define RECOVER_EHIDDEN_NONE 0 -#define RECOVER_EHIDDEN_HIDDEN 1 -#define RECOVER_EHIDDEN_VIRTUAL 2 -#define RECOVER_EHIDDEN_STORED 3 +#define RECOVER_EHIDDEN_NONE 0 /* Normal database column */ +#define RECOVER_EHIDDEN_HIDDEN 1 /* Column is __HIDDEN__ */ +#define RECOVER_EHIDDEN_VIRTUAL 2 /* Virtual generated column */ +#define RECOVER_EHIDDEN_STORED 3 /* Stored generated column */ /* -** When running the ".recover" command, each output table, and the special -** orphaned row table if it is required, is represented by an instance -** of the following struct. +** Bitmap object used to track pages in the input database. Allocated +** and manipulated only by the following functions: ** -** aCol[]: -** Array of nCol columns. In the order in which they appear in the table. +** recoverBitmapAlloc() +** recoverBitmapFree() +** recoverBitmapSet() +** recoverBitmapQuery() +** +** nPg: +** Largest page number that may be stored in the bitmap. The range +** of valid keys is 1 to nPg, inclusive. +** +** aElem[]: +** Array large enough to contain a bit for each key. For key value +** iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32]. +** In other words, the following is true if bit iKey is set, or +** false if it is clear: +** +** (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0 */ -typedef struct RecoverTable RecoverTable; -struct RecoverTable { - u32 iRoot; /* Root page in original database */ - char *zTab; /* Name of table */ - int nCol; /* Number of columns in table */ - RecoverColumn *aCol; /* Array of columns */ - int bIntkey; /* True for intkey, false for without rowid */ - int iRowidBind; /* If >0, bind rowid to INSERT here */ - - RecoverTable *pNext; -}; - typedef struct RecoverBitmap RecoverBitmap; struct RecoverBitmap { i64 nPg; /* Size of bitmap */ u32 aElem[0]; /* Array of 32-bit bitmasks */ }; - +/* +** +*/ struct sqlite3_recover { - sqlite3 *dbIn; - sqlite3 *dbOut; + sqlite3 *dbIn; /* Input database */ + sqlite3 *dbOut; /* Output database */ - sqlite3_stmt *pGetPage; + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ - char *zDb; - char *zUri; - RecoverTable *pTblList; + char *zDb; /* Name of input db ("main" etc.) */ + char *zUri; /* URI for output database */ + RecoverTable *pTblList; /* List of tables recovered from schem */ RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ - char *zStateDb; + char *zStateDb; /* State database to use (or NULL) */ char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ - int bFreelistCorrupt; - int bRecoverRowid; + int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ + int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ - void *pSqlCtx; - int (*xSql)(void*,const char*); + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ }; /* @@ -1340,7 +1412,7 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ int rc = SQLITE_OK; switch( op ){ - case SQLITE_RECOVER_TESTDB: + case 789: sqlite3_free(p->zStateDb); p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index ffe016b4f3..30771d8804 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -37,7 +37,6 @@ ** using sqlite3_recover_finish(). */ - #ifndef _SQLITE_RECOVER_H #define _SQLITE_RECOVER_H @@ -95,13 +94,18 @@ sqlite3_recover *sqlite3_recover_init_sql( ); /* +** Configure an sqlite3_recover object that has just been created using +** sqlite3_recover_init() or sqlite3_recover_init_sql(). The second +** argument passed to this function must be one of the SQLITE_RECOVER_* +** symbols defined below. Valid values for the third argument depend +** on the specific SQLITE_RECOVER_* symbol in use. ** +** SQLITE_OK is returned if the configuration operation was successful, +** or an SQLite error code otherwise. */ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); /* -** SQLITE_RECOVER_TESTDB: -** ** SQLITE_RECOVER_LOST_AND_FOUND: ** The pArg argument points to a string buffer containing the name ** of a "lost-and-found" table in the output database, or NULL. If @@ -118,41 +122,79 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** corrupt and an attempt is made to recover records from pages that ** appear to be linked into the freelist. Otherwise, pages on the freelist ** are ignored. Setting this option can recover more data from the -** database, but often ends up "recovering" deleted records. +** database, but often ends up "recovering" deleted records. The default +** value is 0 (clear). ** ** SQLITE_RECOVER_ROWIDS: -** -** SQLITE_RECOVER_SQLHOOK: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is set +** (argument is 1), then an attempt is made to recover rowid values +** that are not also INTEGER PRIMARY KEY values. If this option is +** clear, then new rowids are assigned to all recovered rows. The +** default value is 1 (set). */ -#define SQLITE_RECOVER_TESTDB 789 -#define SQLITE_RECOVER_LOST_AND_FOUND 790 -#define SQLITE_RECOVER_FREELIST_CORRUPT 791 -#define SQLITE_RECOVER_ROWIDS 792 +#define SQLITE_RECOVER_LOST_AND_FOUND 1 +#define SQLITE_RECOVER_FREELIST_CORRUPT 2 +#define SQLITE_RECOVER_ROWIDS 3 /* -** Run the recovery. Return an SQLite error code if an error occurs, or -** SQLITE_OK otherwise. +** Run the recovery operation. This function does not return until the +** recovery operation is completed - either the new database has been +** created and populated (sqlite3_recover_init()) or all SQL statements have +** been passed to the callback (sqlite3_recover_init_sql()) - or an error +** occurs. If the recovery is completed without error, SQLITE_OK +** is returned. It is not considered an error if data cannot be recovered +** due to database corruption. +** +** If an error (for example an out-of-memory or IO error) occurs, then +** an SQLite error code is returned. The final state of the output database +** or the results of running any SQL statements already passed to the +** callback in this case are undefined. An English language error +** message corresponding to the error may be available via the +** sqlite3_recover_errmsg() API. +** +** This function may only be called once on an sqlite3_recover handle. +** If it is called more than once, the second and subsequent calls +** return SQLITE_MISUSE. The error code and error message returned +** by sqlite3_recover_errcode() and sqlite3_recover_errmsg() are not +** updated in this case. */ int sqlite3_recover_run(sqlite3_recover*); /* -** Return a pointer to a buffer containing the English language error -** message stored in the sqlite3_recover handle. If no error message -** is available (including in the case where no error has occurred), -** NULL is returned. +** If this is called on an sqlite3_recover handle before +** sqlite3_recover_run() has been called, or if the call to +** sqlite3_recover_run() returned SQLITE_OK, then this API always returns +** a NULL pointer. +** +** Otherwise, an attempt is made to return a pointer to a buffer containing +** an English language error message related to the error that occurred +** within the sqlite3_recover_run() call. If no error message is available, +** or if an out-of-memory error occurs while attempting to allocate a buffer +** for one, NULL may still be returned. +** +** The buffer remains valid until the sqlite3_recover handle is destroyed +** using sqlite3_recover_finish(). */ const char *sqlite3_recover_errmsg(sqlite3_recover*); /* -** Return the recover handle error code. SQLITE_OK is returned if no error -** has occurred. +** If this function is called on an sqlite3_recover handle before +** sqlite3_recover_run() has been called, it always returns SQLITE_OK. +** Otherwise, it returns a copy of the value returned by the first +** sqlite3_recover_run() call made on the handle. */ int sqlite3_recover_errcode(sqlite3_recover*); /* ** Clean up a recovery object created by a call to sqlite3_recover_init(). -** This function returns SQLITE_OK if no error occurred, or else a copy -** of the recover handle error code. +** The results of using a recovery object with any API after it has been +** passed to this function are undefined. +** +** If this function is called on an sqlite3_recover handle before +** sqlite3_recover_run() has been called, it always returns SQLITE_OK. +** Otherwise, it returns a copy of the value returned by the first +** sqlite3_recover_run() call made on the handle. */ int sqlite3_recover_finish(sqlite3_recover*); diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index e7fef42fd7..a31ddbb94a 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -117,7 +117,7 @@ static int testRecoverCmd( switch( iOp ){ case 0: res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_TESTDB, (void*)Tcl_GetString(objv[3]) + 789, (void*)Tcl_GetString(objv[3]) /* MAGIC NUMBER! */ ); break; case 1: diff --git a/manifest b/manifest index 3da6b77b47..3f230d9467 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\srecovery\sof\svirtual\stables\sby\srecovering\seach\sshadow\stable\sindividually,\sthen\swriting\sthe\sCREATE\sVIRTUAL\sTABLE\sstatement\sdirectly\sinto\sthe\ssqlite_schema\stable. -D 2022-09-09T16:25:19.064 +C Further\sfixes\sto\scomments\sin\ssqlite3recover.h.\sAlso\srework\ssome\sdata\sstructures\sin\ssqlite3recover.c. +D 2022-09-09T20:44:56.135 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -392,9 +392,9 @@ F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007acef F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 7ed9a573d081435a16982b972c6bb963d273a0a457ec69fa16b0918165991e9d -F ext/recover/sqlite3recover.h 73648878cea3492427b9a8facebdad0464ed15b161ed1cd70f349a02456e8879 -F ext/recover/test_recover.c b973526785e145e7d9d7920ccde72112025ee5204e22b332db9d86796be73f00 +F ext/recover/sqlite3recover.c e8d0eae7da7ba24e733f6247082c12d7a030ce794eb5c42d64993f55c12882f5 +F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 +F ext/recover/test_recover.c ed8d0cc8703ab29cf562f793623b045de109b7937f254108ff4132f35abb37fb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -1398,7 +1398,7 @@ F test/rowvaluefault.test 963ae9cdaed30a85a29668dd514e639f3556cae903ee9f172ea972 F test/rowvaluevtab.test cd9747bb3f308086944c07968f547ad6b05022e698d80b9ffbdfe09ce0b8da6f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 1f8a6b1aea9a0d05837adc463d4bf47bd9d0f1c842f1c2a9caccd639baf34bf9 +F test/savepoint.test 6e9804a17767f08432c7a5e738b9a8f4b891d243110b63d3a41d270d3d1378ec F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 14164047c43e3ca43aa010c77ea00dfa85400e15645ee0f5b90a677898b6a836 -R e52e610730a5083b962d51491c2e90a5 +P 5f2d5ccd56c06c3468377126acfd4be39b79b05bb6fb09b674b2e185df143aa3 +R d4558eaeefd2a718275b323dc4307f9b U dan -Z 0661ae93944ad96dd9dbcc27159d9d3c +Z 6e64e7331e56e0c44a533586bb618920 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c6ebbcda0e..a1b607602a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f2d5ccd56c06c3468377126acfd4be39b79b05bb6fb09b674b2e185df143aa3 \ No newline at end of file +599d1f8ec2f9e24924a6f9e66c85664360c7b95531b07a4efe1dd8c096b3fc99 \ No newline at end of file diff --git a/test/savepoint.test b/test/savepoint.test index eed8a9e702..3952981dab 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -16,6 +16,8 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl +forcedelete test2.db + #---------------------------------------------------------------------- # The following tests - savepoint-1.* - test that the SAVEPOINT, RELEASE # and ROLLBACK TO comands are correctly parsed, and that the auto-commit From 65660916dc906dc2323b704caf937e1c0cfc6c9b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 10 Sep 2022 20:01:49 +0000 Subject: [PATCH 17/42] Rework recover extension code for readability. FossilOrigin-Name: 1a2540960e40e3c8c622448fd3862e249bd463c29ae4ce5e39942e942533f60a --- ext/recover/sqlite3recover.c | 516 ++++++++++++++++++++++++++++------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 421 insertions(+), 109 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index e6975b8771..9c5e10433b 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -17,6 +17,16 @@ #include #include +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + typedef unsigned int u32; typedef sqlite3_int64 i64; @@ -131,29 +141,32 @@ struct RecoverBitmap { }; /* -** +** Main recover handle structure. */ struct sqlite3_recover { + /* Copies of sqlite3_recover_init[_sql]() parameters */ sqlite3 *dbIn; /* Input database */ - sqlite3 *dbOut; /* Output database */ - - sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ - char *zDb; /* Name of input db ("main" etc.) */ char *zUri; /* URI for output database */ - RecoverTable *pTblList; /* List of tables recovered from schem */ - RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ - - int errCode; /* For sqlite3_recover_errcode() */ - char *zErrMsg; /* For sqlite3_recover_errmsg() */ + void *pSqlCtx; /* SQL callback context */ + int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ + /* Values configured by sqlite3_recover_config() */ char *zStateDb; /* State database to use (or NULL) */ char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ - void *pSqlCtx; /* SQL callback context */ - int (*xSql)(void*,const char*); /* Pointer to SQL callback function */ + /* Error code and error message */ + int errCode; /* For sqlite3_recover_errcode() */ + char *zErrMsg; /* For sqlite3_recover_errmsg() */ + + /* Fields used within sqlite3_recover_run() */ + int bRun; /* True once _recover_run() has been called */ + sqlite3 *dbOut; /* Output database */ + sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ + RecoverTable *pTblList; /* List of tables recovered from schem */ + RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ }; /* @@ -172,6 +185,15 @@ static int recoverStrlen(const char *zStr){ return nRet; } +/* +** This function is a no-op if the recover handle passed as the first +** argument already contains an error (if p->errCode!=SQLITE_OK). +** +** Otherwise, an attempt is made to allocate, zero and return a buffer nByte +** bytes in size. If successful, a pointer to the new buffer is returned. Or, +** if an OOM error occurs, NULL is returned and the handle error code +** (p->errCode) set to SQLITE_NOMEM. +*/ static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ void *pRet = 0; assert( nByte>0 ); @@ -186,6 +208,19 @@ static void *recoverMalloc(sqlite3_recover *p, i64 nByte){ return pRet; } +/* +** Set the error code and error message for the recover handle passed as +** the first argument. The error code is set to the value of parameter +** errCode. +** +** Parameter zFmt must be a printf() style formatting string. The handle +** error message is set to the result of using any trailing arguments for +** parameter substitutions in the formatting string. +** +** For example: +** +** recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename); +*/ static int recoverError( sqlite3_recover *p, int errCode, @@ -204,6 +239,14 @@ static int recoverError( } +/* +** This function is a no-op if p->errCode is initially other than SQLITE_OK. +** In this case it returns NULL. +** +** Otherwise, an attempt is made to allocate and return a bitmap object +** large enough to store a bit for all page numbers between 1 and nPg, +** inclusive. The bitmap is initially zeroed. +*/ static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ int nElem = (nPg+1+31) / 32; int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32); @@ -215,10 +258,16 @@ static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){ return pRet; } +/* +** Free a bitmap object allocated by recoverBitmapAlloc(). +*/ static void recoverBitmapFree(RecoverBitmap *pMap){ sqlite3_free(pMap); } +/* +** Set the bit associated with page iPg in bitvec pMap. +*/ static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ if( iPg<=pMap->nPg ){ int iElem = (iPg / 32); @@ -227,6 +276,10 @@ static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ } } +/* +** Query bitmap object pMap for the state of the bit associated with page +** iPg. Return 1 if it is set, or 0 otherwise. +*/ static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ int ret = 1; if( iPg<=pMap->nPg ){ @@ -237,11 +290,24 @@ static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ return ret; } - +/* +** Set the recover handle error to the error code and message returned by +** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database +** handle db. +*/ static int recoverDbError(sqlite3_recover *p, sqlite3 *db){ return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db)); } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, it attempts to prepare the SQL statement in zSql against +** database handle db. If successful, the statement handle is returned. +** Or, if an error occurs, NULL is returned and an error left in the +** recover handle. +*/ static sqlite3_stmt *recoverPrepare( sqlite3_recover *p, sqlite3 *db, @@ -257,7 +323,15 @@ static sqlite3_stmt *recoverPrepare( } /* -** Create a prepared statement using printf-style arguments for the SQL. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zFmt is used as a printf() style format string, +** along with any trailing arguments, to create an SQL statement. This +** SQL statement is prepared against database handle db and, if successful, +** the statment handle returned. Or, if an error occurs - either during +** the printf() formatting or when preparing the resulting SQL - an +** error code and message are left in the recover handle. */ static sqlite3_stmt *recoverPreparePrintf( sqlite3_recover *p, @@ -281,7 +355,15 @@ static sqlite3_stmt *recoverPreparePrintf( return pStmt; } - +/* +** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +** +** This function returns a copy of the statement handle pointer passed +** as the second argument. +*/ static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ int rc = sqlite3_reset(pStmt); if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){ @@ -290,6 +372,12 @@ static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){ return pStmt; } +/* +** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() +** indicates that an error occurred, and there is not already an error +** in the recover handle passed as the first argument, set the error +** code and error message appropriately. +*/ static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ sqlite3 *db = sqlite3_db_handle(pStmt); int rc = sqlite3_finalize(pStmt); @@ -298,6 +386,15 @@ static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){ } } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this +** case. +** +** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK. +** Or, if an error occurs, leave an error code and message in the recover +** handle and return a copy of the error code. +*/ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ if( p->errCode==SQLITE_OK ){ int rc = sqlite3_exec(db, zSql, 0, 0, 0); @@ -308,6 +405,20 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). NULL is returned in this case. +** +** Otherwise, an attempt is made to interpret zFmt as a printf() style +** formatting string and the result of using the trailing arguments for +** parameter substitution with it written into a buffer obtained from +** sqlite3_malloc(). If successful, a pointer to the buffer is returned. +** It is the responsibility of the caller to eventually free the buffer +** using sqlite3_free(). +** +** Or, if an error occurs, an error code and message is left in the recover +** handle and NULL returned. +*/ static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ va_list ap; char *z; @@ -324,9 +435,13 @@ static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){ } /* -** Execute "PRAGMA page_count" against the input database. If successful, -** return the integer result. Or, if an error occurs, leave an error code -** and error message in the sqlite3_recover handle. +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). Zero is returned in this case. +** +** Otherwise, execute "PRAGMA page_count" against the input database. If +** successful, return the integer result. Or, if an error occurs, leave an +** error code and error message in the sqlite3_recover handle and return +** zero. */ static i64 recoverPageCount(sqlite3_recover *p){ i64 nPg = 0; @@ -342,10 +457,12 @@ static i64 recoverPageCount(sqlite3_recover *p){ } /* -** Scalar function "read_i32". The first argument to this function -** must be a blob. The second a non-negative integer. This function -** reads and returns a 32-bit big-endian integer from byte +** Implementation of SQL scalar function "read_i32". The first argument to +** this function must be a blob. The second a non-negative integer. This +** function reads and returns a 32-bit big-endian integer from byte ** offset (4*) of the blob. +** +** SELECT read_i32(, ) */ static void recoverReadI32( sqlite3_context *context, @@ -372,7 +489,16 @@ static void recoverReadI32( } /* -** SELECT page_is_used(pgno); +** Implementation of SQL scalar function "page_is_used". This function +** is used as part of the procedure for locating orphan rows for the +** lost-and-found table, and it depends on those routines having populated +** the sqlite3_recover.pUsed variable. +** +** The only argument to this function is a page-number. It returns true +** if the page has already been used somehow during data recovery, or false +** otherwise. +** +** SELECT page_is_used(); */ static void recoverPageIsUsed( sqlite3_context *pCtx, @@ -394,7 +520,7 @@ static void recoverPageIsUsed( ** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages ** of the database being recovered. ** -** This function always takes a single integer argument. If the arguement +** This function always takes a single integer argument. If the argument ** is zero, then the value returned is the number of pages in the db being ** recovered. If the argument is greater than zero, it is a page number. ** The value returned in this case is an SQL blob containing the data for @@ -462,18 +588,17 @@ static const char *recoverUnusedString( return zBuf; } - /* -** Scalar function "escape_crnl". The argument passed to this function is the -** output of built-in function quote(). If the first character of the input is -** "'", indicating that the value passed to quote() was a text value, then this -** function searches the input for "\n" and "\r" characters and adds a wrapper -** similar to the following: +** Implementation of scalar SQL function "escape_crnl". The argument passed to +** this function is the output of built-in function quote(). If the first +** character of the input is "'", indicating that the value passed to quote() +** was a text value, then this function searches the input for "\n" and "\r" +** characters and adds a wrapper similar to the following: ** ** replace(replace(, '\n', char(10), '\r', char(13)); ** -** Or, if the first character of the input is not "'", then a copy -** of the input is returned. +** Or, if the first character of the input is not "'", then a copy of the input +** is returned. */ static void recoverEscapeCrnl( sqlite3_context *context, @@ -552,12 +677,20 @@ static void recoverEscapeCrnl( sqlite3_result_value(context, argv[0]); } - -#ifdef _WIN32 -__declspec(dllexport) -#endif -int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); - +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, an attempt is made to open the output database, attach +** and create the schema of the temporary database used to store +** intermediate data, and to register all required user functions and +** virtual table modules with the output handle. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. +*/ static int recoverOpenOutput(sqlite3_recover *p){ struct Func { const char *zName; @@ -576,17 +709,19 @@ static int recoverOpenOutput(sqlite3_recover *p){ assert( p->dbOut==0 ); - if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ - recoverDbError(p, db); - }else{ - char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); - recoverExec(p, db, zSql); - recoverExec(p, db, - "PRAGMA writable_schema = 1;" - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - ); - sqlite3_free(zSql); + if( p->errCode==SQLITE_OK ){ + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); + }else{ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, db, zSql); + recoverExec(p, db, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + ); + sqlite3_free(zSql); + } } /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. @@ -624,6 +759,19 @@ static int recoverOpenOutput(sqlite3_recover *p){ return p->errCode; } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. +*/ static int recoverCacheSchema(sqlite3_recover *p){ return recoverExec(p, p->dbOut, "WITH RECURSIVE pages(p) AS (" @@ -643,7 +791,24 @@ static int recoverCacheSchema(sqlite3_recover *p){ ); } -static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). +** +** Otherwise, argument zName must be the name of a table that has just been +** created in the output database. This function queries the output db +** for the schema of said table, and creates a RecoverTable object to +** store the schema in memory. The new RecoverTable object is linked into +** the list at sqlite3_recover.pTblList. +** +** Parameter iRoot must be the root page of table zName in the INPUT +** database. +*/ +static void recoverAddTable( + sqlite3_recover *p, + const char *zName, /* Name of table created in output db */ + i64 iRoot /* Root page of same table in INPUT db */ +){ sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA table_xinfo(%Q)", zName ); @@ -727,6 +892,15 @@ static void recoverAddTable(sqlite3_recover *p, const char *zName, i64 iRoot){ } } +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ if( p->errCode==SQLITE_OK && p->xSql ){ int res = p->xSql(p->pSqlCtx, zSql); @@ -737,7 +911,23 @@ static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ } /* +** This function is called after recoverCacheSchema() has cached those parts +** of the input database schema that could be recovered in temporary table +** "recovery.schema". This function creates in the output database copies +** of all parts of that schema that must be created before the tables can +** be populated. Specifically, this means: ** +** * all tables that are not VIRTUAL, and +** * UNIQUE indexes. +** +** If the recovery handle uses SQL callbacks, then callbacks containing +** the associated "CREATE TABLE" and "CREATE INDEX" statements are made. +** +** Additionally, records are added to the sqlite_schema table of the +** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE +** records are written directly to sqlite_schema, not actually executed. +** If the handle is in SQL callback mode, then callbacks are invoked +** with equivalent SQL statements. */ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; @@ -800,6 +990,19 @@ static int recoverWriteSchema1(sqlite3_recover *p){ return p->errCode; } +/* +** This function is called after the output database has been populated. It +** adds all recovered schema elements that were not created in the output +** database by recoverWriteSchema1() - everything except for tables and +** UNIQUE indexes. Specifically: +** +** * views, +** * triggers, +** * non-UNIQUE indexes. +** +** If the recover handle is in SQL callback mode, then equivalent callbacks +** are issued to create the schema elements. +*/ static int recoverWriteSchema2(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; @@ -825,6 +1028,41 @@ static int recoverWriteSchema2(sqlite3_recover *p){ return p->errCode; } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). In this case it returns NULL. +** +** Otherwise, if the recover handle is configured to create an output +** database (was created by sqlite3_recover_init()), then this function +** prepares and returns an SQL statement to INSERT a new record into table +** pTab, assuming the first nField fields of a record extracted from disk +** are valid. +** +** For example, if table pTab is: +** +** CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e); +** +** And nField is 4, then the SQL statement prepared and returned is: +** +** INSERT INTO (a, c, d) VALUES (?1, ?2, ?3); +** +** In this case even though 4 values were extracted from the input db, +** only 3 are written to the output, as the generated STORED column +** cannot be written. +** +** If the recover handle is in SQL callback mode, then the SQL statement +** prepared is such that evaluating it returns a single row containing +** a single text value - itself an SQL statement similar to the above, +** except with SQL literals in place of the variables. For example: +** +** SELECT 'INSERT INTO (a, c, d) VALUES (' +** || quote(?1) || ', ' +** || quote(?2) || ', ' +** || quote(?3) || ')'; +** +** In either case, it is the responsibility of the caller to eventually +** free the statement handle using sqlite3_finalize(). +*/ static sqlite3_stmt *recoverInsertStmt( sqlite3_recover *p, RecoverTable *pTab, @@ -855,7 +1093,6 @@ static sqlite3_stmt *recoverInsertStmt( zSep = ", "; } - for(ii=0; iiaCol[ii].eHidden; if( eHidden!=RECOVER_EHIDDEN_VIRTUAL @@ -893,6 +1130,11 @@ static sqlite3_stmt *recoverInsertStmt( } +/* +** Search the list of RecoverTable objects at p->pTblList for one that +** has root page iRoot in the input database. If such an object is found, +** return a pointer to it. Otherwise, return NULL. +*/ static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){ RecoverTable *pRet = 0; for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext); @@ -1007,6 +1249,9 @@ static sqlite3_stmt *recoverLostAndFoundInsert( return pRet; } +/* +** Helper function for recoverLostAndFound(). +*/ static void recoverLostAndFoundPopulate( sqlite3_recover *p, sqlite3_stmt *pInsert, @@ -1096,6 +1341,12 @@ static void recoverLostAndFoundPopulate( sqlite3_free(apVal); } +/* +** This function searches for orphaned rows in the input database. If +** any are found, it creates the lost-and-found table in the output +** db and writes all orphaned rows to it. Or, if the recover handle is +** in SQL callback mode, issues equivalent callbacks. +*/ static int recoverLostAndFound(sqlite3_recover *p){ i64 nPg = 0; RecoverBitmap *pMap = 0; @@ -1198,12 +1449,18 @@ static int recoverLostAndFound(sqlite3_recover *p){ recoverFinalize(p, pInsert); sqlite3_free(zTab); } - - recoverBitmapFree(pMap); - p->pUsed = 0; } } +/* +** For each table in the recovered schema, this function extracts as much +** data as possible from the output database and writes it to the input +** database. Or, if the recover handle is in SQL callback mode, issues +** equivalent callbacks. +** +** It does not recover "orphaned" data into the lost-and-found table. +** See recoverLostAndFound() for that. +*/ static int recoverWriteData(sqlite3_recover *p){ RecoverTable *pTbl; int nMax = 0; @@ -1349,12 +1606,79 @@ static int recoverWriteData(sqlite3_recover *p){ return p->errCode; } +/* +** This function does the work of sqlite3_recover_run(). It is assumed that +** no error has occurred when this is called. If an error occurs during +** the recovery operation, an error code and error message are left in +** the recovery handle. +*/ +static void recoverRun(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + int rc = SQLITE_OK; + + assert( p->errCode==SQLITE_OK ); + assert( p->bRun==0 ); + p->bRun = 1; + + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + recoverExec(p, p->dbIn, "BEGIN"); + recoverExec(p, p->dbOut, "BEGIN"); + + recoverCacheSchema(p); + recoverWriteSchema1(p); + recoverWriteData(p); + if( p->zLostAndFound ) recoverLostAndFound(p); + recoverWriteSchema2(p); + + /* If no error has occurred, commit the write transaction on the output + ** database. Then end the read transaction on the input database, regardless + ** of whether or not prior errors have occurred. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + + /* Clean up various resources allocated by this function. */ + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + recoverBitmapFree(p->pUsed); + p->pUsed = 0; + sqlite3_close(p->dbOut); +} + + +/* +** This is a worker function that does the heavy lifting for both init +** functions: +** +** sqlite3_recover_init() +** sqlite3_recover_init_sql() +** +** All this function does is allocate space for the recover handle and +** take copies of the input parameters. All the real work is done within +** sqlite3_recover_run(). +*/ sqlite3_recover *recoverInit( sqlite3* db, const char *zDb, - const char *zUri, - int (*xSql)(void*, const char*), - void *pSqlCtx + const char *zUri, /* Output URI for _recover_init() */ + int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */ + void *pSqlCtx /* Context arg for _recover_init_sql() */ ){ sqlite3_recover *pRet = 0; int nDb = 0; @@ -1384,6 +1708,10 @@ sqlite3_recover *recoverInit( return pRet; } +/* +** Initialize a recovery handle that creates a new database containing +** the recovered data. +*/ sqlite3_recover *sqlite3_recover_init( sqlite3* db, const char *zDb, @@ -1392,6 +1720,10 @@ sqlite3_recover *sqlite3_recover_init( return recoverInit(db, zDb, zUri, 0, 0); } +/* +** Initialize a recovery handle that returns recovered data in the +** form of SQL statements via a callback. +*/ sqlite3_recover *sqlite3_recover_init_sql( sqlite3* db, const char *zDb, @@ -1401,13 +1733,23 @@ sqlite3_recover *sqlite3_recover_init_sql( return recoverInit(db, zDb, "", xSql, pSqlCtx); } +/* +** Return the handle error message, if any. +*/ const char *sqlite3_recover_errmsg(sqlite3_recover *p){ return p ? p->zErrMsg : "not an error"; } + +/* +** Return the handle error code. +*/ int sqlite3_recover_errcode(sqlite3_recover *p){ return p ? p->errCode : SQLITE_NOMEM; } +/* +** Configure the handle. +*/ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ int rc = SQLITE_OK; @@ -1443,57 +1785,27 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ return rc; } -static void recoverStep(sqlite3_recover *p){ - RecoverTable *pTab = 0; - RecoverTable *pNext = 0; - int rc = SQLITE_OK; - assert( p->errCode==SQLITE_OK ); - - recoverSqlCallback(p, "BEGIN"); - recoverSqlCallback(p, "PRAGMA writable_schema = on"); - - /* Open the output database. And register required virtual tables and - ** user functions with the new handle. */ - recoverOpenOutput(p); - - /* Open transactions on both the input and output databases. */ - recoverExec(p, p->dbIn, "BEGIN"); - recoverExec(p, p->dbOut, "BEGIN"); - - recoverCacheSchema(p); - recoverWriteSchema1(p); - recoverWriteData(p); - if( p->zLostAndFound ) recoverLostAndFound(p); - recoverWriteSchema2(p); - - /* If no error has occurred, commit the write transaction on the output - ** database. Then end the read transaction on the input database, regardless - ** of whether or not prior errors have occurred. */ - recoverExec(p, p->dbOut, "COMMIT"); - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); - if( p->errCode==SQLITE_OK ) p->errCode = rc; - - recoverSqlCallback(p, "PRAGMA writable_schema = off"); - recoverSqlCallback(p, "COMMIT"); - - for(pTab=p->pTblList; pTab; pTab=pNext){ - pNext = pTab->pNext; - sqlite3_free(pTab); - } - p->pTblList = 0; - - sqlite3_finalize(p->pGetPage); - sqlite3_close(p->dbOut); - p->pGetPage = 0; -} - +/* +** Do the configured recovery operation. Return SQLITE_OK if successful, or +** else an SQLite error code. +*/ int sqlite3_recover_run(sqlite3_recover *p){ - if( p && p->errCode==SQLITE_OK ){ - recoverStep(p); + if( p ){ + if( p->bRun ) return SQLITE_MISUSE; /* Has already run */ + if( p->errCode==SQLITE_OK ) recoverRun(p); } return p ? p->errCode : SQLITE_NOMEM; } +/* +** Free all resources associated with the recover handle passed as the only +** argument. The results of using a handle with any sqlite3_recover_** +** API function after it has been passed to this function are undefined. +** +** A copy of the value returned by the first call made to sqlite3_recover_run() +** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has +** not been called on this handle. +*/ int sqlite3_recover_finish(sqlite3_recover *p){ int rc = p->errCode; sqlite3_free(p->zErrMsg); diff --git a/manifest b/manifest index 3f230d9467..d505905e12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sfixes\sto\scomments\sin\ssqlite3recover.h.\sAlso\srework\ssome\sdata\sstructures\sin\ssqlite3recover.c. -D 2022-09-09T20:44:56.135 +C Rework\srecover\sextension\scode\sfor\sreadability. +D 2022-09-10T20:01:49.308 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -392,7 +392,7 @@ F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007acef F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c e8d0eae7da7ba24e733f6247082c12d7a030ce794eb5c42d64993f55c12882f5 +F ext/recover/sqlite3recover.c 4ed53fd33639ede83505f4397b8e1e46b6c0c5a5188c42f28f8372487141170a F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c ed8d0cc8703ab29cf562f793623b045de109b7937f254108ff4132f35abb37fb F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2007,8 +2007,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5f2d5ccd56c06c3468377126acfd4be39b79b05bb6fb09b674b2e185df143aa3 -R d4558eaeefd2a718275b323dc4307f9b +P 599d1f8ec2f9e24924a6f9e66c85664360c7b95531b07a4efe1dd8c096b3fc99 +R cd56b05532a91c08b9e673fffba829f5 U dan -Z 6e64e7331e56e0c44a533586bb618920 +Z 624cfda4b7c658530a41a87671068e55 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index a1b607602a..98f37f0563 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -599d1f8ec2f9e24924a6f9e66c85664360c7b95531b07a4efe1dd8c096b3fc99 \ No newline at end of file +1a2540960e40e3c8c622448fd3862e249bd463c29ae4ce5e39942e942533f60a \ No newline at end of file From aede9b5bd3cfaaa36f81a4a435ef623cdafc2670 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Sep 2022 19:23:50 +0000 Subject: [PATCH 18/42] More tests for the recover module. FossilOrigin-Name: 37fb093b95c6b7d7ad07a275697df73b69f9fb5c5549aea8544b26e38f24833f --- ext/recover/recovercorrupt.test | 68 ++++++++++++++++++++++ ext/recover/recoverfault.test | 72 +++++++++++++++++++++++ ext/recover/sqlite3recover.c | 100 +++++++++++++++++++------------- ext/recover/test_recover.c | 4 +- manifest | 16 ++--- manifest.uuid | 2 +- 6 files changed, 211 insertions(+), 51 deletions(-) create mode 100644 ext/recover/recovercorrupt.test create mode 100644 ext/recover/recoverfault.test diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test new file mode 100644 index 0000000000..ff1d2af19f --- /dev/null +++ b/ext/recover/recovercorrupt.test @@ -0,0 +1,68 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recovercorrupt + +do_execsql_test 1.0 { + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(100)), randomblob(200)); + CREATE INDEX i1 ON t1(b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(100)), randomblob(200)); + ANALYZE; + PRAGMA writable_schema = 1; + DELETE FROM sqlite_schema WHERE name='t2'; +} + +do_test 1.1 { + file size test.db +} {5120} + +proc toggle_bit {blob bit} { + set byte [expr {$bit / 8}] + set bit [expr {$bit & 0x0F}] + binary scan $blob a${byte}ca* A x B + set x [expr {$x ^ (1 << $bit)}] + binary format a*ca* $A $x $B +} + + +db_save_and_close +for {set ii 200} {$ii < 10000} {incr ii} { + db_restore_and_reopen + db func toggle_bit toggle_bit + set pg [expr {($ii / 512)+1}] + set byte [expr {$ii % 512}] + db eval { + UPDATE sqlite_dbpage SET data = toggle_bit(data, $byte) WHERE pgno=$pg + } + + do_test 1.2.$ii { + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R step + $R finish + } {} +} + + +finish_test + diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test new file mode 100644 index 0000000000..b2411fb6c7 --- /dev/null +++ b/ext/recover/recoverfault.test @@ -0,0 +1,72 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recoverfault + + +#-------------------------------------------------------------------------- +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} +#-------------------------------------------------------------------------- + +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(1000)), randomblob(2000)); + CREATE INDEX i1 ON t1(b, c); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(1000)), randomblob(2000)); + ANALYZE; + PRAGMA writable_schema = 1; + DELETE FROM sqlite_schema WHERE name='t2'; +} + +do_faultsim_test 1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R step + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 test.db2 + compare_dbs db db2 + db2 close + } +} + +finish_test + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 9c5e10433b..849c3bc801 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -226,12 +226,13 @@ static int recoverError( int errCode, const char *zFmt, ... ){ + char *z = 0; va_list ap; - char *z; va_start(ap, zFmt); - z = sqlite3_vmprintf(zFmt, ap); - va_end(ap); - + if( zFmt ){ + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + } sqlite3_free(p->zErrMsg); p->zErrMsg = z; p->errCode = errCode; @@ -448,7 +449,8 @@ static i64 recoverPageCount(sqlite3_recover *p){ if( p->errCode==SQLITE_OK ){ sqlite3_stmt *pStmt = 0; pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb); - if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ + if( pStmt ){ + sqlite3_step(pStmt); nPg = sqlite3_column_int64(pStmt, 0); } recoverFinalize(p, pStmt); @@ -866,11 +868,11 @@ static void recoverAddTable( pNew->pNext = p->pTblList; p->pTblList = pNew; + pNew->bIntkey = 1; } recoverFinalize(p, pStmt); - pNew->bIntkey = 1; pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName); while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ int iField = sqlite3_column_int(pStmt, 0); @@ -884,10 +886,12 @@ static void recoverAddTable( } recoverFinalize(p, pStmt); - if( iPk>=0 ){ - pNew->aCol[iPk].bIPK = 1; - }else if( pNew->bIntkey ){ - pNew->iRowidBind = iBind++; + if( p->errCode==SQLITE_OK ){ + if( iPk>=0 ){ + pNew->aCol[iPk].bIPK = 1; + }else if( pNew->bIntkey ){ + pNew->iRowidBind = iBind++; + } } } } @@ -1068,6 +1072,7 @@ static sqlite3_stmt *recoverInsertStmt( RecoverTable *pTab, int nField ){ + sqlite3_stmt *pRet = 0; const char *zSep = ""; const char *zSqlSep = ""; char *zSql = 0; @@ -1075,7 +1080,8 @@ static sqlite3_stmt *recoverInsertStmt( char *zBind = 0; int ii; int bSql = p->xSql ? 1 : 0; - sqlite3_stmt *pRet = 0; + + if( nField<=0 ) return 0; assert( nField<=pTab->nCol ); @@ -1525,7 +1531,7 @@ static int recoverWriteData(sqlite3_recover *p){ int bNewCell = (iPrevPage!=iPage || iPrevCell!=iCell); assert( bNewCell==0 || (iField==-1 || iField==0) ); - assert( bNewCell || iField==nVal ); + assert( bNewCell || iField==nVal || nVal==pTab->nCol ); if( bNewCell ){ if( nVal>=0 ){ @@ -1538,29 +1544,31 @@ static int recoverWriteData(sqlite3_recover *p){ pInsert = recoverInsertStmt(p, pTab, nVal); nInsert = nVal; } + if( nVal>0 ){ + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; - for(ii=0; iinCol; ii++){ - RecoverColumn *pCol = &pTab->aCol[ii]; - - if( pCol->iBind>0 ){ - if( pCol->bIPK ){ - sqlite3_bind_int64(pInsert, pCol->iBind, iRowid); - }else if( pCol->iFieldiBind,apVal[pCol->iField]); + if( pCol->iBind>0 ){ + int iBind = pCol->iBind; + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, iRowid); + }else if( pCol->iFieldiField]); + } } } - } - if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ - sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); - } + if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); + } - if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ - const char *zSql = (const char*)sqlite3_column_text(pInsert, 0); - recoverSqlCallback(p, zSql); + if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ + const char *z = (const char*)sqlite3_column_text(pInsert, 0); + recoverSqlCallback(p, z); + } + recoverReset(p, pInsert); + assert( p->errCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); } - recoverReset(p, pInsert); - assert( p->errCode || pInsert ); - if( pInsert ) sqlite3_clear_bindings(pInsert); } for(ii=0; iinCol ){ assert( apVal[iField]==0 ); apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } nVal = iField+1; } iPrevCell = iCell; @@ -1686,7 +1697,6 @@ sqlite3_recover *recoverInit( int nByte = 0; if( zDb==0 ){ zDb = "main"; } - if( zUri==0 ){ zUri = ""; } nDb = recoverStrlen(zDb); nUri = recoverStrlen(zUri); @@ -1699,7 +1709,7 @@ sqlite3_recover *recoverInit( pRet->zDb = (char*)&pRet[1]; pRet->zUri = &pRet->zDb[nDb+1]; memcpy(pRet->zDb, zDb, nDb); - memcpy(pRet->zUri, zUri, nUri); + if( nUri>0 ) memcpy(pRet->zUri, zUri, nUri); pRet->xSql = xSql; pRet->pSqlCtx = pSqlCtx; pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT; @@ -1730,14 +1740,14 @@ sqlite3_recover *sqlite3_recover_init_sql( int (*xSql)(void*, const char*), void *pSqlCtx ){ - return recoverInit(db, zDb, "", xSql, pSqlCtx); + return recoverInit(db, zDb, 0, xSql, pSqlCtx); } /* ** Return the handle error message, if any. */ const char *sqlite3_recover_errmsg(sqlite3_recover *p){ - return p ? p->zErrMsg : "not an error"; + return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory"; } /* @@ -1753,6 +1763,7 @@ int sqlite3_recover_errcode(sqlite3_recover *p){ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ int rc = SQLITE_OK; + if( p==0 ) return SQLITE_NOMEM; switch( op ){ case 789: sqlite3_free(p->zStateDb); @@ -1791,8 +1802,12 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ */ int sqlite3_recover_run(sqlite3_recover *p){ if( p ){ + recoverExec(p, p->dbIn, "PRAGMA writable_schema=1"); if( p->bRun ) return SQLITE_MISUSE; /* Has already run */ if( p->errCode==SQLITE_OK ) recoverRun(p); + if( sqlite3_exec(p->dbIn, "PRAGMA writable_schema=0", 0, 0, 0) ){ + recoverDbError(p, p->dbIn); + } } return p ? p->errCode : SQLITE_NOMEM; } @@ -1807,11 +1822,16 @@ int sqlite3_recover_run(sqlite3_recover *p){ ** not been called on this handle. */ int sqlite3_recover_finish(sqlite3_recover *p){ - int rc = p->errCode; - sqlite3_free(p->zErrMsg); - sqlite3_free(p->zStateDb); - sqlite3_free(p->zLostAndFound); - sqlite3_free(p); + int rc; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = p->errCode; + sqlite3_free(p->zErrMsg); + sqlite3_free(p->zStateDb); + sqlite3_free(p->zLostAndFound); + sqlite3_free(p); + } return rc; } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index a31ddbb94a..a7c5afd3aa 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -165,9 +165,7 @@ static int testRecoverCmd( int res2; if( res!=SQLITE_OK ){ const char *zErr = sqlite3_recover_errmsg(pTest->p); - char *zRes = sqlite3_mprintf("(%d) - %s", res, zErr); - Tcl_SetObjResult(interp, Tcl_NewStringObj(zRes, -1)); - sqlite3_free(zRes); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErr, -1)); } res2 = sqlite3_recover_finish(pTest->p); assert( res2==res ); diff --git a/manifest b/manifest index d505905e12..c61b4a13ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\srecover\sextension\scode\sfor\sreadability. -D 2022-09-10T20:01:49.308 +C More\stests\sfor\sthe\srecover\smodule. +D 2022-09-12T19:23:50.588 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -390,11 +390,13 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282 F ext/recover/recover1.test 942016356f9098ca36933536b194b5878827a3a749e0bf41a83d83530c0d0ea8 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 +F ext/recover/recovercorrupt.test 115cdb67ac29b4e8ec786cee9190ced674f62388f126b20deea22fa5fd11b814 +F ext/recover/recoverfault.test 30e3d1b423b33b4c57f4a97f9754b2d493a411d858b2672ab369d65199bc2420 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 4ed53fd33639ede83505f4397b8e1e46b6c0c5a5188c42f28f8372487141170a +F ext/recover/sqlite3recover.c d40d7c68a118e01d2b06c96325bbe3b85701f3add2ab5aeec289eeafd5e36d6c F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 -F ext/recover/test_recover.c ed8d0cc8703ab29cf562f793623b045de109b7937f254108ff4132f35abb37fb +F ext/recover/test_recover.c 8f5ef0c9b7523c41a393f65e44d727c23cda8f44d5180fff5b698ee068ba538d F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2007,8 +2009,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 599d1f8ec2f9e24924a6f9e66c85664360c7b95531b07a4efe1dd8c096b3fc99 -R cd56b05532a91c08b9e673fffba829f5 +P 1a2540960e40e3c8c622448fd3862e249bd463c29ae4ce5e39942e942533f60a +R 91d8f87cd1493a401d98e4544c1e3f6e U dan -Z 624cfda4b7c658530a41a87671068e55 +Z d3a21f62871d8e066c6c2bd48e5f7060 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 98f37f0563..340320e0cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a2540960e40e3c8c622448fd3862e249bd463c29ae4ce5e39942e942533f60a \ No newline at end of file +37fb093b95c6b7d7ad07a275697df73b69f9fb5c5549aea8544b26e38f24833f \ No newline at end of file From 5ca0b38605b5e147b5a8b846c1b6fef826ba362b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Sep 2022 20:02:33 +0000 Subject: [PATCH 19/42] Fix some problems with handling corrupt database in the recovery extension. FossilOrigin-Name: ed318be8241981ef96334ba13d3201a717cc812a59aed64e3dc67f7e7e71854b --- ext/recover/recoverfault.test | 27 +++++++++++++++++++++------ ext/recover/sqlite3recover.c | 19 +++++++++++++++++-- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/dbpage.c | 2 +- src/shell.c.in | 2 +- 6 files changed, 50 insertions(+), 20 deletions(-) diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test index b2411fb6c7..2ac1c67f34 100644 --- a/ext/recover/recoverfault.test +++ b/ext/recover/recoverfault.test @@ -44,19 +44,14 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(2, hex(randomblob(1000)), randomblob(2000)); CREATE INDEX i1 ON t1(b, c); - CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); - INSERT INTO t2 VALUES(1, 2, 3); - INSERT INTO t2 VALUES(2, hex(randomblob(1000)), randomblob(2000)); ANALYZE; - PRAGMA writable_schema = 1; - DELETE FROM sqlite_schema WHERE name='t2'; } +faultsim_save_and_close do_faultsim_test 1 -faults oom* -prep { faultsim_restore_and_reopen } -body { set R [sqlite3_recover_init db main test.db2] - $R config lostandfound lost_and_found $R step $R finish } -test { @@ -68,5 +63,25 @@ do_faultsim_test 1 -faults oom* -prep { } } +faultsim_restore_and_reopen +do_execsql_test 2.0 { + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(1000)), hex(randomblob(2000))); + PRAGMA writable_schema = 1; + DELETE FROM sqlite_schema WHERE name='t2'; +} + +do_faultsim_test 2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + set R [sqlite3_recover_init db main test.db2] + $R config lostandfound lost_and_found + $R step + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} +} + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 849c3bc801..c1ff3b5439 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -406,6 +406,18 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +static void recoverBindValue( + sqlite3_recover *p, + sqlite3_stmt *pStmt, + int iBind, + sqlite3_value *pVal +){ + if( p->errCode==SQLITE_OK ){ + int rc = sqlite3_bind_value(pStmt, iBind, pVal); + if( rc ) recoverError(p, rc, 0); + } +} + /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). NULL is returned in this case. @@ -1305,7 +1317,7 @@ static void recoverLostAndFoundPopulate( sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ } for(ii=0; iixSql ){ recoverSqlCallback(p, sqlite3_column_text(pInsert, 0)); @@ -1332,6 +1344,9 @@ static void recoverLostAndFoundPopulate( apVal[iField] = sqlite3_value_dup(pVal); assert( iField==nVal || (nVal==-1 && iField==0) ); nVal = iField+1; + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } } iPrevRoot = iRoot; @@ -1553,7 +1568,7 @@ static int recoverWriteData(sqlite3_recover *p){ if( pCol->bIPK ){ sqlite3_bind_int64(pInsert, iBind, iRowid); }else if( pCol->iFieldiField]); + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); } } } diff --git a/manifest b/manifest index c61b4a13ae..f5b2895b91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stests\sfor\sthe\srecover\smodule. -D 2022-09-12T19:23:50.588 +C Fix\ssome\sproblems\swith\shandling\scorrupt\sdatabase\sin\sthe\srecovery\sextension. +D 2022-09-12T20:02:33.923 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -391,10 +391,10 @@ F ext/recover/recover1.test 942016356f9098ca36933536b194b5878827a3a749e0bf41a83d F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 F ext/recover/recovercorrupt.test 115cdb67ac29b4e8ec786cee9190ced674f62388f126b20deea22fa5fd11b814 -F ext/recover/recoverfault.test 30e3d1b423b33b4c57f4a97f9754b2d493a411d858b2672ab369d65199bc2420 +F ext/recover/recoverfault.test db6b0ba5a993c38ec885867a12556fa7c5d73948b168b0b59e7d3053c0787f29 F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c d40d7c68a118e01d2b06c96325bbe3b85701f3add2ab5aeec289eeafd5e36d6c +F ext/recover/sqlite3recover.c 0154617e73430b594f3d8b94eb2847bd5dc3dc3cd6ff6f0511dc6db5d116621d F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c 8f5ef0c9b7523c41a393f65e44d727c23cda8f44d5180fff5b698ee068ba538d F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -546,7 +546,7 @@ F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 -F src/dbpage.c 5808e91bc27fa3981b028000f8fadfdc10ce9e59a34ce7dc4e035a69be3906ec +F src/dbpage.c dc1e8e7ba880009986b2f2050213157d2328b6d8cd4df6cf0f48883734510558 F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c 24e828db6b2fab8aabfb5d2c0d83dbdfc5a1972b1147fa893350e317ab7e282f @@ -599,7 +599,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c ccce37e7fbe71089cf6aec91e7134c9c0c1d4840cff9f02587bbc71240d914a5 -F src/shell.c.in a805e05f3ed90e0236eb4eb5bdec327bf735d12e754b83ac037cbc22e45ecc09 +F src/shell.c.in 36d5792e35515f2389f175b41486b78b1ad6e571799e3c9b211f47c3629f96e5 F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -2009,8 +2009,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1a2540960e40e3c8c622448fd3862e249bd463c29ae4ce5e39942e942533f60a -R 91d8f87cd1493a401d98e4544c1e3f6e +P 37fb093b95c6b7d7ad07a275697df73b69f9fb5c5549aea8544b26e38f24833f +R 4bca180a2c79c76d0affba5c0d731d37 U dan -Z d3a21f62871d8e066c6c2bd48e5f7060 +Z 8bfc7241f27e2d2e9de40f4a070a3e2c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 340320e0cf..f7f9d304f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37fb093b95c6b7d7ad07a275697df73b69f9fb5c5549aea8544b26e38f24833f \ No newline at end of file +ed318be8241981ef96334ba13d3201a717cc812a59aed64e3dc67f7e7e71854b \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 9b565177c5..7990199fcc 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -288,7 +288,7 @@ static int dbpageColumn( break; } } - return SQLITE_OK; + return rc; } static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ diff --git a/src/shell.c.in b/src/shell.c.in index 632d9633fd..739c2d816c 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7309,7 +7309,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ pState->db, "main", recoverSqlCb, (void*)pState ); - sqlite3_recover_config(p, SQLITE_RECOVER_TESTDB, (void*)zRecoveryDb); + sqlite3_recover_config(p, 789, (void*)zRecoveryDb); sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); From 7985e05a9729fa44a4c0969caa694825d56da1c7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Sep 2022 18:08:24 +0000 Subject: [PATCH 20/42] Add tests. Deal issues surrounding aborting recovery from within the SQL callback, and avoiding the pending-byte page. FossilOrigin-Name: 4e97dd31f5240d9231167ae172a5116426c42177a1ed3c5422b9d51b762d5a87 --- ext/recover/recover1.test | 5 +-- ext/recover/recoverclobber.test | 2 +- ext/recover/recovercorrupt.test | 2 +- ext/recover/recoverfault.test | 4 +-- ext/recover/recoverold.test | 8 +++-- ext/recover/recoverrowid.test | 2 +- ext/recover/recoversql.test | 60 +++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 10 ++++-- ext/recover/test_recover.c | 16 ++++++--- manifest | 29 ++++++++-------- manifest.uuid | 2 +- src/dbpage.c | 16 ++++++--- 12 files changed, 120 insertions(+), 36 deletions(-) create mode 100644 ext/recover/recoversql.test diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 5f1bb2b0fe..3ee4c18ad6 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -45,7 +45,7 @@ proc do_recover_test {tn} { uplevel [list do_test $tn.1 { set R [sqlite3_recover_init db main test.db2] $R config testdb rstate.db - $R step + $R run $R finish } {}] @@ -61,7 +61,7 @@ proc do_recover_test {tn} { set R [sqlite3_recover_init_sql db main my_sql_hook] $R config testdb rstate.db $R config rowids 1 - $R step + $R run $R finish } {}] @@ -75,6 +75,7 @@ proc do_recover_test {tn} { proc my_sql_hook {sql} { lappend ::sqlhook $sql + return 0 } do_execsql_test 1.0 { diff --git a/ext/recover/recoverclobber.test b/ext/recover/recoverclobber.test index e6967ec9be..537af8e7c1 100644 --- a/ext/recover/recoverclobber.test +++ b/ext/recover/recoverclobber.test @@ -25,7 +25,7 @@ ifcapable !vtab { proc recover {db output} { set R [sqlite3_recover_init db main test.db2] - $R step + $R run $R finish } diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test index ff1d2af19f..54a94e1b0c 100644 --- a/ext/recover/recovercorrupt.test +++ b/ext/recover/recovercorrupt.test @@ -58,7 +58,7 @@ for {set ii 200} {$ii < 10000} {incr ii} { do_test 1.2.$ii { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found - $R step + $R run $R finish } {} } diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test index 2ac1c67f34..0241034100 100644 --- a/ext/recover/recoverfault.test +++ b/ext/recover/recoverfault.test @@ -52,7 +52,7 @@ do_faultsim_test 1 -faults oom* -prep { faultsim_restore_and_reopen } -body { set R [sqlite3_recover_init db main test.db2] - $R step + $R run $R finish } -test { faultsim_test_result {0 {}} {1 {}} @@ -77,7 +77,7 @@ do_faultsim_test 2 -faults oom* -prep { } -body { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found - $R step + $R run $R finish } -test { faultsim_test_result {0 {}} {1 {}} diff --git a/ext/recover/recoverold.test b/ext/recover/recoverold.test index 83f210aebc..691737bdc5 100644 --- a/ext/recover/recoverold.test +++ b/ext/recover/recoverold.test @@ -47,7 +47,7 @@ proc do_recover_test {tn {tsql {}} {res {}}} { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found - $R step + $R run $R finish sqlite3 db2 test.db2 @@ -65,7 +65,7 @@ proc do_recover_test {tn {tsql {}} {res {}}} { set ::sqlhook [list] set R [sqlite3_recover_init_sql db main my_sql_hook] $R config lostandfound lost_and_found - $R step + $R run $R finish sqlite3 db2 test.db2 @@ -81,6 +81,7 @@ proc do_recover_test {tn {tsql {}} {res {}}} { proc my_sql_hook {sql} { lappend ::sqlhook $sql + return 0 } @@ -154,6 +155,7 @@ do_recover_test 2.4.1 { 2 2 3 {} 8 9 7 } +breakpoint do_execsql_test 2.5 { CREATE TABLE x1(a, b, c); WITH s(i) AS ( @@ -176,7 +178,7 @@ do_test 2.6 { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found $R config freelistcorrupt 1 - $R step + $R run $R finish sqlite3 db2 test.db2 execsql { SELECT count(*) FROM lost_and_found_1; } db2 diff --git a/ext/recover/recoverrowid.test b/ext/recover/recoverrowid.test index 3dfafbcc7f..bd47422eaf 100644 --- a/ext/recover/recoverrowid.test +++ b/ext/recover/recoverrowid.test @@ -28,7 +28,7 @@ proc recover {db bRowids output} { set R [sqlite3_recover_init db main test.db2] $R config rowids $bRowids - $R step + $R run $R finish } diff --git a/ext/recover/recoversql.test b/ext/recover/recoversql.test new file mode 100644 index 0000000000..19f7ec7a1c --- /dev/null +++ b/ext/recover/recoversql.test @@ -0,0 +1,60 @@ +# 2022 September 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoversql + +ifcapable !vtab { + finish_test; return +} + +do_execsql_test 1.0 { + CREATE TABLE "x.1" (x, y); + INSERT INTO "x.1" VALUES(1, 1), (2, 2), (3, 3); + CREATE INDEX "i.1" ON "x.1"(y, x); +} + +proc sql_hook {sql} { + incr ::iSqlHook + if {$::iSqlHook==$::sql_hook_cnt} { return 4 } + return 0 +} + +do_test 1.1 { + set ::sql_hook_cnt -1 + set ::iSqlHook 0 + set R [sqlite3_recover_init_sql db main sql_hook] + $R run + $R finish +} {} + +set nSqlCall $iSqlHook + +for {set ii 1} {$ii<$nSqlCall} {incr ii} { + set iSqlHook 0 + set sql_hook_cnt $ii + do_test 1.$ii.a { + set R [sqlite3_recover_init_sql db main sql_hook] + $R run + } {1} + do_test 1.$ii.b { + list [catch { $R finish } msg] $msg + } {1 {callback returned an error - 4}} +} + + +finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index c1ff3b5439..206a6fad5b 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -406,6 +406,11 @@ static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){ return p->errCode; } +/* +** Bind the value pVal to parameter iBind of statement pStmt. Leave an +** error in the recover handle passed as the first argument if an error +** (e.g. an OOM) occurs. +*/ static void recoverBindValue( sqlite3_recover *p, sqlite3_stmt *pStmt, @@ -559,7 +564,7 @@ static void recoverGetPage( return; }else{ if( p->pGetPage==0 ){ - pStmt = recoverPreparePrintf( + pStmt = p->pGetPage = recoverPreparePrintf( p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb ); }else{ @@ -567,11 +572,12 @@ static void recoverGetPage( } if( pStmt ){ + int rc = SQLITE_OK; sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); } - p->pGetPage = recoverReset(p, pStmt); + recoverReset(p, pStmt); } } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index a7c5afd3aa..b273fbcd1d 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -42,8 +42,16 @@ static int xSqlCallback(void *pSqlArg, const char *zSql){ if( res ){ Tcl_BackgroundError(p->interp); return TCL_ERROR; + }else{ + Tcl_Obj *pObj = Tcl_GetObjResult(p->interp); + if( Tcl_GetCharLength(pObj)==0 ){ + res = 0; + }else if( Tcl_GetIntFromObj(p->interp, pObj, &res) ){ + Tcl_BackgroundError(p->interp); + return TCL_ERROR; + } } - return SQLITE_OK; + return res; } static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ @@ -60,7 +68,7 @@ static int getDbPointer(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){ ** Implementation of the command created by [sqlite3_recover_init]: ** ** $cmd config OP ARG -** $cmd step +** $cmd run ** $cmd errmsg ** $cmd errcode ** $cmd finalize @@ -77,7 +85,7 @@ static int testRecoverCmd( const char *zMsg; } aSub[] = { { "config", 2, "REBASE-BLOB" }, /* 0 */ - { "step", 0, "" }, /* 1 */ + { "run", 0, "" }, /* 1 */ { "errmsg", 0, "" }, /* 2 */ { "errcode", 0, "" }, /* 3 */ { "finish", 0, "" }, /* 4 */ @@ -145,7 +153,7 @@ static int testRecoverCmd( Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; } - case 1: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + case 1: assert( sqlite3_stricmp("run", aSub[iSub].zSub)==0 ); { int res = sqlite3_recover_run(pTest->p); Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; diff --git a/manifest b/manifest index f5b2895b91..3f6d9ab674 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\shandling\scorrupt\sdatabase\sin\sthe\srecovery\sextension. -D 2022-09-12T20:02:33.923 +C Add\stests.\sDeal\sissues\ssurrounding\saborting\srecovery\sfrom\swithin\sthe\sSQL\scallback,\sand\savoiding\sthe\spending-byte\spage. +D 2022-09-13T18:08:24.770 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,16 +387,17 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 942016356f9098ca36933536b194b5878827a3a749e0bf41a83d83530c0d0ea8 +F ext/recover/recover1.test d0fa2f945aac4754e0abb802941b9b80efd4828a775a090f9d2253f5a0ee1d0e F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c -F ext/recover/recoverclobber.test e6537ebf99f57bfff6cca59550b5f4278319b57a89865abb98d755a8fd561d84 -F ext/recover/recovercorrupt.test 115cdb67ac29b4e8ec786cee9190ced674f62388f126b20deea22fa5fd11b814 -F ext/recover/recoverfault.test db6b0ba5a993c38ec885867a12556fa7c5d73948b168b0b59e7d3053c0787f29 -F ext/recover/recoverold.test f368a6ae2db12b6017257b332a19ab5df527f4061e43f12f5c85d8e2b236f074 -F ext/recover/recoverrowid.test ec4436cd69e6cdacb48dd2963ff6dd9dbd5fe648376de5e7c0c2f4f6cbacb417 -F ext/recover/sqlite3recover.c 0154617e73430b594f3d8b94eb2847bd5dc3dc3cd6ff6f0511dc6db5d116621d +F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 +F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 +F ext/recover/recoverfault.test 862ab02bd503922281a5ee9f81a3a92312561bb7a39e2fdb06e6afd60c3a167e +F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 +F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 +F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f +F ext/recover/sqlite3recover.c ed5ab827433823ebf3ad7ada192ecefdc8ef9327af68fd4db9ae463a11bbc476 F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 -F ext/recover/test_recover.c 8f5ef0c9b7523c41a393f65e44d727c23cda8f44d5180fff5b698ee068ba538d +F ext/recover/test_recover.c 5941ecf484b6158be26e34c6f7b6c7f03967c72a63db0c08e7fd0fa43023eafa F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -546,7 +547,7 @@ F src/callback.c 4cd7225b26a97f7de5fee5ae10464bed5a78f2adefe19534cc2095b3a8ca484 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 93e4b5f4faf6d3f688988a116773259a4fbfb4ddac0e9bf9d0ae0429390c2543 F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5 -F src/dbpage.c dc1e8e7ba880009986b2f2050213157d2328b6d8cd4df6cf0f48883734510558 +F src/dbpage.c 433f13b34f1e5a84cbb5f10214c12471c6f0197f276389634a1cc393ec821c3d F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e F src/expr.c 24e828db6b2fab8aabfb5d2c0d83dbdfc5a1972b1147fa893350e317ab7e282f @@ -2009,8 +2010,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 37fb093b95c6b7d7ad07a275697df73b69f9fb5c5549aea8544b26e38f24833f -R 4bca180a2c79c76d0affba5c0d731d37 +P ed318be8241981ef96334ba13d3201a717cc812a59aed64e3dc67f7e7e71854b +R 11a8fecaa81e021bbb20605615a33134 U dan -Z 8bfc7241f27e2d2e9de40f4a070a3e2c +Z 28892e8b77cb290f00759179724671d9 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f7f9d304f7..9694fdd0c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed318be8241981ef96334ba13d3201a717cc812a59aed64e3dc67f7e7e71854b \ No newline at end of file +4e97dd31f5240d9231167ae172a5116426c42177a1ed3c5422b9d51b762d5a87 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index 7990199fcc..e13740d81d 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -274,12 +274,18 @@ static int dbpageColumn( } case 1: { /* data */ DbPage *pDbPage = 0; - rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); - if( rc==SQLITE_OK ){ - sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, - SQLITE_TRANSIENT); + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); } - sqlite3PagerUnref(pDbPage); break; } default: { /* schema */ From a3a6d63b4087895764ce15a8cf2e6afde76b6f9a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Sep 2022 19:47:01 +0000 Subject: [PATCH 21/42] Fix a broken error handling case in the recovery extension. FossilOrigin-Name: 60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 --- ext/recover/recoverfault.test | 1 + ext/recover/sqlite3recover.c | 11 ++++++++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test index 0241034100..e6d8392a44 100644 --- a/ext/recover/recoverfault.test +++ b/ext/recover/recoverfault.test @@ -49,6 +49,7 @@ do_execsql_test 1.0 { faultsim_save_and_close do_faultsim_test 1 -faults oom* -prep { + catch { db2 close } faultsim_restore_and_reopen } -body { set R [sqlite3_recover_init db main test.db2] diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 206a6fad5b..c67d9b3d49 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -495,9 +495,9 @@ static void recoverReadI32( assert( argc==2 ); nBlob = sqlite3_value_bytes(argv[0]); pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); - iInt = sqlite3_value_int(argv[1]); + iInt = sqlite3_value_int(argv[1]) & 0xFFFF; - if( iInt>=0 && (iInt+1)*4<=nBlob ){ + if( (iInt+1)*4<=nBlob ){ const unsigned char *a = &pBlob[iInt*4]; i64 iVal = ((i64)a[0]<<24) + ((i64)a[1]<<16) @@ -770,7 +770,11 @@ static int recoverOpenOutput(sqlite3_recover *p){ if( pBackup ){ while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); } + }else{ + recoverDbError(p, db2); } sqlite3_close(db2); } @@ -1791,7 +1795,7 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); break; - case SQLITE_RECOVER_LOST_AND_FOUND: + case SQLITE_RECOVER_LOST_AND_FOUND: { const char *zArg = (const char*)pArg; sqlite3_free(p->zLostAndFound); if( zArg ){ @@ -1800,6 +1804,7 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->zLostAndFound = 0; } break; + } case SQLITE_RECOVER_FREELIST_CORRUPT: p->bFreelistCorrupt = *(int*)pArg; diff --git a/manifest b/manifest index 3f6d9ab674..954e80712a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests.\sDeal\sissues\ssurrounding\saborting\srecovery\sfrom\swithin\sthe\sSQL\scallback,\sand\savoiding\sthe\spending-byte\spage. -D 2022-09-13T18:08:24.770 +C Fix\sa\sbroken\serror\shandling\scase\sin\sthe\srecovery\sextension. +D 2022-09-13T19:47:01.338 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -391,11 +391,11 @@ F ext/recover/recover1.test d0fa2f945aac4754e0abb802941b9b80efd4828a775a090f9d22 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 -F ext/recover/recoverfault.test 862ab02bd503922281a5ee9f81a3a92312561bb7a39e2fdb06e6afd60c3a167e +F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c ed5ab827433823ebf3ad7ada192ecefdc8ef9327af68fd4db9ae463a11bbc476 +F ext/recover/sqlite3recover.c 1afcac2bbfcf5ef67a3391f59678dc866348ac88745067e683052ede02c425fb F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c 5941ecf484b6158be26e34c6f7b6c7f03967c72a63db0c08e7fd0fa43023eafa F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2010,8 +2010,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ed318be8241981ef96334ba13d3201a717cc812a59aed64e3dc67f7e7e71854b -R 11a8fecaa81e021bbb20605615a33134 +P 4e97dd31f5240d9231167ae172a5116426c42177a1ed3c5422b9d51b762d5a87 +R bab9a5e0534df0c970f1689fb0655c4b U dan -Z 28892e8b77cb290f00759179724671d9 +Z ca04715ddf53f08e8c8270238268a615 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9694fdd0c2..d9dc13270c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e97dd31f5240d9231167ae172a5116426c42177a1ed3c5422b9d51b762d5a87 \ No newline at end of file +60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 \ No newline at end of file From 322967df59cf549e6836f45cf38bc2dcb81f35b1 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Sep 2022 20:40:57 +0000 Subject: [PATCH 22/42] Add OOM tests for the recovery extension. FossilOrigin-Name: 9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 --- ext/recover/recover1.test | 26 ++++++++ ext/recover/recoverfault2.test | 108 +++++++++++++++++++++++++++++++++ manifest | 13 ++-- manifest.uuid | 2 +- 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 ext/recover/recoverfault2.test diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 3ee4c18ad6..e7a45e8e90 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -167,6 +167,32 @@ ifcapable fts5 { do_recover_test 9 } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 10.1 { + CREATE TABLE x1(a PRIMARY KEY, str TEXT) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, ' + \nhello\012world(\n0)(\n1) + '); + INSERT INTO x1 VALUES(2, ' + \nhello + '); +} +do_execsql_test 10.2 " + INSERT INTO x1 VALUES(3, '\012hello there\015world'); + INSERT INTO x1 VALUES(4, '\015hello there\015world'); +" +do_recover_test 10 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 11.1 { + PRAGMA encoding='utf16'; + CREATE TABLE u1(u, v); + INSERT INTO u1 VALUES('edvin marton', 'bond'); + INSERT INTO u1 VALUES(1, 4.0); +} +do_recover_test 11 diff --git a/ext/recover/recoverfault2.test b/ext/recover/recoverfault2.test new file mode 100644 index 0000000000..7489206050 --- /dev/null +++ b/ext/recover/recoverfault2.test @@ -0,0 +1,108 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recoverfault2 + + +#-------------------------------------------------------------------------- +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} +#-------------------------------------------------------------------------- + +do_execsql_test 1.0 " + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(2, '\012hello\015world\012today\n'); +" +faultsim_save_and_close + +proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 +} + +do_faultsim_test 1 -faults oom* -prep { + catch { db2 close } + faultsim_restore_and_reopen + set ::lSql [list] +} -body { + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R run + $R finish +} -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 "" + db2 eval [join $::lSql ";"] + compare_dbs db db2 + db2 close + } +} + +ifcapable utf16 { + reset_db + do_execsql_test 2.0 " + PRAGMA encoding='utf-16'; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(2, '\012hello\015world\012today\n'); + " + faultsim_save_and_close + + proc my_sql_hook {sql} { + puts "HOOK $sql" + lappend ::lSql $sql + return 0 + } + + do_faultsim_test 2 -faults oom-t* -prep { + catch { db2 close } + faultsim_restore_and_reopen + set ::lSql [list] + } -body { + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R run + $R finish + } -test { + faultsim_test_result {0 {}} {1 {}} + if {$testrc==0} { + sqlite3 db2 "" + db2 eval [join $::lSql ";"] + compare_dbs db db2 + db2 close + } + } +} + + + +finish_test + diff --git a/manifest b/manifest index 954e80712a..18c614dd00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\serror\shandling\scase\sin\sthe\srecovery\sextension. -D 2022-09-13T19:47:01.338 +C Add\sOOM\stests\sfor\sthe\srecovery\sextension. +D 2022-09-13T20:40:57.096 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,11 +387,12 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test d0fa2f945aac4754e0abb802941b9b80efd4828a775a090f9d2253f5a0ee1d0e +F ext/recover/recover1.test 623afa77b91996bb1319b069ced5245c243caa995f66793d6879fbc2c190f0be F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e +F ext/recover/recoverfault2.test 699b3ec07ba6982291e65e1811807f18d7f115234b407c819eaf3529878867f5 F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f @@ -2010,8 +2011,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4e97dd31f5240d9231167ae172a5116426c42177a1ed3c5422b9d51b762d5a87 -R bab9a5e0534df0c970f1689fb0655c4b +P 60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 +R 4e62121f5a47cd1728b6e036c3f3c8c0 U dan -Z ca04715ddf53f08e8c8270238268a615 +Z 5fc633ab904798afb52a452a7e949f53 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d9dc13270c..2c3276310d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 \ No newline at end of file +9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 \ No newline at end of file From b8eaf9a10dbbcb217ffa5c85f2f2c99f9c584dc6 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Sep 2022 16:37:59 +0000 Subject: [PATCH 23/42] Fix recovery of utf-16 databases. FossilOrigin-Name: 5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 --- ext/misc/dbdata.c | 61 ++++++++++++++++++++++++++++------ ext/recover/recover1.test | 1 + ext/recover/recoverfault2.test | 1 - ext/recover/test_recover.c | 35 +++++++++++++++++++ manifest | 20 +++++------ manifest.uuid | 2 +- test/tester.tcl | 41 +++++++++++++++++++++++ 7 files changed, 138 insertions(+), 23 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index a18304b96b..29a6497bef 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -75,6 +75,7 @@ #include "sqlite3ext.h" typedef unsigned char u8; +typedef unsigned int u32; #endif SQLITE_EXTENSION_INIT1 @@ -107,6 +108,7 @@ struct DbdataCursor { int iField; /* Current field number */ u8 *pHdrPtr; u8 *pPtr; + u32 enc; /* Text encoding */ sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -299,14 +301,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ /* ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. */ -static unsigned int get_uint16(unsigned char *a){ +static u32 get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } -static unsigned int get_uint32(unsigned char *a){ - return ((unsigned int)a[0]<<24) - | ((unsigned int)a[1]<<16) - | ((unsigned int)a[2]<<8) - | ((unsigned int)a[3]); +static u32 get_uint32(unsigned char *a){ + return ((u32)a[0]<<24) + | ((u32)a[1]<<16) + | ((u32)a[2]<<8) + | ((u32)a[3]); } /* @@ -321,7 +323,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - unsigned int pgno, /* Page number of page to load */ + u32 pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -405,6 +407,7 @@ static int dbdataValueBytes(int eType){ */ static void dbdataValue( sqlite3_context *pCtx, + u32 enc, int eType, u8 *pData, int nData @@ -449,7 +452,17 @@ static void dbdataValue( default: { int n = ((eType-12) / 2); if( eType % 2 ){ - sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + switch( enc ){ + case SQLITE_UTF16BE: + sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + case SQLITE_UTF16LE: + sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT); + break; + default: + sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT); + break; + } }else{ sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); } @@ -588,7 +601,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; @@ -703,6 +716,25 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } +/* +** Attempt to figure out the encoding of the database by retrieving page 1 +** and inspecting the header field. If successful, set the pCsr->enc variable +** and return SQLITE_OK. Otherwise, return an SQLite error code. +*/ +static int dbdataGetEncoding(DbdataCursor *pCsr){ + int rc = SQLITE_OK; + int nPg1 = 0; + u8 *aPg1 = 0; + rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1); + assert( rc!=SQLITE_OK || nPg1==0 || nPg1>=512 ); + if( rc==SQLITE_OK && nPg1>0 ){ + pCsr->enc = get_uint32(&aPg1[56]); + } + sqlite3_free(aPg1); + return rc; +} + + /* ** xFilter method for sqlite_dbdata and sqlite_dbptr. */ @@ -725,7 +757,6 @@ static int dbdataFilter( pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; }else{ - pCsr->nPage = dbdataDbsize(pCsr, zSchema); rc = dbdataDbsize(pCsr, zSchema); } @@ -754,6 +785,13 @@ static int dbdataFilter( }else{ pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } + + /* Try to determine the encoding of the db by inspecting the header + ** field on page 1. */ + if( rc==SQLITE_OK ){ + rc = dbdataGetEncoding(pCsr); + } + if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); } @@ -808,7 +846,8 @@ static int dbdataColumn( sqlite3_int64 iType; dbdataGetVarint(pCsr->pHdrPtr, &iType); dbdataValue( - ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr + ctx, pCsr->enc, iType, pCsr->pPtr, + &pCsr->pRec[pCsr->nRec] - pCsr->pPtr ); } break; diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index e7a45e8e90..358015a591 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -192,6 +192,7 @@ do_execsql_test 11.1 { INSERT INTO u1 VALUES('edvin marton', 'bond'); INSERT INTO u1 VALUES(1, 4.0); } + do_recover_test 11 diff --git a/ext/recover/recoverfault2.test b/ext/recover/recoverfault2.test index 7489206050..4f7131ecfb 100644 --- a/ext/recover/recoverfault2.test +++ b/ext/recover/recoverfault2.test @@ -78,7 +78,6 @@ ifcapable utf16 { faultsim_save_and_close proc my_sql_hook {sql} { - puts "HOOK $sql" lappend ::lSql $sql return 0 } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index b273fbcd1d..7ab440fa37 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -229,6 +229,40 @@ static int test_sqlite3_recover_init( return TCL_OK; } +/* +** Declaration for public API function in file dbdata.c. This may be called +** with NULL as the final two arguments to register the sqlite_dbptr and +** sqlite_dbdata virtual tables with a database handle. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); + +/* +** sqlite3_recover_init DB DBNAME URI +*/ +static int test_sqlite3_dbdata_init( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db = 0; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; + sqlite3_dbdata_init(db, 0, 0); + + Tcl_ResetResult(interp); + return TCL_OK; +} + + + int TestRecover_Init(Tcl_Interp *interp){ struct Cmd { const char *zCmd; @@ -237,6 +271,7 @@ int TestRecover_Init(Tcl_Interp *interp){ } aCmd[] = { { "sqlite3_recover_init", test_sqlite3_recover_init, 0 }, { "sqlite3_recover_init_sql", test_sqlite3_recover_init, (void*)1 }, + { "sqlite3_dbdata_init", test_sqlite3_dbdata_init, (void*)1 }, }; int i; diff --git a/manifest b/manifest index 18c614dd00..b17390fa0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sOOM\stests\sfor\sthe\srecovery\sextension. -D 2022-09-13T20:40:57.096 +C Fix\srecovery\sof\sutf-16\sdatabases. +D 2022-09-14T16:37:59.285 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -299,7 +299,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c 9bb3666519bd8a54cce4934076a557fe6441c5bafce7e9c24d8b5ced148e8154 +F ext/misc/dbdata.c ca7b235fa2396e8fc2e950826872f820f31268ac2cb51368b0d655bb71568f07 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -387,18 +387,18 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 623afa77b91996bb1319b069ced5245c243caa995f66793d6879fbc2c190f0be +F ext/recover/recover1.test 167ad4b267f6db5c06963a72196527cb42369c7cd77de2b4273d9fdc8bd7a254 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e -F ext/recover/recoverfault2.test 699b3ec07ba6982291e65e1811807f18d7f115234b407c819eaf3529878867f5 +F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f F ext/recover/sqlite3recover.c 1afcac2bbfcf5ef67a3391f59678dc866348ac88745067e683052ede02c425fb F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 -F ext/recover/test_recover.c 5941ecf484b6158be26e34c6f7b6c7f03967c72a63db0c08e7fd0fa43023eafa +F ext/recover/test_recover.c 6a6f86ea61d728c67382047d574c62df83e6a28db23c329e93a177093689cd20 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -1536,7 +1536,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc -F test/tester.tcl d759ac44a501fb832f2ea966429ca18acfba0f9a8d34ad5c499332b079b37023 +F test/tester.tcl 65c29b6f1dbf71b0e59a7b221d7e849dfa5a55fa7d0a2902811e8abdecdb1d44 F test/testrunner.tcl 86b57135754ab2160aeb04b4829d321fb285a5cfa7a505fe61d69aed605854cc F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899 F test/thread002.test c24c83408e35ba5a952a3638b7ac03ccdf1ce4409289c54a050ac4c5f1de7502 @@ -2011,8 +2011,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 60089547e1fc77ecc02f207ebf75ee3160e5ff25f41d12e02e170fd7fde66602 -R 4e62121f5a47cd1728b6e036c3f3c8c0 +P 9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 +R 7d7577824671621531c65dae0de3846f U dan -Z 5fc633ab904798afb52a452a7e949f53 +Z b3ef1682460d7df284b8cd64863e6715 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2c3276310d..05a5c951ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 \ No newline at end of file +5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 5612311dab..9353b91244 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1548,6 +1548,47 @@ proc explain_i {sql {db db}} { output2 "---- ------------ ------ ------ ------ ---------------- -- -" } +proc execsql_pp {sql {db db}} { + set nCol 0 + $db eval $sql A { + if {$nCol==0} { + set nCol [llength $A(*)] + foreach c $A(*) { + set aWidth($c) [string length $c] + lappend data $c + } + } + foreach c $A(*) { + set n [string length $A($c)] + if {$n > $aWidth($c)} { + set aWidth($c) $n + } + lappend data $A($c) + } + } + if {$nCol>0} { + set nTotal 0 + foreach e [array names aWidth] { incr nTotal $aWidth($e) } + incr nTotal [expr ($nCol-1) * 3] + incr nTotal 4 + + set fmt "" + foreach c $A(*) { + lappend fmt "% -$aWidth($c)s" + } + set fmt "| [join $fmt { | }] |" + + puts [string repeat - $nTotal] + for {set i 0} {$i < [llength $data]} {incr i $nCol} { + set vals [lrange $data $i [expr $i+$nCol-1]] + puts [format $fmt {*}$vals] + if {$i==0} { puts [string repeat - $nTotal] } + } + puts [string repeat - $nTotal] + } +} + + # Show the VDBE program for an SQL statement but omit the Trace # opcode at the beginning. This procedure can be used to prove # that different SQL statements generate exactly the same VDBE code. From 3c24342841614c8f31d0ae27cad111a950959c3e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Sep 2022 18:57:46 +0000 Subject: [PATCH 24/42] Extra tests and fixes. FossilOrigin-Name: deed5336931b23bb507c064d08e9899b33f04e4f7eee03beb8d7147eb4caa030 --- ext/recover/recover1.test | 12 +++++ ext/recover/recovercorrupt2.test | 76 ++++++++++++++++++++++++++++++++ ext/recover/recoverfault.test | 1 + ext/recover/sqlite3recover.c | 16 +++---- ext/recover/test_recover.c | 14 ++++-- manifest | 19 ++++---- manifest.uuid | 2 +- 7 files changed, 118 insertions(+), 22 deletions(-) create mode 100644 ext/recover/recovercorrupt2.test diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 358015a591..79258b7e8f 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -195,7 +195,19 @@ do_execsql_test 11.1 { do_recover_test 11 +do_test 12.1 { + set R [sqlite3_recover_init db "" test.db2] + $R config lostandfound "" + $R config invalid xyz +} {12} +do_test 12.2 { + $R run + $R run +} {21} +do_test 12.3 { + $R finish +} {} finish_test diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test new file mode 100644 index 0000000000..58dee54815 --- /dev/null +++ b/ext/recover/recovercorrupt2.test @@ -0,0 +1,76 @@ +# 2022 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +set testprefix recovercorrupt2 + +do_execsql_test 1.0 { + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, hex(randomblob(100)), randomblob(200)); + CREATE INDEX i1 ON t1(b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, hex(randomblob(100)), randomblob(200)); + ANALYZE; + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE INDEX i1 ON o(world)' WHERE name='i1'; +} + +do_test 1.1 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +sqlite3 db2 test.db2 +do_execsql_test -db db2 1.2 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} + {CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID} + {CREATE TABLE sqlite_stat1(tbl,idx,stat)} + {CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample)} +} +db2 close + +do_execsql_test 1.3 { + PRAGMA writable_schema = 1; + UPDATE sqlite_schema SET sql = 'CREATE TABLE t2 syntax error!' WHERE name='t2'; +} + +do_test 1.4 { + set R [sqlite3_recover_init db main test.db2] + $R run + $R finish +} {} + +sqlite3 db2 test.db2 +do_execsql_test -db db2 1.5 { + SELECT sql FROM sqlite_schema +} { + {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} + {CREATE TABLE sqlite_stat1(tbl,idx,stat)} + {CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample)} +} +db2 close + + + +finish_test + diff --git a/ext/recover/recoverfault.test b/ext/recover/recoverfault.test index e6d8392a44..2ea87860b3 100644 --- a/ext/recover/recoverfault.test +++ b/ext/recover/recoverfault.test @@ -72,6 +72,7 @@ do_execsql_test 2.0 { PRAGMA writable_schema = 1; DELETE FROM sqlite_schema WHERE name='t2'; } +faultsim_save_and_close do_faultsim_test 2 -faults oom* -prep { faultsim_restore_and_reopen diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index c67d9b3d49..6c52d99855 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -768,7 +768,7 @@ static int recoverOpenOutput(sqlite3_recover *p){ if( rc==SQLITE_OK ){ sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); if( pBackup ){ - while( sqlite3_backup_step(pBackup, 1000)==SQLITE_OK ); + sqlite3_backup_step(pBackup, -1); p->errCode = sqlite3_backup_finish(pBackup); }else{ recoverDbError(p, db); @@ -1042,10 +1042,10 @@ static int recoverWriteSchema2(sqlite3_recover *p){ i64 iRoot = sqlite3_column_int64(pSelect, 0); const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); - if( rc!=SQLITE_OK && rc!=SQLITE_ERROR ){ - recoverDbError(p, p->dbOut); - }else if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ recoverSqlCallback(p, zSql); + }else if( rc!=SQLITE_ERROR ){ + recoverDbError(p, p->dbOut); } } } @@ -1316,9 +1316,7 @@ static void recoverLostAndFoundPopulate( int iCell = sqlite3_column_int64(pStmt, 2); int iField = sqlite3_column_int64(pStmt, 3); - if( iPrevRoot>0 && ( - iPrevRoot!=iRoot || iPrevPage!=iPage || iPrevCell!=iCell - )){ + if( iPrevRoot>0 && (iPrevPage!=iPage || iPrevCell!=iCell) ){ /* Insert the new row */ sqlite3_bind_int64(pInsert, 1, iPrevRoot); /* rootpgno */ sqlite3_bind_int64(pInsert, 2, iPrevPage); /* pgno */ @@ -1329,7 +1327,7 @@ static void recoverLostAndFoundPopulate( for(ii=0; iixSql ){ + if( sqlite3_step(pInsert)==SQLITE_ROW ){ recoverSqlCallback(p, sqlite3_column_text(pInsert, 0)); } recoverReset(p, pInsert); @@ -1586,7 +1584,7 @@ static int recoverWriteData(sqlite3_recover *p){ sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); } - if( SQLITE_ROW==sqlite3_step(pInsert) && p->xSql ){ + if( SQLITE_ROW==sqlite3_step(pInsert) ){ const char *z = (const char*)sqlite3_column_text(pInsert, 0); recoverSqlCallback(p, z); } diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index 7ab440fa37..d660a5345e 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -84,7 +84,7 @@ static int testRecoverCmd( int nArg; const char *zMsg; } aSub[] = { - { "config", 2, "REBASE-BLOB" }, /* 0 */ + { "config", 2, "ARG" }, /* 0 */ { "run", 0, "" }, /* 1 */ { "errmsg", 0, "" }, /* 2 */ { "errcode", 0, "" }, /* 3 */ @@ -115,6 +115,7 @@ static int testRecoverCmd( "lostandfound", /* 1 */ "freelistcorrupt", /* 2 */ "rowids", /* 3 */ + "invalid", /* 4 */ 0 }; int iOp = 0; @@ -128,11 +129,13 @@ static int testRecoverCmd( 789, (void*)Tcl_GetString(objv[3]) /* MAGIC NUMBER! */ ); break; - case 1: + case 1: { + const char *zStr = Tcl_GetString(objv[3]); res = sqlite3_recover_config(pTest->p, - SQLITE_RECOVER_LOST_AND_FOUND, (void*)Tcl_GetString(objv[3]) + SQLITE_RECOVER_LOST_AND_FOUND, (void*)(zStr[0] ? zStr : 0) ); break; + } case 2: { int iVal = 0; if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; @@ -149,6 +152,10 @@ static int testRecoverCmd( ); break; } + case 4: { + res = sqlite3_recover_config(pTest->p, 12345, 0); + break; + } } Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); break; @@ -210,6 +217,7 @@ static int test_sqlite3_recover_init( } if( getDbPointer(interp, objv[1], &db) ) return TCL_ERROR; zDb = Tcl_GetString(objv[2]); + if( zDb[0]=='\0' ) zDb = 0; pNew = ckalloc(sizeof(TestRecover)); if( bSql==0 ){ diff --git a/manifest b/manifest index b17390fa0e..cc05de1c51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\srecovery\sof\sutf-16\sdatabases. -D 2022-09-14T16:37:59.285 +C Extra\stests\sand\sfixes. +D 2022-09-14T18:57:46.380 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,18 +387,19 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 167ad4b267f6db5c06963a72196527cb42369c7cd77de2b4273d9fdc8bd7a254 +F ext/recover/recover1.test 3f26db7692ed98c0de4fc9511f3d3a2f2c91a137b7e6666f3b85b593438b5d09 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 -F ext/recover/recoverfault.test f3587c218c448545a082b99d59294dff5ec0b7daa15b0556cf926f6c350f221e +F ext/recover/recovercorrupt2.test 7a1e3fe43231e7571aff4a09f359dcc0c98ef8f74d507c17953d35f2ad9f4c1f +F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 1afcac2bbfcf5ef67a3391f59678dc866348ac88745067e683052ede02c425fb +F ext/recover/sqlite3recover.c 29cb0ef3ff22e4a813d741930be123b7a44d59ba4c93868e74c4b0ebbee90a62 F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 -F ext/recover/test_recover.c 6a6f86ea61d728c67382047d574c62df83e6a28db23c329e93a177093689cd20 +F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2011,8 +2012,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9b6b4c7162439a889144edb561356afc66436db921a867c20871f0c556716502 -R 7d7577824671621531c65dae0de3846f +P 5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 +R 7d672439d169025c8f5a2255e79dee5d U dan -Z b3ef1682460d7df284b8cd64863e6715 +Z 7e26bca497bd6fe58832c34f4ec41a30 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 05a5c951ed..9efa9051fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 \ No newline at end of file +deed5336931b23bb507c064d08e9899b33f04e4f7eee03beb8d7147eb4caa030 \ No newline at end of file From 6fa0a11c7b9e8b8c86454e266e158b79c22f471b Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Sep 2022 20:45:46 +0000 Subject: [PATCH 25/42] Include recovery extension in configure/make builds of testfixture. FossilOrigin-Name: d2f4652144c4a54012a2176c178c5ade3463bd017515f472cfa8dafdde192df3 --- Makefile.in | 6 ++++++ ext/recover/recovercorrupt2.test | 3 +-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index cc4943f2c4..ca3dfdcae8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -433,6 +433,9 @@ TESTSRC = \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/test_recover.c \ $(TOP)/ext/rbu/test_rbu.c # Statically linked extensions @@ -1102,6 +1105,9 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/ext/recover/recovercorrupt2.test b/ext/recover/recovercorrupt2.test index 58dee54815..cb5fa05baa 100644 --- a/ext/recover/recovercorrupt2.test +++ b/ext/recover/recovercorrupt2.test @@ -30,6 +30,7 @@ do_execsql_test 1.0 { ANALYZE; PRAGMA writable_schema = 1; UPDATE sqlite_schema SET sql = 'CREATE INDEX i1 ON o(world)' WHERE name='i1'; + DELETE FROM sqlite_schema WHERE name='sqlite_stat4'; } do_test 1.1 { @@ -45,7 +46,6 @@ do_execsql_test -db db2 1.2 { {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} {CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID} {CREATE TABLE sqlite_stat1(tbl,idx,stat)} - {CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample)} } db2 close @@ -66,7 +66,6 @@ do_execsql_test -db db2 1.5 { } { {CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c)} {CREATE TABLE sqlite_stat1(tbl,idx,stat)} - {CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample)} } db2 close diff --git a/manifest b/manifest index cc05de1c51..77ef3c20e0 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Extra\stests\sand\sfixes. -D 2022-09-14T18:57:46.380 +C Include\srecovery\sextension\sin\sconfigure/make\sbuilds\sof\stestfixture. +D 2022-09-14T20:45:46.778 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 525bccb89e36a927f9312a231f054a2dc029f6af75901c7fc1a781d51b260323 +F Makefile.in 1dd98f0349f14a7b106ae05904d4d040b4d20f2ab8e2d3c34f8231ba08948e42 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 F Makefile.msc d547a2fdba38a1c6cd1954977d0b0cc017f5f8fbfbc65287bf8d335808938016 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e @@ -391,7 +391,7 @@ F ext/recover/recover1.test 3f26db7692ed98c0de4fc9511f3d3a2f2c91a137b7e6666f3b85 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 -F ext/recover/recovercorrupt2.test 7a1e3fe43231e7571aff4a09f359dcc0c98ef8f74d507c17953d35f2ad9f4c1f +F ext/recover/recovercorrupt2.test a131d8005337c092e2dfa3b84909ed67ae82d22399a8cfb8c984b2939969ca42 F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 @@ -2012,8 +2012,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5b05be0861f35804270fbd184ad4b89c23e98cc2fbd56b9e4fe6197daef5fe49 -R 7d672439d169025c8f5a2255e79dee5d +P deed5336931b23bb507c064d08e9899b33f04e4f7eee03beb8d7147eb4caa030 +R 280ffbfae11c88d50514166a9090fc16 U dan -Z 7e26bca497bd6fe58832c34f4ec41a30 +Z 303ba5cc3bab2dbc239e16c7c361938c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 9efa9051fd..bf8054daff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -deed5336931b23bb507c064d08e9899b33f04e4f7eee03beb8d7147eb4caa030 \ No newline at end of file +d2f4652144c4a54012a2176c178c5ade3463bd017515f472cfa8dafdde192df3 \ No newline at end of file From 73390f09c2e93c61e4da507d2b61e4cf03a98b59 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Sep 2022 03:54:52 +0000 Subject: [PATCH 26/42] Fix windows builds of recover extension. FossilOrigin-Name: abcbb6abfe08fc590123f0aa1bd72cfdecc75908f078f1348dc0e957de98bf52 --- Makefile.msc | 6 ++++++ ext/recover/sqlite3recover.c | 18 +++++++----------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 9baa4ae937..4abf2d8e82 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1587,6 +1587,9 @@ TESTEXT = \ $(TOP)\ext\misc\unionvtab.c \ $(TOP)\ext\misc\wholenumber.c \ $(TOP)\ext\rtree\test_rtreedoc.c \ + $(TOP)\ext\recover\sqlite3recover.c \ + $(TOP)\ext\recover\test_recover.c \ + $(TOP)\ext\misc\dbdata.c \ fts5.c # If use of zlib is enabled, add the "zipfile.c" source file. @@ -2224,6 +2227,9 @@ SHELL_SRC = \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\misc\memtrace.c \ + $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/sqlite3recover.c \ + $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)\src\test_windirent.c # If use of zlib is enabled, add the "zipfile.c" source file. diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 6c52d99855..a370ca0c46 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -137,7 +137,7 @@ struct RecoverColumn { typedef struct RecoverBitmap RecoverBitmap; struct RecoverBitmap { i64 nPg; /* Size of bitmap */ - u32 aElem[0]; /* Array of 32-bit bitmasks */ + u32 aElem[1]; /* Array of 32-bit bitmasks */ }; /* @@ -999,8 +999,8 @@ static int recoverWriteSchema1(sqlite3_recover *p){ recoverSqlCallback(p, zSql); if( bTable && !bVirtual ){ if( SQLITE_ROW==sqlite3_step(pTblname) ){ - const char *zName = sqlite3_column_text(pTblname, 0); - recoverAddTable(p, zName, iRoot); + const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0); + recoverAddTable(p, zTbl, iRoot); } recoverReset(p, pTblname); } @@ -1252,7 +1252,6 @@ static sqlite3_stmt *recoverLostAndFoundInsert( int nTotal = nField + 4; int ii; char *zBind = 0; - const char *zSep = ""; sqlite3_stmt *pRet = 0; if( p->xSql==0 ){ @@ -1328,7 +1327,7 @@ static void recoverLostAndFoundPopulate( recoverBindValue(p, pInsert, 5+ii, apVal[ii]); } if( sqlite3_step(pInsert)==SQLITE_ROW ){ - recoverSqlCallback(p, sqlite3_column_text(pInsert, 0)); + recoverSqlCallback(p, (const char*)sqlite3_column_text(pInsert, 0)); } recoverReset(p, pInsert); @@ -1376,7 +1375,7 @@ static void recoverLostAndFoundPopulate( ** db and writes all orphaned rows to it. Or, if the recover handle is ** in SQL callback mode, issues equivalent callbacks. */ -static int recoverLostAndFound(sqlite3_recover *p){ +static void recoverLostAndFound(sqlite3_recover *p){ i64 nPg = 0; RecoverBitmap *pMap = 0; @@ -1558,9 +1557,7 @@ static int recoverWriteData(sqlite3_recover *p){ if( bNewCell ){ if( nVal>=0 ){ - int ii; int iVal = 0; - int iBind = 1; if( pInsert==0 || nVal!=nInsert ){ recoverFinalize(p, pInsert); @@ -1570,9 +1567,8 @@ static int recoverWriteData(sqlite3_recover *p){ if( nVal>0 ){ for(ii=0; iinCol; ii++){ RecoverColumn *pCol = &pTab->aCol[ii]; - - if( pCol->iBind>0 ){ - int iBind = pCol->iBind; + int iBind = pCol->iBind; + if( iBind>0 ){ if( pCol->bIPK ){ sqlite3_bind_int64(pInsert, iBind, iRowid); }else if( pCol->iField Date: Fri, 23 Sep 2022 11:40:43 +0000 Subject: [PATCH 27/42] Fix various compiler warnings in new code on this branch. FossilOrigin-Name: ae49e9efde3012158061def6e0a8d993abbc5933514a21f84bc10f700f61b5e2 --- ext/recover/sqlite3recover.c | 10 +--------- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index a370ca0c46..9993aa42ac 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -526,12 +526,8 @@ static void recoverPageIsUsed( ){ sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); i64 pgno = sqlite3_value_int64(apArg[0]); - sqlite3_stmt *pStmt = 0; - int bRet; - assert( nArg==1 ); - bRet = recoverBitmapQuery(p->pUsed, pgno); - sqlite3_result_int(pCtx, bRet); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->pUsed, pgno)); } /* @@ -572,7 +568,6 @@ static void recoverGetPage( } if( pStmt ){ - int rc = SQLITE_OK; sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); @@ -1039,7 +1034,6 @@ static int recoverWriteSchema2(sqlite3_recover *p){ if( pSelect ){ while( sqlite3_step(pSelect)==SQLITE_ROW ){ - i64 iRoot = sqlite3_column_int64(pSelect, 0); const char *zSql = (const char*)sqlite3_column_text(pSelect, 1); int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0); if( rc==SQLITE_OK ){ @@ -1557,8 +1551,6 @@ static int recoverWriteData(sqlite3_recover *p){ if( bNewCell ){ if( nVal>=0 ){ - int iVal = 0; - if( pInsert==0 || nVal!=nInsert ){ recoverFinalize(p, pInsert); pInsert = recoverInsertStmt(p, pTab, nVal); diff --git a/manifest b/manifest index 92d350697a..3e97238582 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2022-09-23T11:30:24.654 +C Fix\svarious\scompiler\swarnings\sin\snew\scode\son\sthis\sbranch. +D 2022-09-23T11:40:43.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -397,7 +397,7 @@ F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72dd F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 699d14bf128f103f823dbeef0aa4987ae0f8cd822b113fca5be5615ca57f28de +F ext/recover/sqlite3recover.c e4a31b4f1f7a6d18a9c71774049262cd91c3f256f41ee257fbaa7bc71dbd5622 F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -602,7 +602,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c bb18acf4eded647fef88d4d543c673874dbebff516fbeba90a85e6c13f2a58cd -F src/shell.c.in 36d5792e35515f2389f175b41486b78b1ad6e571799e3c9b211f47c3629f96e5 +F src/shell.c.in e3efe4cf89be18c96993b714c3784946ef7af9be682ad8f0e97eb1926510befd F src/sqlite.h.in b9b7fd73239d94db20332bb6e504688001e5564b655e1318a4427a1caef4b99e F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h a988810c9b21c0dc36dc7a62735012339dc76fc7ab448fb0792721d30eacb69d @@ -2013,8 +2013,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P abcbb6abfe08fc590123f0aa1bd72cfdecc75908f078f1348dc0e957de98bf52 38aaf26e082bd95df6b64df43e1772fe6e20c4eb71307dcd97559cac7f11f8f1 -R 6b2b96167d95e43311ba941f50355c02 +P e87fa70ab0f9b95bbcde18567f47906a222a3fd02b4f3c2903d2bb087d361b7c +R cddb3dc3765037df525a9b042c70398c U dan -Z abbafc4cd8c328672c6dcbf15b485242 +Z 42c3428e8988c9082d113fe0563e38ab # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 696d9dd5ab..2f79bbe0c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e87fa70ab0f9b95bbcde18567f47906a222a3fd02b4f3c2903d2bb087d361b7c \ No newline at end of file +ae49e9efde3012158061def6e0a8d993abbc5933514a21f84bc10f700f61b5e2 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 739c2d816c..ba9f0b8eb5 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -7261,7 +7261,7 @@ end_ar_command: */ static int recoverSqlCb(void *pCtx, const char *zSql){ ShellState *pState = (ShellState*)pCtx; - raw_printf(stdout, "%s;\n", zSql); + utf8_printf(pState->out, "%s;\n", zSql); return SQLITE_OK; } From 9bbd91b39dbcf1c6657fa7d2b79d5c9be3137293 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Sep 2022 21:01:10 +0000 Subject: [PATCH 28/42] Internal changes to the recover extension towards a step() style interface. FossilOrigin-Name: 73926d5c8cd1ecece134b5a73b44ee1dfca74dc200606e3f009b06cdecf8cee9 --- ext/recover/sqlite3recover.c | 344 ++++++++++++++++++++--------------- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 208 insertions(+), 150 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 9993aa42ac..74732faca5 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -140,6 +140,23 @@ struct RecoverBitmap { u32 aElem[1]; /* Array of 32-bit bitmasks */ }; +typedef struct RecoverStateW1 RecoverStateW1; +struct RecoverStateW1 { + sqlite3_stmt *pTbls; + sqlite3_stmt *pSel; + sqlite3_stmt *pInsert; + int nInsert; + + RecoverTable *pTab; /* Table currently being written */ + int nMax; /* Max column count in any schema table */ + sqlite3_value **apVal; /* Array of nMax values */ + int nVal; /* Number of valid entries in apVal[] */ + int bHaveRowid; + i64 iRowid; + i64 iPrevPage; + int iPrevCell; +}; + /* ** Main recover handle structure. */ @@ -161,14 +178,21 @@ struct sqlite3_recover { int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ + /* Variables used with eState==RECOVER_STATE_WRITING */ + RecoverStateW1 w1; + /* Fields used within sqlite3_recover_run() */ int bRun; /* True once _recover_run() has been called */ sqlite3 *dbOut; /* Output database */ sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ - RecoverTable *pTblList; /* List of tables recovered from schem */ + RecoverTable *pTblList; /* List of tables recovered from schema */ RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ }; +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND 2 + /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). */ @@ -1381,10 +1405,26 @@ static void recoverLostAndFound(sqlite3_recover *p){ int nField = 0; /* Add all pages that are part of any tree in the recoverable part of - ** the input database schema to the bitmap. */ + ** the input database schema to the bitmap. And, if !p->bFreelistCorrupt, + ** add all pages that appear to be part of the freelist to the bitmap. + */ sqlite3_stmt *pStmt = recoverPrepare( p, p->dbOut, - "WITH roots(r) AS (" + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" " SELECT 1 UNION ALL" " SELECT rootpage FROM recovery.schema WHERE rootpage>0" ")," @@ -1395,37 +1435,16 @@ static void recoverLostAndFound(sqlite3_recover *p){ " WHERE pgno=page" ") " "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ i64 iPg = sqlite3_column_int64(pStmt, 0); recoverBitmapSet(pMap, iPg); } recoverFinalize(p, pStmt); - /* Add all pages that appear to be part of the freelist to the bitmap. */ - if( p->bFreelistCorrupt==0 ){ - pStmt = recoverPrepare(p, p->dbOut, - "WITH trunk(pgno) AS (" - " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" - " UNION" - " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" - ")," - "trunkdata(pgno, data) AS (" - " SELECT pgno, getpage(pgno) FROM trunk" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" - " UNION ALL" - " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" - ")" - "SELECT freepgno FROM freelist" - ); - while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iPg = sqlite3_column_int64(pStmt, 0); - recoverBitmapSet(pMap, iPg); - } - recoverFinalize(p, pStmt); - } /* Add an entry for each page not already added to the bitmap to ** the recovery.map table. This loop leaves the "parent" column @@ -1474,38 +1493,31 @@ static void recoverLostAndFound(sqlite3_recover *p){ } } -/* -** For each table in the recovered schema, this function extracts as much -** data as possible from the output database and writes it to the input -** database. Or, if the recover handle is in SQL callback mode, issues -** equivalent callbacks. -** -** It does not recover "orphaned" data into the lost-and-found table. -** See recoverLostAndFound() for that. -*/ -static int recoverWriteData(sqlite3_recover *p){ - RecoverTable *pTbl; - int nMax = 0; - sqlite3_value **apVal = 0; - - sqlite3_stmt *pTbls = 0; - sqlite3_stmt *pSel = 0; +static int recoverWriteDataInit(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + RecoverTable *pTbl = 0; + int nByte = 0; /* Figure out the maximum number of columns for any table in the schema */ + assert( p1->nMax==0 ); for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){ - if( pTbl->nCol>nMax ) nMax = pTbl->nCol; + if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol; } - apVal = (sqlite3_value**)recoverMalloc(p, sizeof(sqlite3_value*) * (nMax+1)); - if( apVal==0 ) return p->errCode; + /* Allocate an array of (sqlite3_value*) in which to accumulate the values + ** that will be written to the output database in a single row. */ + nByte = sizeof(sqlite3_value*) * (p1->nMax+1); + p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte); + if( p1->apVal==0 ) return p->errCode; - pTbls = recoverPrepare(p, p->dbOut, + /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT + ** to loop through cells that appear to belong to a single table (pSel). */ + p1->pTbls = recoverPrepare(p, p->dbOut, "SELECT rootpage FROM recovery.schema " " WHERE type='table' AND (sql NOT LIKE 'create virtual%')" " ORDER BY (tbl_name='sqlite_sequence') ASC" ); - - pSel = recoverPrepare(p, p->dbOut, + p1->pSel = recoverPrepare(p, p->dbOut, "WITH RECURSIVE pages(page) AS (" " SELECT ?1" " UNION" @@ -1517,114 +1529,160 @@ static int recoverWriteData(sqlite3_recover *p){ "UNION ALL " "SELECT 0, 0, 0, 0" ); - if( pSel ){ - /* The outer loop runs once for each table to recover. */ - while( sqlite3_step(pTbls)==SQLITE_ROW ){ - i64 iRoot = sqlite3_column_int64(pTbls, 0); - RecoverTable *pTab = recoverFindTable(p, iRoot); - if( pTab ){ - int ii; - sqlite3_stmt *pInsert = 0; - int nInsert = -1; - i64 iPrevPage = -1; - int iPrevCell = -1; - int bHaveRowid = 0; /* True if iRowid is valid */ - i64 iRowid = 0; - int nVal = -1; + return p->errCode; +} - if( sqlite3_stricmp("sqlite_sequence", pTab->zTab)==0 ){ - recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); - recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); +/* +** Clean up resources allocated by recoverWriteDataInit() (stuff in +** sqlite3_recover.w1). +*/ +static void recoverWriteDataCleanup(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + int ii; + for(ii=0; iinVal; ii++){ + sqlite3_value_free(p1->apVal[ii]); + } + sqlite3_free(p1->apVal); + recoverFinalize(p, p1->pInsert); + recoverFinalize(p, p1->pTbls); + recoverFinalize(p, p1->pSel); +} + +static int recoverWriteDataStep(sqlite3_recover *p){ + RecoverStateW1 *p1 = &p->w1; + sqlite3_stmt *pSel = p1->pSel; + sqlite3_value **apVal = p1->apVal; + + if( p1->pTab==0 ){ + if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ + i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); + p1->pTab = recoverFindTable(p, iRoot); + + recoverFinalize(p, p1->pInsert); + p1->pInsert = 0; + + /* If this table is unknown, return early. The caller will invoke this + ** function again and it will move on to the next table. */ + if( p1->pTab==0 ) return p->errCode; + + /* If this is the sqlite_sequence table, delete any rows added by + ** earlier INSERT statements on tables with AUTOINCREMENT primary + ** keys before recovering its contents. The p1->pTbls SELECT statement + ** is rigged to deliver "sqlite_sequence" last of all, so we don't + ** worry about it being modified after it is recovered. */ + if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){ + recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence"); + recoverSqlCallback(p, "DELETE FROM sqlite_sequence"); + } + + /* Bind the root page of this table within the original database to + ** SELECT statement p1->pSel. The SELECT statement will then iterate + ** through cells that look like they belong to table pTab. */ + sqlite3_bind_int64(pSel, 1, iRoot); + + p1->nVal = 0; + p1->bHaveRowid = 0; + p1->iPrevPage = -1; + p1->iPrevCell = -1; + }else{ + return SQLITE_DONE; + } + } + assert( p1->pTab ); + + if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ + RecoverTable *pTab = p1->pTab; + + i64 iPage = sqlite3_column_int64(pSel, 0); + int iCell = sqlite3_column_int(pSel, 1); + int iField = sqlite3_column_int(pSel, 2); + sqlite3_value *pVal = sqlite3_column_value(pSel, 3); + int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell); + + assert( bNewCell==0 || (iField==-1 || iField==0) ); + assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol ); + + if( bNewCell ){ + int ii = 0; + if( p1->nVal>=0 ){ + if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){ + recoverFinalize(p, p1->pInsert); + p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal); + p1->nInsert = p1->nVal; } - - sqlite3_bind_int64(pSel, 1, iRoot); - while( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ - i64 iPage = sqlite3_column_int64(pSel, 0); - int iCell = sqlite3_column_int(pSel, 1); - int iField = sqlite3_column_int(pSel, 2); - sqlite3_value *pVal = sqlite3_column_value(pSel, 3); - - int bNewCell = (iPrevPage!=iPage || iPrevCell!=iCell); - assert( bNewCell==0 || (iField==-1 || iField==0) ); - assert( bNewCell || iField==nVal || nVal==pTab->nCol ); - - if( bNewCell ){ - if( nVal>=0 ){ - if( pInsert==0 || nVal!=nInsert ){ - recoverFinalize(p, pInsert); - pInsert = recoverInsertStmt(p, pTab, nVal); - nInsert = nVal; - } - if( nVal>0 ){ - for(ii=0; iinCol; ii++){ - RecoverColumn *pCol = &pTab->aCol[ii]; - int iBind = pCol->iBind; - if( iBind>0 ){ - if( pCol->bIPK ){ - sqlite3_bind_int64(pInsert, iBind, iRowid); - }else if( pCol->iFieldiField]); - } - } - } - if( p->bRecoverRowid && pTab->iRowidBind>0 && bHaveRowid ){ - sqlite3_bind_int64(pInsert, pTab->iRowidBind, iRowid); - } - - if( SQLITE_ROW==sqlite3_step(pInsert) ){ - const char *z = (const char*)sqlite3_column_text(pInsert, 0); - recoverSqlCallback(p, z); - } - recoverReset(p, pInsert); - assert( p->errCode || pInsert ); - if( pInsert ) sqlite3_clear_bindings(pInsert); + if( p1->nVal>0 ){ + sqlite3_stmt *pInsert = p1->pInsert; + for(ii=0; iinCol; ii++){ + RecoverColumn *pCol = &pTab->aCol[ii]; + int iBind = pCol->iBind; + if( iBind>0 ){ + if( pCol->bIPK ){ + sqlite3_bind_int64(pInsert, iBind, p1->iRowid); + }else if( pCol->iFieldnVal ){ + recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]); } } - - for(ii=0; iinCol ){ - assert( apVal[iField]==0 ); - apVal[iField] = sqlite3_value_dup( pVal ); - if( apVal[iField]==0 ){ - recoverError(p, SQLITE_NOMEM, 0); - } - nVal = iField+1; - } - iPrevCell = iCell; - iPrevPage = iPage; + if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){ + sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid); } - } - - recoverReset(p, pSel); - recoverFinalize(p, pInsert); - pInsert = 0; - for(ii=0; iierrCode || pInsert ); + if( pInsert ) sqlite3_clear_bindings(pInsert); } } + + for(ii=0; iinVal; ii++){ + sqlite3_value_free(apVal[ii]); + apVal[ii] = 0; + } + p1->nVal = -1; + p1->bHaveRowid = 0; } + if( iPage!=0 ){ + if( iField<0 ){ + p1->iRowid = sqlite3_column_int64(pSel, 3); + assert( p1->nVal==-1 ); + p1->nVal = 0; + p1->bHaveRowid = 1; + }else if( iFieldnCol ){ + assert( apVal[iField]==0 ); + apVal[iField] = sqlite3_value_dup( pVal ); + if( apVal[iField]==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + p1->nVal = iField+1; + } + p1->iPrevCell = iCell; + p1->iPrevPage = iPage; + } + }else{ + recoverReset(p, pSel); + p1->pTab = 0; } - recoverFinalize(p, pTbls); - recoverFinalize(p, pSel); + return p->errCode; +} - sqlite3_free(apVal); +/* +** For each table in the recovered schema, this function extracts as much +** data as possible from the output database and writes it to the input +** database. Or, if the recover handle is in SQL callback mode, issues +** equivalent callbacks. +** +** It does not recover "orphaned" data into the lost-and-found table. +** See recoverLostAndFound() for that. +*/ +static int recoverWriteData(sqlite3_recover *p){ + recoverWriteDataInit(p); + while( p->errCode==SQLITE_OK && SQLITE_OK==recoverWriteDataStep(p) ); + recoverWriteDataCleanup(p); return p->errCode; } diff --git a/manifest b/manifest index 3e97238582..7ebf1eb76a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\scompiler\swarnings\sin\snew\scode\son\sthis\sbranch. -D 2022-09-23T11:40:43.554 +C Internal\schanges\sto\sthe\srecover\sextension\stowards\sa\sstep()\sstyle\sinterface. +D 2022-09-23T21:01:10.829 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -397,7 +397,7 @@ F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72dd F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c e4a31b4f1f7a6d18a9c71774049262cd91c3f256f41ee257fbaa7bc71dbd5622 +F ext/recover/sqlite3recover.c cfb2e7df6ac7405109fa4f04b9b1fea4dbb704f37985f2f34d5526949734817f F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2013,8 +2013,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e87fa70ab0f9b95bbcde18567f47906a222a3fd02b4f3c2903d2bb087d361b7c -R cddb3dc3765037df525a9b042c70398c +P ae49e9efde3012158061def6e0a8d993abbc5933514a21f84bc10f700f61b5e2 +R 84e101ee349aff282424e130c05c161d U dan -Z 42c3428e8988c9082d113fe0563e38ab +Z 2ba1da8e5fc1c584cb2093e5d330b18c # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2f79bbe0c8..8e7e75e988 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae49e9efde3012158061def6e0a8d993abbc5933514a21f84bc10f700f61b5e2 \ No newline at end of file +73926d5c8cd1ecece134b5a73b44ee1dfca74dc200606e3f009b06cdecf8cee9 \ No newline at end of file From aee404912877cd2dd0c8d026fd4484deb89c2048 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Sep 2022 18:05:52 +0000 Subject: [PATCH 29/42] Update things to use sqlite3_recover_step() internally. FossilOrigin-Name: f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f --- ext/recover/recover1.test | 2 +- ext/recover/sqlite3recover.c | 143 +++++++++++++++++++++++++++++++++-- manifest | 14 ++-- manifest.uuid | 2 +- 4 files changed, 145 insertions(+), 16 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 79258b7e8f..58774e8434 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -203,7 +203,7 @@ do_test 12.1 { do_test 12.2 { $R run $R run -} {21} +} {0} do_test 12.3 { $R finish diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 74732faca5..f2dfa3cf08 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -178,20 +178,32 @@ struct sqlite3_recover { int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ + int eState; + int bCloseTransaction; + /* Variables used with eState==RECOVER_STATE_WRITING */ RecoverStateW1 w1; /* Fields used within sqlite3_recover_run() */ - int bRun; /* True once _recover_run() has been called */ sqlite3 *dbOut; /* Output database */ sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ RecoverTable *pTblList; /* List of tables recovered from schema */ RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ }; +/* +** The various states in which an sqlite3_recover object may exist: +** +** RECOVER_STATE_INIT: +** The object is initially created in this state. sqlite3_recover_step() +** has yet to be called. This is the only state in which it is permitted +** to call sqlite3_recover_config(). +*/ #define RECOVER_STATE_INIT 0 #define RECOVER_STATE_WRITING 1 #define RECOVER_STATE_LOSTANDFOUND 2 +#define RECOVER_STATE_SCHEMA2 3 +#define RECOVER_STATE_DONE 4 /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). @@ -758,7 +770,7 @@ static int recoverOpenOutput(sqlite3_recover *p){ "PRAGMA writable_schema = 1;" "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - ); + ); sqlite3_free(zSql); } } @@ -1547,6 +1559,7 @@ static void recoverWriteDataCleanup(sqlite3_recover *p){ recoverFinalize(p, p1->pInsert); recoverFinalize(p, p1->pTbls); recoverFinalize(p, p1->pSel); + memset(p1, 0, sizeof(*p1)); } static int recoverWriteDataStep(sqlite3_recover *p){ @@ -1554,7 +1567,7 @@ static int recoverWriteDataStep(sqlite3_recover *p){ sqlite3_stmt *pSel = p1->pSel; sqlite3_value **apVal = p1->apVal; - if( p1->pTab==0 ){ + if( p->errCode==SQLITE_OK && p1->pTab==0 ){ if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(p1->pTbls, 0); p1->pTab = recoverFindTable(p, iRoot); @@ -1589,7 +1602,7 @@ static int recoverWriteDataStep(sqlite3_recover *p){ return SQLITE_DONE; } } - assert( p1->pTab ); + assert( p->errCode!=SQLITE_OK || p1->pTab ); if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){ RecoverTable *pTab = p1->pTab; @@ -1698,8 +1711,7 @@ static void recoverRun(sqlite3_recover *p){ int rc = SQLITE_OK; assert( p->errCode==SQLITE_OK ); - assert( p->bRun==0 ); - p->bRun = 1; + p->eState = 1; recoverSqlCallback(p, "BEGIN"); recoverSqlCallback(p, "PRAGMA writable_schema = on"); @@ -1741,6 +1753,100 @@ static void recoverRun(sqlite3_recover *p){ sqlite3_close(p->dbOut); } +static void recoverFinalCleanup(sqlite3_recover *p){ + RecoverTable *pTab = 0; + RecoverTable *pNext = 0; + + recoverWriteDataCleanup(p); + for(pTab=p->pTblList; pTab; pTab=pNext){ + pNext = pTab->pNext; + sqlite3_free(pTab); + } + p->pTblList = 0; + sqlite3_finalize(p->pGetPage); + p->pGetPage = 0; + recoverBitmapFree(p->pUsed); + p->pUsed = 0; + { + int res = sqlite3_close(p->dbOut); + assert( res==SQLITE_OK ); + } + p->dbOut = 0; +} + +static void recoverStep(sqlite3_recover *p){ + assert( p && p->errCode==SQLITE_OK ); + switch( p->eState ){ + case RECOVER_STATE_INIT: + /* This is the very first call to sqlite3_recover_step() on this object. + */ + recoverSqlCallback(p, "BEGIN"); + recoverSqlCallback(p, "PRAGMA writable_schema = on"); + + /* Open the output database. And register required virtual tables and + ** user functions with the new handle. */ + recoverOpenOutput(p); + + /* Open transactions on both the input and output databases. */ + recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); + recoverExec(p, p->dbIn, "BEGIN"); + if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; + recoverExec(p, p->dbOut, "BEGIN"); + + recoverCacheSchema(p); + recoverWriteSchema1(p); + + p->eState = RECOVER_STATE_WRITING; + break; + + case RECOVER_STATE_WRITING: { + if( p->w1.pTbls==0 ){ + recoverWriteDataInit(p); + } + if( SQLITE_DONE==recoverWriteDataStep(p) ){ + recoverWriteDataCleanup(p); + if( p->zLostAndFound ){ + p->eState = RECOVER_STATE_LOSTANDFOUND; + }else{ + p->eState = RECOVER_STATE_SCHEMA2; + } + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND: { + recoverLostAndFound(p); + p->eState = RECOVER_STATE_SCHEMA2; + break; + } + + case RECOVER_STATE_SCHEMA2: { + int rc = SQLITE_OK; + + recoverWriteSchema2(p); + p->eState = RECOVER_STATE_DONE; + + /* If no error has occurred, commit the write transaction on the output + ** database. Regardless of whether or not an error has occurred, make + ** an attempt to end the read transaction on the input database. */ + recoverExec(p, p->dbOut, "COMMIT"); + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + + recoverSqlCallback(p, "PRAGMA writable_schema = off"); + recoverSqlCallback(p, "COMMIT"); + p->eState = RECOVER_STATE_DONE; + recoverFinalCleanup(p); + break; + }; + + case RECOVER_STATE_DONE: { + /* no-op */ + break; + }; + } +} + /* ** This is a worker function that does the heavy lifting for both init @@ -1866,14 +1972,24 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ return rc; } +int sqlite3_recover_step(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + if( p->errCode==SQLITE_OK ) recoverStep(p); + if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){ + return SQLITE_DONE; + } + return p->errCode; +} + /* ** Do the configured recovery operation. Return SQLITE_OK if successful, or ** else an SQLite error code. */ +#if 0 int sqlite3_recover_run(sqlite3_recover *p){ if( p ){ recoverExec(p, p->dbIn, "PRAGMA writable_schema=1"); - if( p->bRun ) return SQLITE_MISUSE; /* Has already run */ + if( p->eState ) return SQLITE_MISUSE; /* Has already run */ if( p->errCode==SQLITE_OK ) recoverRun(p); if( sqlite3_exec(p->dbIn, "PRAGMA writable_schema=0", 0, 0, 0) ){ recoverDbError(p, p->dbIn); @@ -1881,6 +1997,14 @@ int sqlite3_recover_run(sqlite3_recover *p){ } return p ? p->errCode : SQLITE_NOMEM; } +#else +int sqlite3_recover_run(sqlite3_recover *p){ + if( p==0 ) return SQLITE_NOMEM; + while( SQLITE_OK==sqlite3_recover_step(p) ); + return p->errCode; +} +#endif + /* ** Free all resources associated with the recover handle passed as the only @@ -1896,6 +2020,11 @@ int sqlite3_recover_finish(sqlite3_recover *p){ if( p==0 ){ rc = SQLITE_NOMEM; }else{ + recoverFinalCleanup(p); + if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){ + rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); + if( p->errCode==SQLITE_OK ) p->errCode = rc; + } rc = p->errCode; sqlite3_free(p->zErrMsg); sqlite3_free(p->zStateDb); diff --git a/manifest b/manifest index 7ebf1eb76a..f482b45299 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Internal\schanges\sto\sthe\srecover\sextension\stowards\sa\sstep()\sstyle\sinterface. -D 2022-09-23T21:01:10.829 +C Update\sthings\sto\suse\ssqlite3_recover_step()\sinternally. +D 2022-09-24T18:05:52.914 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -387,7 +387,7 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 3f26db7692ed98c0de4fc9511f3d3a2f2c91a137b7e6666f3b85b593438b5d09 +F ext/recover/recover1.test 2b0f048ba572c3ca4e4235b632bf75702975a9656146eab766f6c75861f946a0 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 @@ -397,7 +397,7 @@ F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72dd F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c cfb2e7df6ac7405109fa4f04b9b1fea4dbb704f37985f2f34d5526949734817f +F ext/recover/sqlite3recover.c efed077977b195d2ac6c12f1485abf9c7e8c8d282834b05beee7151dabf21e88 F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2013,8 +2013,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ae49e9efde3012158061def6e0a8d993abbc5933514a21f84bc10f700f61b5e2 -R 84e101ee349aff282424e130c05c161d +P 73926d5c8cd1ecece134b5a73b44ee1dfca74dc200606e3f009b06cdecf8cee9 +R 0646ea13d6a7cac9dceb7b54cdbc6bf0 U dan -Z 2ba1da8e5fc1c584cb2093e5d330b18c +Z 958e4f167968f03754a4ebdb81e44d68 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8e7e75e988..7a6fad060d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73926d5c8cd1ecece134b5a73b44ee1dfca74dc200606e3f009b06cdecf8cee9 \ No newline at end of file +f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f \ No newline at end of file From 351563efe120aa1382136184aa8419d04f84d55f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Sep 2022 19:17:20 +0000 Subject: [PATCH 30/42] Add sqlite3_recover_step() to header file sqlite3recover.h. Update the API docs in this file. FossilOrigin-Name: 47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 --- ext/recover/sqlite3recover.c | 151 +++++++++-------------------------- ext/recover/sqlite3recover.h | 111 +++++++++++++------------ manifest | 14 ++-- manifest.uuid | 2 +- 4 files changed, 105 insertions(+), 173 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index f2dfa3cf08..fb046fbfc8 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1683,76 +1683,6 @@ static int recoverWriteDataStep(sqlite3_recover *p){ return p->errCode; } -/* -** For each table in the recovered schema, this function extracts as much -** data as possible from the output database and writes it to the input -** database. Or, if the recover handle is in SQL callback mode, issues -** equivalent callbacks. -** -** It does not recover "orphaned" data into the lost-and-found table. -** See recoverLostAndFound() for that. -*/ -static int recoverWriteData(sqlite3_recover *p){ - recoverWriteDataInit(p); - while( p->errCode==SQLITE_OK && SQLITE_OK==recoverWriteDataStep(p) ); - recoverWriteDataCleanup(p); - return p->errCode; -} - -/* -** This function does the work of sqlite3_recover_run(). It is assumed that -** no error has occurred when this is called. If an error occurs during -** the recovery operation, an error code and error message are left in -** the recovery handle. -*/ -static void recoverRun(sqlite3_recover *p){ - RecoverTable *pTab = 0; - RecoverTable *pNext = 0; - int rc = SQLITE_OK; - - assert( p->errCode==SQLITE_OK ); - p->eState = 1; - - recoverSqlCallback(p, "BEGIN"); - recoverSqlCallback(p, "PRAGMA writable_schema = on"); - - /* Open the output database. And register required virtual tables and - ** user functions with the new handle. */ - recoverOpenOutput(p); - - /* Open transactions on both the input and output databases. */ - recoverExec(p, p->dbIn, "BEGIN"); - recoverExec(p, p->dbOut, "BEGIN"); - - recoverCacheSchema(p); - recoverWriteSchema1(p); - recoverWriteData(p); - if( p->zLostAndFound ) recoverLostAndFound(p); - recoverWriteSchema2(p); - - /* If no error has occurred, commit the write transaction on the output - ** database. Then end the read transaction on the input database, regardless - ** of whether or not prior errors have occurred. */ - recoverExec(p, p->dbOut, "COMMIT"); - rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0); - if( p->errCode==SQLITE_OK ) p->errCode = rc; - - recoverSqlCallback(p, "PRAGMA writable_schema = off"); - recoverSqlCallback(p, "COMMIT"); - - /* Clean up various resources allocated by this function. */ - for(pTab=p->pTblList; pTab; pTab=pNext){ - pNext = pTab->pNext; - sqlite3_free(pTab); - } - p->pTblList = 0; - sqlite3_finalize(p->pGetPage); - p->pGetPage = 0; - recoverBitmapFree(p->pUsed); - p->pUsed = 0; - sqlite3_close(p->dbOut); -} - static void recoverFinalCleanup(sqlite3_recover *p){ RecoverTable *pTab = 0; RecoverTable *pNext = 0; @@ -1937,41 +1867,51 @@ int sqlite3_recover_errcode(sqlite3_recover *p){ */ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ int rc = SQLITE_OK; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else if( p->eState!=RECOVER_STATE_INIT ){ + rc = SQLITE_MISUSE; + }else{ + switch( op ){ + case 789: + sqlite3_free(p->zStateDb); + p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); + break; - if( p==0 ) return SQLITE_NOMEM; - switch( op ){ - case 789: - sqlite3_free(p->zStateDb); - p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg); - break; - - case SQLITE_RECOVER_LOST_AND_FOUND: { - const char *zArg = (const char*)pArg; - sqlite3_free(p->zLostAndFound); - if( zArg ){ - p->zLostAndFound = recoverMPrintf(p, "%s", zArg); - }else{ - p->zLostAndFound = 0; + case SQLITE_RECOVER_LOST_AND_FOUND: { + const char *zArg = (const char*)pArg; + sqlite3_free(p->zLostAndFound); + if( zArg ){ + p->zLostAndFound = recoverMPrintf(p, "%s", zArg); + }else{ + p->zLostAndFound = 0; + } + break; } - break; + + case SQLITE_RECOVER_FREELIST_CORRUPT: + p->bFreelistCorrupt = *(int*)pArg; + break; + + case SQLITE_RECOVER_ROWIDS: + p->bRecoverRowid = *(int*)pArg; + break; + + default: + rc = SQLITE_NOTFOUND; + break; } - - case SQLITE_RECOVER_FREELIST_CORRUPT: - p->bFreelistCorrupt = *(int*)pArg; - break; - - case SQLITE_RECOVER_ROWIDS: - p->bRecoverRowid = *(int*)pArg; - break; - - default: - rc = SQLITE_NOTFOUND; - break; } return rc; } +/* +** Do a unit of work towards the recovery job. Return SQLITE_OK if +** no error has occurred but database recovery is not finished, SQLITE_DONE +** if database recovery has been successfully completed, or an SQLite +** error code if an error has occurred. +*/ int sqlite3_recover_step(sqlite3_recover *p){ if( p==0 ) return SQLITE_NOMEM; if( p->errCode==SQLITE_OK ) recoverStep(p); @@ -1985,25 +1925,10 @@ int sqlite3_recover_step(sqlite3_recover *p){ ** Do the configured recovery operation. Return SQLITE_OK if successful, or ** else an SQLite error code. */ -#if 0 int sqlite3_recover_run(sqlite3_recover *p){ - if( p ){ - recoverExec(p, p->dbIn, "PRAGMA writable_schema=1"); - if( p->eState ) return SQLITE_MISUSE; /* Has already run */ - if( p->errCode==SQLITE_OK ) recoverRun(p); - if( sqlite3_exec(p->dbIn, "PRAGMA writable_schema=0", 0, 0, 0) ){ - recoverDbError(p, p->dbIn); - } - } - return p ? p->errCode : SQLITE_NOMEM; -} -#else -int sqlite3_recover_run(sqlite3_recover *p){ - if( p==0 ) return SQLITE_NOMEM; while( SQLITE_OK==sqlite3_recover_step(p) ); - return p->errCode; + return sqlite3_recover_errcode(p); } -#endif /* diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 30771d8804..128d823795 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -21,20 +21,30 @@ ** To use the API to recover data from a corrupted database, an ** application: ** -** 1) Creates an sqlite3_recover handle by calling either -** sqlite3_recover_init() or sqlite3_recover_init_sql(). +** 1) Creates an sqlite3_recover handle by calling either +** sqlite3_recover_init() or sqlite3_recover_init_sql(). ** -** 2) Configures the new handle using one or more calls to -** sqlite3_recover_config(). +** 2) Configures the new handle using one or more calls to +** sqlite3_recover_config(). ** -** 3) Executes the recovery by calling sqlite3_recover_run() on the handle. +** 3) Executes the recovery by repeatedly calling sqlite3_recover_step() on +** the handle until it returns something other than SQLITE_OK. If it +** returns SQLITE_DONE, then the recovery operation completed without +** error. If it returns some other non-SQLITE_OK value, then an error +** has occurred. ** -** 4) Retrieves any error code and English language error message using the -** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, -** respectively. +** 4) Retrieves any error code and English language error message using the +** sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs, +** respectively. ** -** 5) Destroys the sqlite3_recover handle and frees all resources -** using sqlite3_recover_finish(). +** 5) Destroys the sqlite3_recover handle and frees all resources +** using sqlite3_recover_finish(). +** +** The application may abandon the recovery operation at any point +** before it is finished by passing the sqlite3_recover handle to +** sqlite3_recover_finish(). This is not an error, but the final state +** of the output database, or the results of running the partial script +** delivered to the SQL callback, are undefined. */ #ifndef _SQLITE_RECOVER_H @@ -95,10 +105,13 @@ sqlite3_recover *sqlite3_recover_init_sql( /* ** Configure an sqlite3_recover object that has just been created using -** sqlite3_recover_init() or sqlite3_recover_init_sql(). The second -** argument passed to this function must be one of the SQLITE_RECOVER_* -** symbols defined below. Valid values for the third argument depend -** on the specific SQLITE_RECOVER_* symbol in use. +** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function +** may only be called before the first call to sqlite3_recover_step() +** or sqlite3_recover_run() on the object. +** +** The second argument passed to this function must be one of the +** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument +** depend on the specific SQLITE_RECOVER_* symbol in use. ** ** SQLITE_OK is returned if the configuration operation was successful, ** or an SQLite error code otherwise. @@ -137,52 +150,49 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); #define SQLITE_RECOVER_FREELIST_CORRUPT 2 #define SQLITE_RECOVER_ROWIDS 3 -/* -** Run the recovery operation. This function does not return until the -** recovery operation is completed - either the new database has been -** created and populated (sqlite3_recover_init()) or all SQL statements have -** been passed to the callback (sqlite3_recover_init_sql()) - or an error -** occurs. If the recovery is completed without error, SQLITE_OK -** is returned. It is not considered an error if data cannot be recovered +/* +** Perform a unit of work towards the recovery operation. This function +** must normally be called multiple times to complete database recovery. +** +** If no error occurs but the recovery operation is not completed, this +** function returns SQLITE_OK. If recovery has been completed successfully +** then SQLITE_DONE is returned. If an error has occurred, then an SQLite +** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not +** considered an error if some or all of the data cannot be recovered ** due to database corruption. ** -** If an error (for example an out-of-memory or IO error) occurs, then -** an SQLite error code is returned. The final state of the output database -** or the results of running any SQL statements already passed to the -** callback in this case are undefined. An English language error -** message corresponding to the error may be available via the -** sqlite3_recover_errmsg() API. +** Once sqlite3_recover_step() has returned a value other than SQLITE_OK, +** all further such calls on the same recover handle are no-ops that return +** the same non-SQLITE_OK value. +*/ +int sqlite3_recover_step(sqlite3_recover*); + +/* +** Run the recovery operation to completion. Return SQLITE_OK if successful, +** or an SQLite error code otherwise. Calling this function is the same +** as executing: ** -** This function may only be called once on an sqlite3_recover handle. -** If it is called more than once, the second and subsequent calls -** return SQLITE_MISUSE. The error code and error message returned -** by sqlite3_recover_errcode() and sqlite3_recover_errmsg() are not -** updated in this case. +** while( SQLITE_OK==sqlite3_recover_step(p) ); +** return sqlite3_recover_errcode(p); */ int sqlite3_recover_run(sqlite3_recover*); /* -** If this is called on an sqlite3_recover handle before -** sqlite3_recover_run() has been called, or if the call to -** sqlite3_recover_run() returned SQLITE_OK, then this API always returns -** a NULL pointer. +** If an error has been encountered during a prior call to +** sqlite3_recover_step(), then this function attempts to return a +** pointer to a buffer containing an English language explanation of +** the error. If no error message is available, or if an out-of memory +** error occurs while attempting to allocate a buffer in which to format +** the error message, NULL is returned. ** -** Otherwise, an attempt is made to return a pointer to a buffer containing -** an English language error message related to the error that occurred -** within the sqlite3_recover_run() call. If no error message is available, -** or if an out-of-memory error occurs while attempting to allocate a buffer -** for one, NULL may still be returned. -** -** The buffer remains valid until the sqlite3_recover handle is destroyed -** using sqlite3_recover_finish(). +** The returned buffer remains valid until the sqlite3_recover handle is +** destroyed using sqlite3_recover_finish(). */ const char *sqlite3_recover_errmsg(sqlite3_recover*); /* -** If this function is called on an sqlite3_recover handle before -** sqlite3_recover_run() has been called, it always returns SQLITE_OK. -** Otherwise, it returns a copy of the value returned by the first -** sqlite3_recover_run() call made on the handle. +** If this function is called on an sqlite3_recover handle after +** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK. */ int sqlite3_recover_errcode(sqlite3_recover*); @@ -191,10 +201,7 @@ int sqlite3_recover_errcode(sqlite3_recover*); ** The results of using a recovery object with any API after it has been ** passed to this function are undefined. ** -** If this function is called on an sqlite3_recover handle before -** sqlite3_recover_run() has been called, it always returns SQLITE_OK. -** Otherwise, it returns a copy of the value returned by the first -** sqlite3_recover_run() call made on the handle. +** This function returns the same value as sqlite3_recover_errcode(). */ int sqlite3_recover_finish(sqlite3_recover*); diff --git a/manifest b/manifest index f482b45299..b345ea4aaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthings\sto\suse\ssqlite3_recover_step()\sinternally. -D 2022-09-24T18:05:52.914 +C Add\ssqlite3_recover_step()\sto\sheader\sfile\ssqlite3recover.h.\sUpdate\sthe\sAPI\sdocs\sin\sthis\sfile. +D 2022-09-24T19:17:20.785 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -397,8 +397,8 @@ F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72dd F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c efed077977b195d2ac6c12f1485abf9c7e8c8d282834b05beee7151dabf21e88 -F ext/recover/sqlite3recover.h 81108efb8c4618d3d9c6da4df785212b0e4501aa0d25edfc463405fe839a6640 +F ext/recover/sqlite3recover.c 7752d374356a2346b5959161d5512d6f8a0bd7fb259ea81fd2473ad00dd39132 +F ext/recover/sqlite3recover.h 09a0f70b821954fc219d7583d74871e8578e4f1e9267304ec029c2bfc02bd27a F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 @@ -2013,8 +2013,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 73926d5c8cd1ecece134b5a73b44ee1dfca74dc200606e3f009b06cdecf8cee9 -R 0646ea13d6a7cac9dceb7b54cdbc6bf0 +P f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f +R b60fe138c3457719a54bec9e37633f69 U dan -Z 958e4f167968f03754a4ebdb81e44d68 +Z c514b2b66b0aba624f215b19ab2fba51 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 7a6fad060d..0de769690b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f \ No newline at end of file +47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 \ No newline at end of file From 8d8fc58413a78f932567e7d7021aa96b54270959 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Sep 2022 19:54:49 +0000 Subject: [PATCH 31/42] Add the SQLITE_RECOVER_SLOWINDEXES option, for specifying that indexes should be created and populated along with tables, instead of separately at the end of the recovery operation. FossilOrigin-Name: ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 --- ext/recover/recoverslowidx.test | 84 +++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 16 +++++-- ext/recover/sqlite3recover.h | 18 +++++++ ext/recover/test_recover.c | 17 ++++++- manifest | 17 +++---- manifest.uuid | 2 +- 6 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 ext/recover/recoverslowidx.test diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test new file mode 100644 index 0000000000..838b771eaf --- /dev/null +++ b/ext/recover/recoverslowidx.test @@ -0,0 +1,84 @@ +# 2022 September 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the SQLITE_RECOVER_SLOWINDEXES option. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl +set testprefix recoverrowid + +ifcapable !vtab { + finish_test; return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES(1, 1), (2, 2), (3, 3), (4, 4); +} + +proc my_sql_hook {sql} { + lappend ::lSql $sql + return 0 +} + +do_test 1.1 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.2 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {CREATE TABLE t1(a, b)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {CREATE INDEX i1 ON t1(a)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + +do_test 1.3 { + set lSql [list] + set R [sqlite3_recover_init_sql db main my_sql_hook] + $R config slowindexes 1 + while {[$R step]==0} { } + $R finish +} {} + +do_test 1.4 { + set lSql +} [list {*}{ + {BEGIN} + {PRAGMA writable_schema = on} + {CREATE TABLE t1(a, b)} + {CREATE INDEX i1 ON t1(a)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (3, 3, 3)} + {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (4, 4, 4)} + {PRAGMA writable_schema = off} + {COMMIT} +}] + + +finish_test + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index fb046fbfc8..aac1eea306 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -173,6 +173,7 @@ struct sqlite3_recover { char *zLostAndFound; /* Name of lost-and-found table (or NULL) */ int bFreelistCorrupt; /* SQLITE_RECOVER_FREELIST_CORRUPT setting */ int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ + int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ /* Error code and error message */ int errCode; /* For sqlite3_recover_errcode() */ @@ -991,16 +992,16 @@ static int recoverWriteSchema1(sqlite3_recover *p){ sqlite3_stmt *pTblname = 0; pSelect = recoverPrepare(p, p->dbOut, - "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isUnique) AS (" + "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS (" " SELECT rootpage, name, sql, " " type='table', " " sql LIKE 'create virtual%'," - " (type='index' AND sql LIKE '%unique%')" + " (type='index' AND (sql LIKE '%unique%' OR ?1))" " FROM recovery.schema" ")" "SELECT rootpage, tbl, isVirtual, name, sql" " FROM dbschema " - " WHERE tbl OR isUnique" + " WHERE tbl OR isIndex" " ORDER BY tbl DESC, name=='sqlite_sequence' DESC" ); @@ -1010,6 +1011,7 @@ static int recoverWriteSchema1(sqlite3_recover *p){ ); if( pSelect ){ + sqlite3_bind_int(pSelect, 1, p->bSlowIndexes); while( sqlite3_step(pSelect)==SQLITE_ROW ){ i64 iRoot = sqlite3_column_int64(pSelect, 0); int bTable = sqlite3_column_int(pSelect, 1); @@ -1064,6 +1066,10 @@ static int recoverWriteSchema2(sqlite3_recover *p){ sqlite3_stmt *pSelect = 0; pSelect = recoverPrepare(p, p->dbOut, + p->bSlowIndexes ? + "SELECT rootpage, sql FROM recovery.schema " + " WHERE type!='table' AND type!='index'" + : "SELECT rootpage, sql FROM recovery.schema " " WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')" ); @@ -1897,6 +1903,10 @@ int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){ p->bRecoverRowid = *(int*)pArg; break; + case SQLITE_RECOVER_SLOWINDEXES: + p->bSlowIndexes = *(int*)pArg; + break; + default: rc = SQLITE_NOTFOUND; break; diff --git a/ext/recover/sqlite3recover.h b/ext/recover/sqlite3recover.h index 128d823795..d0a37504e7 100644 --- a/ext/recover/sqlite3recover.h +++ b/ext/recover/sqlite3recover.h @@ -145,10 +145,28 @@ int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg); ** that are not also INTEGER PRIMARY KEY values. If this option is ** clear, then new rowids are assigned to all recovered rows. The ** default value is 1 (set). +** +** SQLITE_RECOVER_SLOWINDEXES: +** The pArg value must actually be a pointer to a value of type +** int containing value 0 or 1 cast as a (void*). If this option is clear +** (argument is 0), then when creating an output database, the recover +** module creates and populates non-UNIQUE indexes right at the end of the +** recovery operation - after all recoverable data has been inserted +** into the new database. This is faster overall, but means that the +** final call to sqlite3_recover_step() for a recovery operation may +** be need to create a large number of indexes, which may be very slow. +** +** Or, if this option is set (argument is 1), then non-UNIQUE indexes +** are created in the output database before it is populated with +** recovered data. This is slower overall, but avoids the slow call +** to sqlite3_recover_step() at the end of the recovery operation. +** +** The default option value is 0. */ #define SQLITE_RECOVER_LOST_AND_FOUND 1 #define SQLITE_RECOVER_FREELIST_CORRUPT 2 #define SQLITE_RECOVER_ROWIDS 3 +#define SQLITE_RECOVER_SLOWINDEXES 4 /* ** Perform a unit of work towards the recovery operation. This function diff --git a/ext/recover/test_recover.c b/ext/recover/test_recover.c index d660a5345e..99c7aeca34 100644 --- a/ext/recover/test_recover.c +++ b/ext/recover/test_recover.c @@ -89,6 +89,7 @@ static int testRecoverCmd( { "errmsg", 0, "" }, /* 2 */ { "errcode", 0, "" }, /* 3 */ { "finish", 0, "" }, /* 4 */ + { "step", 0, "" }, /* 5 */ { 0 } }; int rc = TCL_OK; @@ -115,7 +116,8 @@ static int testRecoverCmd( "lostandfound", /* 1 */ "freelistcorrupt", /* 2 */ "rowids", /* 3 */ - "invalid", /* 4 */ + "slowindexes", /* 4 */ + "invalid", /* 5 */ 0 }; int iOp = 0; @@ -153,6 +155,14 @@ static int testRecoverCmd( break; } case 4: { + int iVal = 0; + if( Tcl_GetBooleanFromObj(interp, objv[3], &iVal) ) return TCL_ERROR; + res = sqlite3_recover_config(pTest->p, + SQLITE_RECOVER_SLOWINDEXES, (void*)&iVal + ); + break; + } + case 5: { res = sqlite3_recover_config(pTest->p, 12345, 0); break; } @@ -187,6 +197,11 @@ static int testRecoverCmd( if( res ) return TCL_ERROR; break; } + case 5: assert( sqlite3_stricmp("step", aSub[iSub].zSub)==0 ); { + int res = sqlite3_recover_step(pTest->p); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + break; + } } return TCL_OK; diff --git a/manifest b/manifest index b345ea4aaf..f6cebeee1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssqlite3_recover_step()\sto\sheader\sfile\ssqlite3recover.h.\sUpdate\sthe\sAPI\sdocs\sin\sthis\sfile. -D 2022-09-24T19:17:20.785 +C Add\sthe\sSQLITE_RECOVER_SLOWINDEXES\soption,\sfor\sspecifying\sthat\sindexes\sshould\sbe\screated\sand\spopulated\salong\swith\stables,\sinstead\sof\sseparately\sat\sthe\send\sof\sthe\srecovery\soperation. +D 2022-09-24T19:54:49.642 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -396,10 +396,11 @@ F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 +F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 7752d374356a2346b5959161d5512d6f8a0bd7fb259ea81fd2473ad00dd39132 -F ext/recover/sqlite3recover.h 09a0f70b821954fc219d7583d74871e8578e4f1e9267304ec029c2bfc02bd27a -F ext/recover/test_recover.c 72a765616a3fa9dae2ed537d79b00f365d9f639d347858341b71bda7a3a45f56 +F ext/recover/sqlite3recover.c 7cd26aeb413b5b1cec1ae6213488dcec3e4e822911d3f255272b32cf0f32c406 +F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 +F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c e21f06995ff4efdc1622dcceaea4dcba2caa83ca2f31a1607b98a8509168a996 F ext/repair/checkindex.c 4383e4469c21e5b9ae321d0d63cec53e981af9d7a6564be6374f0eeb93dfc890 @@ -2013,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f4b15aad3005237b7ac507eed2b9b07e0f5c9407ab28f2656a21c9845f13d35f -R b60fe138c3457719a54bec9e37633f69 +P 47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 +R 9e568d3e4f6455e4cabb845dec54247e U dan -Z c514b2b66b0aba624f215b19ab2fba51 +Z a8bd0fc6063574ee27e4134a0a1e02e3 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 0de769690b..8f9e8a36bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 \ No newline at end of file +ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 \ No newline at end of file From 7e33f8f43b19296152920abfa78d02ab93793af9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Sep 2022 20:51:16 +0000 Subject: [PATCH 32/42] Modify implementation of recovery to break recovery of the lost_and_found table into multiple steps. FossilOrigin-Name: adedfd040bb2d1c2dcda0b91c6073f12953e45d9b860a8f217d990e861d66dfb --- ext/recover/sqlite3recover.c | 457 +++++++++++++++++++++++------------ manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 312 insertions(+), 159 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index aac1eea306..e492cdf246 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -157,6 +157,22 @@ struct RecoverStateW1 { int iPrevCell; }; +typedef struct RecoverStateLAF RecoverStateLAF; +struct RecoverStateLAF { + RecoverBitmap *pUsed; + i64 nPg; /* Size of db in pages */ + sqlite3_stmt *pAllAndParent; + sqlite3_stmt *pMapInsert; + sqlite3_stmt *pMaxField; + sqlite3_stmt *pUsedPages; + sqlite3_stmt *pFindRoot; + sqlite3_stmt *pInsert; /* INSERT INTO lost_and_found ... */ + sqlite3_stmt *pAllPage; + sqlite3_stmt *pPageData; + sqlite3_value **apVal; + int nMaxField; +}; + /* ** Main recover handle structure. */ @@ -185,11 +201,13 @@ struct sqlite3_recover { /* Variables used with eState==RECOVER_STATE_WRITING */ RecoverStateW1 w1; + /* Variables used with states RECOVER_STATE_LOSTANDFOUND* */ + RecoverStateLAF laf; + /* Fields used within sqlite3_recover_run() */ sqlite3 *dbOut; /* Output database */ sqlite3_stmt *pGetPage; /* SELECT against input db sqlite_dbdata */ RecoverTable *pTblList; /* List of tables recovered from schema */ - RecoverBitmap *pUsed; /* Used by recoverLostAndFound() */ }; /* @@ -199,12 +217,28 @@ struct sqlite3_recover { ** The object is initially created in this state. sqlite3_recover_step() ** has yet to be called. This is the only state in which it is permitted ** to call sqlite3_recover_config(). +** +** RECOVER_STATE_WRITING: +** +** RECOVER_STATE_LOSTANDFOUND1: +** State to populate the bitmap of pages used by other tables or the +** database freelist. +** +** RECOVER_STATE_LOSTANDFOUND2: +** Populate the recovery.map table - used to figure out a "root" page +** for each lost page from in the database from which records are +** extracted. +** +** RECOVER_STATE_LOSTANDFOUND3: +** Populate the lost-and-found table itself. */ -#define RECOVER_STATE_INIT 0 -#define RECOVER_STATE_WRITING 1 -#define RECOVER_STATE_LOSTANDFOUND 2 -#define RECOVER_STATE_SCHEMA2 3 -#define RECOVER_STATE_DONE 4 +#define RECOVER_STATE_INIT 0 +#define RECOVER_STATE_WRITING 1 +#define RECOVER_STATE_LOSTANDFOUND1 2 +#define RECOVER_STATE_LOSTANDFOUND2 3 +#define RECOVER_STATE_LOSTANDFOUND3 4 +#define RECOVER_STATE_SCHEMA2 5 +#define RECOVER_STATE_DONE 6 /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). @@ -320,7 +354,7 @@ static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){ */ static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){ int ret = 1; - if( iPg<=pMap->nPg ){ + if( iPg<=pMap->nPg && iPg>0 ){ int iElem = (iPg / 32); int iBit = (iPg % 32); ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0; @@ -548,7 +582,7 @@ static void recoverReadI32( ** Implementation of SQL scalar function "page_is_used". This function ** is used as part of the procedure for locating orphan rows for the ** lost-and-found table, and it depends on those routines having populated -** the sqlite3_recover.pUsed variable. +** the sqlite3_recover.laf.pUsed variable. ** ** The only argument to this function is a page-number. It returns true ** if the page has already been used somehow during data recovery, or false @@ -564,7 +598,7 @@ static void recoverPageIsUsed( sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx); i64 pgno = sqlite3_value_int64(apArg[0]); assert( nArg==1 ); - sqlite3_result_int(pCtx, recoverBitmapQuery(p->pUsed, pgno)); + sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno)); } /* @@ -600,13 +634,14 @@ static void recoverGetPage( pStmt = p->pGetPage = recoverPreparePrintf( p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb ); - }else{ + }else if( p->errCode==SQLITE_OK ){ pStmt = p->pGetPage; } if( pStmt ){ sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ + assert( p->errCode==SQLITE_OK ); sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); } recoverReset(p, pStmt); @@ -1312,49 +1347,59 @@ static sqlite3_stmt *recoverLostAndFoundInsert( return pRet; } -/* -** Helper function for recoverLostAndFound(). -*/ -static void recoverLostAndFoundPopulate( +static int recoverLostAndFoundFindRoot( sqlite3_recover *p, - sqlite3_stmt *pInsert, - int nField + i64 iPg, + i64 *piRoot ){ - sqlite3_stmt *pStmt = recoverPrepare(p, p->dbOut, - "WITH RECURSIVE pages(root, page) AS (" - " SELECT pgno, pgno FROM recovery.map WHERE parent IS NULL" - " UNION" - " SELECT root, child FROM sqlite_dbptr('getpage()'), pages " - " WHERE pgno=page" - ") " - "SELECT root, page, cell, field, value " - "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno " - " AND NOT page_is_used(page) " - "UNION ALL " - "SELECT 0, 0, 0, 0, 0" - ); + RecoverStateLAF *pLaf = &p->laf; + + if( pLaf->pFindRoot==0 ){ + pLaf->pFindRoot = recoverPrepare(p, p->dbOut, + "WITH RECURSIVE p(pgno) AS (" + " SELECT ?" + " UNION" + " SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno" + ") " + "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno " + " AND m.parent IS NULL" + ); + } + if( p->errCode==SQLITE_OK ){ + sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg); + if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){ + *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0); + }else{ + *piRoot = iPg; + } + recoverReset(p, pLaf->pFindRoot); + } + return p->errCode; +} + +static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_value **apVal = pLaf->apVal; + sqlite3_stmt *pPageData = pLaf->pPageData; + sqlite3_stmt *pInsert = pLaf->pInsert; - sqlite3_value **apVal = 0; int nVal = -1; - i64 iRowid = 0; + int iPrevCell = 0; + i64 iRoot = 0; int bHaveRowid = 0; - int ii; + i64 iRowid = 0; + int ii = 0; - i64 iPrevRoot = -1; - i64 iPrevPage = -1; - int iPrevCell = -1; + if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return; + sqlite3_bind_int64(pPageData, 1, iPage); + while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){ + int iCell = sqlite3_column_int64(pPageData, 0); + int iField = sqlite3_column_int64(pPageData, 1); - apVal = (sqlite3_value**)recoverMalloc(p, nField*sizeof(sqlite3_value*)); - while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iRoot = sqlite3_column_int64(pStmt, 0); - i64 iPage = sqlite3_column_int64(pStmt, 1); - int iCell = sqlite3_column_int64(pStmt, 2); - int iField = sqlite3_column_int64(pStmt, 3); - - if( iPrevRoot>0 && (iPrevPage!=iPage || iPrevCell!=iCell) ){ + if( iPrevCell!=iCell && nVal>=0 ){ /* Insert the new row */ - sqlite3_bind_int64(pInsert, 1, iPrevRoot); /* rootpgno */ - sqlite3_bind_int64(pInsert, 2, iPrevPage); /* pgno */ + sqlite3_bind_int64(pInsert, 1, iRoot); /* rootpgno */ + sqlite3_bind_int64(pInsert, 2, iPage); /* pgno */ sqlite3_bind_int(pInsert, 3, nVal); /* nfield */ if( bHaveRowid ){ sqlite3_bind_int64(pInsert, 4, iRowid); /* id */ @@ -1377,13 +1422,15 @@ static void recoverLostAndFoundPopulate( nVal = -1; } + if( iCell<0 ) break; + if( iField<0 ){ assert( nVal==-1 ); - iRowid = sqlite3_column_int64(pStmt, 4); + iRowid = sqlite3_column_int64(pPageData, 2); bHaveRowid = 1; nVal = 0; - }else if( iFieldnMaxField ){ + sqlite3_value *pVal = sqlite3_column_value(pPageData, 2); apVal[iField] = sqlite3_value_dup(pVal); assert( iField==nVal || (nVal==-1 && iField==0) ); nVal = iField+1; @@ -1392,122 +1439,65 @@ static void recoverLostAndFoundPopulate( } } - iPrevRoot = iRoot; - iPrevPage = iPage; iPrevCell = iCell; } - recoverFinalize(p, pStmt); + recoverReset(p, pPageData); for(ii=0; iipUsed = recoverBitmapAlloc(p, nPg); - if( pMap ){ - char *zTab = 0; /* Name of lost_and_found table */ - sqlite3_stmt *pInsert = 0; /* INSERT INTO lost_and_found ... */ - int nField = 0; - - /* Add all pages that are part of any tree in the recoverable part of - ** the input database schema to the bitmap. And, if !p->bFreelistCorrupt, - ** add all pages that appear to be part of the freelist to the bitmap. - */ - sqlite3_stmt *pStmt = recoverPrepare( - p, p->dbOut, - "WITH trunk(pgno) AS (" - " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" - " UNION" - " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" - ")," - "trunkdata(pgno, data) AS (" - " SELECT pgno, getpage(pgno) FROM trunk" - ")," - "freelist(data, n, freepgno) AS (" - " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" - " UNION ALL" - " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" - ")," - "" - "roots(r) AS (" - " SELECT 1 UNION ALL" - " SELECT rootpage FROM recovery.schema WHERE rootpage>0" - ")," - "used(page) AS (" - " SELECT r FROM roots" - " UNION" - " SELECT child FROM sqlite_dbptr('getpage()'), used " - " WHERE pgno=page" - ") " - "SELECT page FROM used" - " UNION ALL " - "SELECT freepgno FROM freelist WHERE NOT ?" - ); - if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); - while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ - i64 iPg = sqlite3_column_int64(pStmt, 0); - recoverBitmapSet(pMap, iPg); +static int recoverLostAndFound3Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + if( pLaf->pInsert==0 ){ + return SQLITE_DONE; + }else{ + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllPage); + if( res==SQLITE_ROW ){ + i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0); + if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){ + recoverLostAndFoundOnePage(p, iPage); + } + }else{ + recoverReset(p, pLaf->pAllPage); + return SQLITE_DONE; + } + } } - recoverFinalize(p, pStmt); + } + return SQLITE_OK; +} +static void recoverLostAndFound3Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; - /* Add an entry for each page not already added to the bitmap to - ** the recovery.map table. This loop leaves the "parent" column - ** of each recovery.map row set to NULL - to be filled in below. */ - pStmt = recoverPreparePrintf( - p, p->dbOut, + if( pLaf->nMaxField>0 ){ + char *zTab = 0; /* Name of lost_and_found table */ + + zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField); + pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField); + sqlite3_free(zTab); + + pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut, "WITH RECURSIVE seq(ii) AS (" " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" ")" - "INSERT INTO recovery.map(pgno) " - " SELECT ii FROM seq WHERE NOT page_is_used(ii)", nPg + "SELECT ii FROM seq" , p->laf.nPg + ); + pLaf->pPageData = recoverPrepare(p, p->dbOut, + "SELECT cell, field, value " + "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? " + "UNION ALL " + "SELECT -1, -1, -1" ); - sqlite3_step(pStmt); - recoverFinalize(p, pStmt); - /* Set the "parent" column for each row of the recovery.map table */ - pStmt = recoverPrepare( - p, p->dbOut, - "UPDATE recovery.map SET parent = ptr.pgno " - " FROM sqlite_dbptr('getpage()') AS ptr " - " WHERE recovery.map.pgno=ptr.child" + pLaf->apVal = (sqlite3_value**)recoverMalloc(p, + pLaf->nMaxField*sizeof(sqlite3_value*) ); - sqlite3_step(pStmt); - recoverFinalize(p, pStmt); - - /* Figure out the number of fields in the longest record that will be - ** recovered into the lost_and_found table. Set nField to this value. */ - pStmt = recoverPrepare( - p, p->dbOut, - "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno IN (" - " SELECT pgno FROM recovery.map" - ")" - ); - if( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){ - nField = sqlite3_column_int64(pStmt, 0); - } - recoverFinalize(p, pStmt); - - if( nField>0 ){ - zTab = recoverLostAndFoundCreate(p, nField); - pInsert = recoverLostAndFoundInsert(p, zTab, nField); - recoverLostAndFoundPopulate(p, pInsert, nField); - recoverFinalize(p, pInsert); - sqlite3_free(zTab); - } } } @@ -1689,11 +1679,152 @@ static int recoverWriteDataStep(sqlite3_recover *p){ return p->errCode; } +static void recoverLostAndFound1Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + sqlite3_stmt *pStmt = 0; + + assert( p->laf.pUsed==0 ); + pLaf->nPg = recoverPageCount(p); + pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg); + + /* Prepare a statement to iterate through all pages that are part of any tree + ** in the recoverable part of the input database schema to the bitmap. And, + ** if !p->bFreelistCorrupt, add all pages that appear to be part of the + ** freelist. */ + pStmt = recoverPrepare( + p, p->dbOut, + "WITH trunk(pgno) AS (" + " SELECT read_i32(getpage(1), 8) AS x WHERE x>0" + " UNION" + " SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0" + ")," + "trunkdata(pgno, data) AS (" + " SELECT pgno, getpage(pgno) FROM trunk" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata" + " UNION ALL" + " SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0" + ")," + "" + "roots(r) AS (" + " SELECT 1 UNION ALL" + " SELECT rootpage FROM recovery.schema WHERE rootpage>0" + ")," + "used(page) AS (" + " SELECT r FROM roots" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), used " + " WHERE pgno=page" + ") " + "SELECT page FROM used" + " UNION ALL " + "SELECT freepgno FROM freelist WHERE NOT ?" + ); + if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt); + pLaf->pUsedPages = pStmt; +} + +static int recoverLostAndFound1Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + int rc = p->errCode; + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pLaf->pUsedPages); + if( rc==SQLITE_ROW ){ + i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0); + recoverBitmapSet(pLaf->pUsed, iPg); + rc = SQLITE_OK; + }else{ + recoverFinalize(p, pLaf->pUsedPages); + pLaf->pUsedPages = 0; + } + } + return rc; +} + +static void recoverLostAndFound2Init(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + + assert( p->laf.pAllAndParent==0 ); + assert( p->laf.pMapInsert==0 ); + assert( p->laf.pMaxField==0 ); + assert( p->laf.nMaxField==0 ); + + pLaf->pMapInsert = recoverPrepare(p, p->dbOut, + "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)" + ); + pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut, + "WITH RECURSIVE seq(ii) AS (" + " SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld" + ")" + "SELECT pgno, child FROM sqlite_dbptr('getpage()') " + " UNION ALL " + "SELECT NULL, ii FROM seq", p->laf.nPg + ); + pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut, + "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?" + ); +} + +static int recoverLostAndFound2Step(sqlite3_recover *p){ + RecoverStateLAF *pLaf = &p->laf; + if( p->errCode==SQLITE_OK ){ + int res = sqlite3_step(pLaf->pAllAndParent); + if( res==SQLITE_ROW ){ + i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1); + if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){ + sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild); + sqlite3_bind_value(pLaf->pMapInsert, 2, + sqlite3_column_value(pLaf->pAllAndParent, 0) + ); + sqlite3_step(pLaf->pMapInsert); + recoverReset(p, pLaf->pMapInsert); + sqlite3_bind_int64(pLaf->pMaxField, 1, iChild); + if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){ + int nMax = sqlite3_column_int(pLaf->pMaxField, 0); + if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax; + } + recoverReset(p, pLaf->pMaxField); + } + }else{ + recoverFinalize(p, pLaf->pAllAndParent); + pLaf->pAllAndParent =0; + return SQLITE_DONE; + } + } + return p->errCode; +} + +static void recoverLostAndFoundCleanup(sqlite3_recover *p){ + recoverBitmapFree(p->laf.pUsed); + p->laf.pUsed = 0; + sqlite3_finalize(p->laf.pUsedPages); + sqlite3_finalize(p->laf.pAllAndParent); + sqlite3_finalize(p->laf.pMapInsert); + sqlite3_finalize(p->laf.pMaxField); + sqlite3_finalize(p->laf.pFindRoot); + sqlite3_finalize(p->laf.pInsert); + sqlite3_finalize(p->laf.pAllPage); + sqlite3_finalize(p->laf.pPageData); + p->laf.pUsedPages = 0; + p->laf.pAllAndParent = 0; + p->laf.pMapInsert = 0; + p->laf.pMaxField = 0; + p->laf.pFindRoot = 0; + p->laf.pInsert = 0; + p->laf.pAllPage = 0; + p->laf.pPageData = 0; + sqlite3_free(p->laf.apVal); + p->laf.apVal = 0; +} + static void recoverFinalCleanup(sqlite3_recover *p){ RecoverTable *pTab = 0; RecoverTable *pNext = 0; recoverWriteDataCleanup(p); + recoverLostAndFoundCleanup(p); + for(pTab=p->pTblList; pTab; pTab=pNext){ pNext = pTab->pNext; sqlite3_free(pTab); @@ -1701,8 +1832,7 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->pTblList = 0; sqlite3_finalize(p->pGetPage); p->pGetPage = 0; - recoverBitmapFree(p->pUsed); - p->pUsed = 0; + { int res = sqlite3_close(p->dbOut); assert( res==SQLITE_OK ); @@ -1742,7 +1872,7 @@ static void recoverStep(sqlite3_recover *p){ if( SQLITE_DONE==recoverWriteDataStep(p) ){ recoverWriteDataCleanup(p); if( p->zLostAndFound ){ - p->eState = RECOVER_STATE_LOSTANDFOUND; + p->eState = RECOVER_STATE_LOSTANDFOUND1; }else{ p->eState = RECOVER_STATE_SCHEMA2; } @@ -1750,9 +1880,32 @@ static void recoverStep(sqlite3_recover *p){ break; } - case RECOVER_STATE_LOSTANDFOUND: { - recoverLostAndFound(p); - p->eState = RECOVER_STATE_SCHEMA2; + case RECOVER_STATE_LOSTANDFOUND1: { + if( p->laf.pUsed==0 ){ + recoverLostAndFound1Init(p); + } + if( SQLITE_DONE==recoverLostAndFound1Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND2; + } + break; + } + case RECOVER_STATE_LOSTANDFOUND2: { + if( p->laf.pAllAndParent==0 ){ + recoverLostAndFound2Init(p); + } + if( SQLITE_DONE==recoverLostAndFound2Step(p) ){ + p->eState = RECOVER_STATE_LOSTANDFOUND3; + } + break; + } + + case RECOVER_STATE_LOSTANDFOUND3: { + if( p->laf.pInsert==0 ){ + recoverLostAndFound3Init(p); + } + if( SQLITE_DONE==recoverLostAndFound3Step(p) ){ + p->eState = RECOVER_STATE_SCHEMA2; + } break; } diff --git a/manifest b/manifest index f6cebeee1e..66d4798e47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_RECOVER_SLOWINDEXES\soption,\sfor\sspecifying\sthat\sindexes\sshould\sbe\screated\sand\spopulated\salong\swith\stables,\sinstead\sof\sseparately\sat\sthe\send\sof\sthe\srecovery\soperation. -D 2022-09-24T19:54:49.642 +C Modify\simplementation\sof\srecovery\sto\sbreak\srecovery\sof\sthe\slost_and_found\stable\sinto\smultiple\ssteps. +D 2022-09-26T20:51:16.327 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -398,7 +398,7 @@ F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e9 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 7cd26aeb413b5b1cec1ae6213488dcec3e4e822911d3f255272b32cf0f32c406 +F ext/recover/sqlite3recover.c 5aad3f4e569e8637abdadfeb748ec1b99198d6d45bf983c6b168a527f5891b7f F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 47f416153035d6bf0ae27587583d2957c87c02caecbd4e7f363bcb2bc27cf159 -R 9e568d3e4f6455e4cabb845dec54247e +P ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 +R d7188e5461609c695208ab621d6c4574 U dan -Z a8bd0fc6063574ee27e4134a0a1e02e3 +Z d49040b82cb2efa369225859bd08a957 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 8f9e8a36bd..d70afc5526 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 \ No newline at end of file +adedfd040bb2d1c2dcda0b91c6073f12953e45d9b860a8f217d990e861d66dfb \ No newline at end of file From 041cc06139b0a6878c5dc850d9abf7bc79d76e1f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 4 Oct 2022 19:43:12 +0000 Subject: [PATCH 33/42] Better handling for databases with corrupt headers. FossilOrigin-Name: 17f68d803685405d880025134c71bac01077962d677f2902a608952d0b9cdb5a --- ext/recover/sqlite3recover.c | 269 ++++++++++++++++++++++++++++++++++- manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 275 insertions(+), 8 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index e492cdf246..dd294dcc7b 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -28,6 +28,7 @@ __declspec(dllexport) int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*); typedef unsigned int u32; +typedef unsigned char u8; typedef sqlite3_int64 i64; typedef struct RecoverTable RecoverTable; @@ -240,11 +241,30 @@ struct sqlite3_recover { #define RECOVER_STATE_SCHEMA2 5 #define RECOVER_STATE_DONE 6 + +/* +** Global variables used by this extension. +*/ +typedef struct RecoverGlobal RecoverGlobal; +struct RecoverGlobal { + const sqlite3_io_methods *pMethods; +}; +static RecoverGlobal recover_g; + +/* +** Use this static SQLite mutex to protect the globals during the +** first call to sqlite3_recover_step(). +*/ +#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 + + + /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). */ #define RECOVER_ROWID_DEFAULT 1 + /* ** Like strlen(). But handles NULL pointer arguments. */ @@ -1840,6 +1860,247 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->dbOut = 0; } +static int recoverVfsClose(sqlite3_file*); +static int recoverVfsRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int recoverVfsWrite(sqlite3_file*, const void*, int, sqlite3_int64); +static int recoverVfsTruncate(sqlite3_file*, sqlite3_int64 size); +static int recoverVfsSync(sqlite3_file*, int flags); +static int recoverVfsFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int recoverVfsLock(sqlite3_file*, int); +static int recoverVfsUnlock(sqlite3_file*, int); +static int recoverVfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int recoverVfsFileControl(sqlite3_file*, int op, void *pArg); +static int recoverVfsSectorSize(sqlite3_file*); +static int recoverVfsDeviceCharacteristics(sqlite3_file*); +static int recoverVfsShmMap(sqlite3_file*, int, int, int, void volatile**); +static int recoverVfsShmLock(sqlite3_file*, int offset, int n, int flags); +static void recoverVfsShmBarrier(sqlite3_file*); +static int recoverVfsShmUnmap(sqlite3_file*, int deleteFlag); + +static sqlite3_io_methods recover_methods = { + 2, /* iVersion */ + recoverVfsClose, + recoverVfsRead, + recoverVfsWrite, + recoverVfsTruncate, + recoverVfsSync, + recoverVfsFileSize, + recoverVfsLock, + recoverVfsUnlock, + recoverVfsCheckReservedLock, + recoverVfsFileControl, + recoverVfsSectorSize, + recoverVfsDeviceCharacteristics, + recoverVfsShmMap, + recoverVfsShmLock, + recoverVfsShmBarrier, + recoverVfsShmUnmap, + 0, 0 +}; + +static int recoverVfsClose(sqlite3_file *pFd){ + assert( pFd->pMethods!=&recover_methods ); + return pFd->pMethods->xClose(pFd); +} + +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} + +static void recoverPutU16(u8 *a, u32 v){ + a[0] = (v>>8) & 0x00FF; + a[1] = (v>>0) & 0x00FF; +} +static u32 recoverPutU32(u8 *a, u32 v){ + a[0] = (v>>24) & 0x00FF; + a[1] = (v>>16) & 0x00FF; + a[2] = (v>>8) & 0x00FF; + a[3] = (v>>0) & 0x00FF; +} + +static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ + int rc = SQLITE_OK; + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + if( rc==SQLITE_OK && iOff==0 && nByte>=100 ){ + /* Ensure that the database has a valid header file. The only fields + ** that really matter to recovery are: + ** + ** + Database page size (16-bits at offset 16) + ** + Size of db in pages (32-bits at offset 28) + ** + Database encoding (32-bits at offset 56) + ** + ** Also preserved are: + ** + ** + first freelist page (32-bits at offset 32) + ** + size of freelist (32-bits at offset 36) + ** + ** We also try to preserve the auto-vacuum, incr-value, user-version + ** and application-id fields - all 32 bit quantities at offsets + ** 52, 60, 64 and 68. All other fields are set to known good values. + */ + const int aPreserve[] = {32, 36, 52, 60, 64, 68}; + u8 aHdr[100] = { + 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, + 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, + 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, + 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x2e, 0x5b, 0x30 + }; + u8 *a = (u8*)aBuf; + + u32 pgsz = recoverGetU16(&a[16]); + u32 enc = recoverGetU32(&a[56]); + u32 dbsz = 0; + i64 dbFileSize = 0; + int ii; + + if( pgsz==0x01 ) pgsz = 65536; + rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); + + dbsz = dbFileSize / pgsz; + if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ + enc = SQLITE_UTF8; + } + + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + for(ii=0; iipMethods = &recover_methods; + }else{ + rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); + } + return rc; +} + +#define RECOVER_VFS_WRAPPER(code) \ + int rc = SQLITE_OK; \ + if( pFd->pMethods==&recover_methods ){ \ + pFd->pMethods = recover_g.pMethods; \ + rc = code; \ + pFd->pMethods = &recover_methods; \ + }else{ \ + rc = code; \ + } \ + return rc; + +static int recoverVfsWrite( + sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff) + ); +} +static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xTruncate(pFd, size) + ); +} + +static int recoverVfsSync(sqlite3_file *pFd, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSync(pFd, flags) + ); +} +static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileSize(pFd, pSize) + ); +} +static int recoverVfsLock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xLock(pFd, eLock) + ); +} +static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xUnlock(pFd, eLock) + ); +} +static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xCheckReservedLock(pFd, pResOut) + ); +} +static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xFileControl(pFd, op, pArg) + ); +} +static int recoverVfsSectorSize(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xSectorSize(pFd) + ); +} +static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xDeviceCharacteristics(pFd) + ); +} +static int recoverVfsShmMap( + sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp +){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp) + ); +} +static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmLock(pFd, offset, n, flags) + ); +} +static void recoverVfsShmBarrier(sqlite3_file *pFd){ + if( pFd->pMethods==&recover_methods ){ + pFd->pMethods = recover_g.pMethods; + pFd->pMethods->xShmBarrier(pFd); + pFd->pMethods = &recover_methods; + }else{ + pFd->pMethods->xShmBarrier(pFd); + } +} +static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ + RECOVER_VFS_WRAPPER ( + pFd->pMethods->xShmUnmap(pFd, deleteFlag) + ); +} + +static void recoverInstallWrapper(sqlite3_recover *p){ + sqlite3_file *pFd = 0; + assert( recover_g.pMethods==0 ); + sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); + if( pFd ){ + recover_g.pMethods = pFd->pMethods; + pFd->pMethods = &recover_methods; + } +} +static void recoverUninstallWrapper(sqlite3_recover *p){ + if( recover_g.pMethods ){ + sqlite3_file *pFd = 0; + sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); + assert( pFd ); + pFd->pMethods = recover_g.pMethods; + recover_g.pMethods = 0; + } +} + static void recoverStep(sqlite3_recover *p){ assert( p && p->errCode==SQLITE_OK ); switch( p->eState ){ @@ -1853,6 +2114,9 @@ static void recoverStep(sqlite3_recover *p){ ** user functions with the new handle. */ recoverOpenOutput(p); + sqlite3_mutex_enter( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + recoverInstallWrapper(p); + /* Open transactions on both the input and output databases. */ recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); recoverExec(p, p->dbIn, "BEGIN"); @@ -1860,8 +2124,11 @@ static void recoverStep(sqlite3_recover *p){ recoverExec(p, p->dbOut, "BEGIN"); recoverCacheSchema(p); - recoverWriteSchema1(p); + recoverUninstallWrapper(p); + sqlite3_mutex_leave( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + + recoverWriteSchema1(p); p->eState = RECOVER_STATE_WRITING; break; diff --git a/manifest b/manifest index 66d4798e47..d3de1f076b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\simplementation\sof\srecovery\sto\sbreak\srecovery\sof\sthe\slost_and_found\stable\sinto\smultiple\ssteps. -D 2022-09-26T20:51:16.327 +C Better\shandling\sfor\sdatabases\swith\scorrupt\sheaders. +D 2022-10-04T19:43:12.989 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -398,7 +398,7 @@ F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e9 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 5aad3f4e569e8637abdadfeb748ec1b99198d6d45bf983c6b168a527f5891b7f +F ext/recover/sqlite3recover.c 9c4d742223c519ef7322c51cb8997ca1046ca1971c06b617d455ec0bf0a9b390 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ad9dba9d1eae786575c7f31e34b342b6f5b26e719bbe27b61609cad8cfd0a505 -R d7188e5461609c695208ab621d6c4574 +P adedfd040bb2d1c2dcda0b91c6073f12953e45d9b860a8f217d990e861d66dfb +R bababa544bff189a544b5800da5cb030 U dan -Z d49040b82cb2efa369225859bd08a957 +Z 620b6c10a3ee6d6290114869c2142ea0 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index d70afc5526..f42c857e04 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adedfd040bb2d1c2dcda0b91c6073f12953e45d9b860a8f217d990e861d66dfb \ No newline at end of file +17f68d803685405d880025134c71bac01077962d677f2902a608952d0b9cdb5a \ No newline at end of file From adc686043e09883de2d2e4a46ca1a35b4c1edbc8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Oct 2022 16:58:28 +0000 Subject: [PATCH 34/42] Improve handling of corruption in the sqlite_schema table b-tree structure. FossilOrigin-Name: 2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b --- ext/recover/recovercorrupt.test | 12 ++++-- ext/recover/sqlite3recover.c | 75 +++++++++++++++++++++++++++------ manifest | 16 +++---- manifest.uuid | 2 +- 4 files changed, 79 insertions(+), 26 deletions(-) diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test index 54a94e1b0c..290ba63b0c 100644 --- a/ext/recover/recovercorrupt.test +++ b/ext/recover/recovercorrupt.test @@ -18,6 +18,8 @@ source $testdir/tester.tcl set testprefix recovercorrupt +database_may_be_corrupt + do_execsql_test 1.0 { PRAGMA page_size = 512; CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); @@ -46,19 +48,21 @@ proc toggle_bit {blob bit} { db_save_and_close -for {set ii 200} {$ii < 10000} {incr ii} { +for {set ii 0} {$ii < 10000} {incr ii} { db_restore_and_reopen db func toggle_bit toggle_bit - set pg [expr {($ii / 512)+1}] - set byte [expr {$ii % 512}] + set bitsperpage [expr 512*8] + + set pg [expr {($ii / $bitsperpage)+1}] + set byte [expr {$ii % $bitsperpage}] db eval { UPDATE sqlite_dbpage SET data = toggle_bit(data, $byte) WHERE pgno=$pg } - do_test 1.2.$ii { set R [sqlite3_recover_init db main test.db2] $R config lostandfound lost_and_found $R run + do_test 1.2.$ii { $R finish } {} } diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index dd294dcc7b..ea08d141cc 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -192,6 +192,10 @@ struct sqlite3_recover { int bRecoverRowid; /* SQLITE_RECOVER_ROWIDS setting */ int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ + int pgsz; + u8 *pPage1Disk; + u8 *pPage1Cache; + /* Error code and error message */ int errCode; /* For sqlite3_recover_errcode() */ char *zErrMsg; /* For sqlite3_recover_errmsg() */ @@ -248,6 +252,7 @@ struct sqlite3_recover { typedef struct RecoverGlobal RecoverGlobal; struct RecoverGlobal { const sqlite3_io_methods *pMethods; + sqlite3_recover *p; }; static RecoverGlobal recover_g; @@ -661,8 +666,20 @@ static void recoverGetPage( if( pStmt ){ sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int bDone = 0; assert( p->errCode==SQLITE_OK ); - sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + if( pgno==1 ){ + const u8 *aPg = sqlite3_column_blob(pStmt, 0); + int nPg = sqlite3_column_bytes(pStmt, 0); + if( nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + sqlite3_result_blob(pCtx, p->pPage1Disk, nPg, SQLITE_STATIC); + bDone = 1; + } + } + + if( !bDone ){ + sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + } } recoverReset(p, pStmt); } @@ -1914,7 +1931,7 @@ static void recoverPutU16(u8 *a, u32 v){ a[0] = (v>>8) & 0x00FF; a[1] = (v>>0) & 0x00FF; } -static u32 recoverPutU32(u8 *a, u32 v){ +static void recoverPutU32(u8 *a, u32 v){ a[0] = (v>>24) & 0x00FF; a[1] = (v>>16) & 0x00FF; a[2] = (v>>8) & 0x00FF; @@ -1926,7 +1943,10 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( pFd->pMethods==&recover_methods ){ pFd->pMethods = recover_g.pMethods; rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff); - if( rc==SQLITE_OK && iOff==0 && nByte>=100 ){ + if( nByte==16 ){ + sqlite3_randomness(16, aBuf); + }else + if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){ /* Ensure that the database has a valid header file. The only fields ** that really matter to recovery are: ** @@ -1942,9 +1962,12 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ ** We also try to preserve the auto-vacuum, incr-value, user-version ** and application-id fields - all 32 bit quantities at offsets ** 52, 60, 64 and 68. All other fields are set to known good values. + ** + ** Byte offset 105 should also contain the page-size as a 16-bit + ** integer. */ const int aPreserve[] = {32, 36, 52, 60, 64, 68}; - u8 aHdr[100] = { + u8 aHdr[108] = { 0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00, 0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20, @@ -1957,7 +1980,9 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x2e, 0x5b, 0x30 + 0x00, 0x2e, 0x5b, 0x30, + + 0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00 }; u8 *a = (u8*)aBuf; @@ -1966,23 +1991,44 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ u32 dbsz = 0; i64 dbFileSize = 0; int ii; + sqlite3_recover *p = recover_g.p; if( pgsz==0x01 ) pgsz = 65536; rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); - dbsz = dbFileSize / pgsz; + if( pgsz ){ + dbsz = dbFileSize / pgsz; + } if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){ enc = SQLITE_UTF8; } - recoverPutU32(&aHdr[28], dbsz); - recoverPutU32(&aHdr[56], enc); - if( pgsz==65536 ) pgsz = 1; - recoverPutU16(&aHdr[16], pgsz); - for(ii=0; iipPage1Cache); + p->pPage1Cache = 0; + p->pPage1Disk = 0; + + p->pgsz = nByte; + p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2); + if( p->pPage1Cache ){ + p->pPage1Disk = &p->pPage1Cache[nByte]; + memcpy(p->pPage1Disk, aBuf, nByte); + + recoverPutU32(&aHdr[28], dbsz); + recoverPutU32(&aHdr[56], enc); + recoverPutU16(&aHdr[105], pgsz); + if( pgsz==65536 ) pgsz = 1; + recoverPutU16(&aHdr[16], pgsz); + for(ii=0; iipPage1Cache, aBuf, nByte); + }else{ + rc = p->errCode; } - memcpy(aBuf, aHdr, sizeof(aHdr)); + } pFd->pMethods = &recover_methods; }else{ @@ -2088,6 +2134,7 @@ static void recoverInstallWrapper(sqlite3_recover *p){ sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); if( pFd ){ recover_g.pMethods = pFd->pMethods; + recover_g.p = p; pFd->pMethods = &recover_methods; } } @@ -2098,6 +2145,7 @@ static void recoverUninstallWrapper(sqlite3_recover *p){ assert( pFd ); pFd->pMethods = recover_g.pMethods; recover_g.pMethods = 0; + recover_g.p = 0; } } @@ -2384,6 +2432,7 @@ int sqlite3_recover_finish(sqlite3_recover *p){ sqlite3_free(p->zErrMsg); sqlite3_free(p->zStateDb); sqlite3_free(p->zLostAndFound); + sqlite3_free(p->pPage1Cache); sqlite3_free(p); } return rc; diff --git a/manifest b/manifest index aaf7917e93..c168d21c56 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2022-10-04T19:47:17.725 +C Improve\shandling\sof\scorruption\sin\sthe\ssqlite_schema\stable\sb-tree\sstructure. +D 2022-10-05T16:58:28.898 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -389,7 +389,7 @@ F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f5974282 F ext/recover/recover1.test 2b0f048ba572c3ca4e4235b632bf75702975a9656146eab766f6c75861f946a0 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 -F ext/recover/recovercorrupt.test e3f3cbe0162ba681518aac9ea0ae8119f32ac93fb0900b5f09b6318966108e54 +F ext/recover/recovercorrupt.test 69af3d68aedc2cf1c3c41dbd6afb45b3cebadb57796d513550b5fd1e2a8b3fba F ext/recover/recovercorrupt2.test a131d8005337c092e2dfa3b84909ed67ae82d22399a8cfb8c984b2939969ca42 F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b @@ -397,7 +397,7 @@ F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e9 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 9c4d742223c519ef7322c51cb8997ca1046ca1971c06b617d455ec0bf0a9b390 +F ext/recover/sqlite3recover.c c40381677e18f841be9e878591bf68cab043e99b408374561179829d0b041095 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -534,7 +534,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 w config.h.in +F sqlite_cfg.h.in baf2e409c63d4e7a765e17769b6ff17c5a82bbd9cbf1e284fd2e4cefaff3fcf2 F src/alter.c 0390ca1d69ec3626cfa9f153114b7ab233e6b2bada6a9eb91361ed385fe90deb F src/analyze.c aabdf3769c7fd9954a8ec508eb7041ae174b66f88d12c47199fabbea9a646467 F src/attach.c 4431f82f0247bf3aaf91589acafdff77d1882235c95407b36da1585c765fbbc8 @@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17f68d803685405d880025134c71bac01077962d677f2902a608952d0b9cdb5a 3efa811251d4510a074231a1e0b667783cc5f90777466b9dcae675cd9892b423 -R a7e16b1f572172e278e4e3a2a9c939da +P b0feecaa84b88f534edb49056cf7e92256f4a687c8b4c44e1d1186709a304378 +R c8e986f8000e60f0f5539ba1d6775390 U dan -Z 733ee477925392618f80fcf6c31db3f8 +Z 4a7b8684cf08c1b2ec449e50db6581b5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 55d6dceb0f..665a625d5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0feecaa84b88f534edb49056cf7e92256f4a687c8b4c44e1d1186709a304378 \ No newline at end of file +2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b \ No newline at end of file From 4df12ce5a8ae7c71444988fec15e7eb08e87a3da Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Oct 2022 17:20:12 +0000 Subject: [PATCH 35/42] Add code to determine the database page-size by searching for well-formed pages. FossilOrigin-Name: 0dbd0ccef532e076fa82c9fad4dc9fe632c0ef43e3483cc288cbb7eca336a9b9 --- ext/recover/recover1.test | 1 - ext/recover/sqlite3recover.c | 169 +++++++++++++++++++++++++++++++++-- manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 170 insertions(+), 16 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 58774e8434..aadc5b556e 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -86,7 +86,6 @@ do_execsql_test 1.0 { ) INSERT INTO t1 SELECT i*2, hex(randomblob(250)) FROM s; INSERT INTO t2 SELECT * FROM t1; - } do_recover_test 1 diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index ea08d141cc..3c70542989 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -1877,6 +1877,123 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->dbOut = 0; } +static u32 recoverGetU16(const u8 *a){ + return (((u32)a[0])<<8) + ((u32)a[1]); +} +static u32 recoverGetU32(const u8 *a){ + return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); +} +static int recoverGetVarint(const u8 *a, i64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (a[i]&0xff); + *pVal = v; + return 9; +} + +/* +** The second argument points to a buffer n bytes in size. If this buffer +** or a prefix thereof appears to contain a well-formed SQLite b-tree page, +** return the page-size in bytes. Otherwise, if the buffer does not +** appear to contain a well-formed b-tree page, return 0. +*/ +static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ + u8 *aUsed = aTmp; + int nFrag = 0; + int nActual = 0; + int iFree = 0; + int nCell = 0; /* Number of cells on page */ + int iCellOff = 0; /* Offset of cell array in page */ + int iContent = 0; + int eType = 0; + int ii = 0; + + eType = (int)a[0]; + if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0; + + iFree = (int)recoverGetU16(&a[1]); + nCell = (int)recoverGetU16(&a[3]); + iContent = (int)recoverGetU16(&a[5]); + if( iContent==0 ) iContent = 65536; + nFrag = (int)a[7]; + + if( iContent>n ) return 0; + + memset(aUsed, 0, n); + memset(aUsed, 0xFF, iContent); + + /* Follow the free-list. This is the same format for all b-tree pages. */ + if( iFree && iFree<=iContent ) return 0; + while( iFree ){ + int iNext = 0; + int nByte = 0; + if( iFree>(n-4) ) return 0; + iNext = recoverGetU16(&a[iFree]); + nByte = recoverGetU16(&a[iFree+2]); + if( iFree+nByte>n ) return 0; + if( iNextiContent ) return 0; + for(ii=0; iin ){ + return 0; + } + if( eType==0x05 || eType==0x02 ) nByte += 4; + nByte += recoverGetVarint(&a[iOff+nByte], &nPayload); + if( eType==0x0D ){ + i64 dummy = 0; + nByte += recoverGetVarint(&a[iOff+nByte], &dummy); + } + if( eType!=0x05 ){ + int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23); + int M = ((n-12)*32/255)-23; + int K = M+((nPayload-M)%(n-4)); + + if( nPayloadn ){ + return 0; + } + for(iByte=iOff; iByte<(iOff+nByte); iByte++){ + if( aUsed[iByte]!=0 ){ + return 0; + } + aUsed[iByte] = 0xFF; + } + } + + nActual = 0; + for(ii=0; iipMethods->xClose(pFd); } -static u32 recoverGetU16(const u8 *a){ - return (((u32)a[0])<<8) + ((u32)a[1]); -} -static u32 recoverGetU32(const u8 *a){ - return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); -} - static void recoverPutU16(u8 *a, u32 v){ a[0] = (v>>8) & 0x00FF; a[1] = (v>>0) & 0x00FF; @@ -1938,6 +2048,43 @@ static void recoverPutU32(u8 *a, u32 v){ a[3] = (v>>0) & 0x00FF; } +static int recoverVfsDetectPagesize(sqlite3_file *pFd, i64 nSz, u32 *pPgsz){ + int rc = SQLITE_OK; + const int nMin = 512; + const int nMax = 65536; + const int nMaxBlk = 4; + u32 pgsz = 0; + int iBlk = 0; + u8 *aPg = 0; + u8 *aTmp = 0; + + aPg = (u8*)sqlite3_malloc(2*nMax); + if( aPg==0 ) return SQLITE_NOMEM; + aTmp = &aPg[nMax]; + + for(iBlk=0; rc==SQLITE_OK && iBlk<((nSz+nMax-1)/nMax); iBlk++){ + int nByte = (nSz>=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOffpMethods==&recover_methods ){ @@ -1996,6 +2143,14 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( pgsz==0x01 ) pgsz = 65536; rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); + if( rc==SQLITE_OK ){ + u32 pgsz2 = 0; + rc = recoverVfsDetectPagesize(pFd, dbFileSize, &pgsz2); + if( pgsz2!=0 ){ + pgsz = pgsz2; + } + } + if( pgsz ){ dbsz = dbFileSize / pgsz; } diff --git a/manifest b/manifest index c168d21c56..4d01f35c31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\shandling\sof\scorruption\sin\sthe\ssqlite_schema\stable\sb-tree\sstructure. -D 2022-10-05T16:58:28.898 +C Add\scode\sto\sdetermine\sthe\sdatabase\spage-size\sby\ssearching\sfor\swell-formed\spages. +D 2022-10-06T17:20:12.045 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -386,7 +386,7 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test 2b0f048ba572c3ca4e4235b632bf75702975a9656146eab766f6c75861f946a0 +F ext/recover/recover1.test d9844f37b2e6da021816d699424d4aaf85712f50a9a207bf5f85f8ea7b48eb9b F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test 69af3d68aedc2cf1c3c41dbd6afb45b3cebadb57796d513550b5fd1e2a8b3fba @@ -397,7 +397,7 @@ F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e9 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c c40381677e18f841be9e878591bf68cab043e99b408374561179829d0b041095 +F ext/recover/sqlite3recover.c 1b68d2593b83a130cea794c8921ad7af564cfafd848ccc80e4b9a0e893942c51 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b0feecaa84b88f534edb49056cf7e92256f4a687c8b4c44e1d1186709a304378 -R c8e986f8000e60f0f5539ba1d6775390 +P 2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b +R 428f4add1572ea8a473b960c2888f37e U dan -Z 4a7b8684cf08c1b2ec449e50db6581b5 +Z 0bbefdeff86a57418ac328248f517762 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 665a625d5c..3fa25f41b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b \ No newline at end of file +0dbd0ccef532e076fa82c9fad4dc9fe632c0ef43e3483cc288cbb7eca336a9b9 \ No newline at end of file From 88a0637f5781420a2811ba09dc20e3af2f4e7df0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 6 Oct 2022 21:00:23 +0000 Subject: [PATCH 36/42] Ensure the page-size, auto-vacuum and other settings are copied into the recovered database. FossilOrigin-Name: 078520f2825bcb82d01b30209409e9bf8b6c99084c192345df41c9f531d08f51 --- ext/recover/recover1.test | 14 ++ ext/recover/recoverslowidx.test | 12 +- ext/recover/sqlite3recover.c | 224 ++++++++++++++++++++------------ manifest | 16 +-- manifest.uuid | 2 +- 5 files changed, 172 insertions(+), 96 deletions(-) diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index aadc5b556e..99ddc1f4b3 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -36,6 +36,12 @@ proc compare_dbs {db1 db2} { foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { compare_result $db1 $db2 "SELECT * FROM $tbl" } + + compare_result $db1 $db2 "PRAGMA page_size" + compare_result $db1 $db2 "PRAGMA auto_vacuum" + compare_result $db1 $db2 "PRAGMA encoding" + compare_result $db1 $db2 "PRAGMA user_version" + compare_result $db1 $db2 "PRAGMA application_id" } proc do_recover_test {tn} { @@ -186,11 +192,19 @@ do_recover_test 10 #------------------------------------------------------------------------- reset_db do_execsql_test 11.1 { + PRAGMA page_size = 4096; PRAGMA encoding='utf16'; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 45; + PRAGMA application_id = 22; + CREATE TABLE u1(u, v); INSERT INTO u1 VALUES('edvin marton', 'bond'); INSERT INTO u1 VALUES(1, 4.0); } +do_execsql_test 11.1a { + PRAGMA auto_vacuum; +} {2} do_recover_test 11 diff --git a/ext/recover/recoverslowidx.test b/ext/recover/recoverslowidx.test index 838b771eaf..4dfb7e5f53 100644 --- a/ext/recover/recoverslowidx.test +++ b/ext/recover/recoverslowidx.test @@ -17,7 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] recover_common.tcl] source $testdir/tester.tcl -set testprefix recoverrowid +set testprefix recoverslowidx ifcapable !vtab { finish_test; return @@ -46,6 +46,11 @@ do_test 1.2 { } [list {*}{ {BEGIN} {PRAGMA writable_schema = on} + {PRAGMA encoding = 'UTF-8'} + {PRAGMA page_size = '1024'} + {PRAGMA auto_vacuum = '0'} + {PRAGMA user_version = '0'} + {PRAGMA application_id = '0'} {CREATE TABLE t1(a, b)} {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (2, 2, 2)} @@ -69,6 +74,11 @@ do_test 1.4 { } [list {*}{ {BEGIN} {PRAGMA writable_schema = on} + {PRAGMA encoding = 'UTF-8'} + {PRAGMA page_size = '1024'} + {PRAGMA auto_vacuum = '0'} + {PRAGMA user_version = '0'} + {PRAGMA application_id = '0'} {CREATE TABLE t1(a, b)} {CREATE INDEX i1 ON t1(a)} {INSERT OR IGNORE INTO 't1'(_rowid_, 'a', 'b') VALUES (1, 1, 1)} diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 3c70542989..4b61d6b3ec 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -801,6 +801,120 @@ static void recoverEscapeCrnl( sqlite3_result_value(context, argv[0]); } +/* +** This function is a no-op if recover handle p already contains an error +** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in +** this case. +** +** Otherwise, attempt to populate temporary table "recovery.schema" with the +** parts of the database schema that can be extracted from the input database. +** +** If no error occurs, SQLITE_OK is returned. Otherwise, an error code +** and error message are left in the recover handle and a copy of the +** error code returned. It is not considered an error if part of all of +** the database schema cannot be recovered due to corruption. +*/ +static int recoverCacheSchema(sqlite3_recover *p){ + return recoverExec(p, p->dbOut, + "WITH RECURSIVE pages(p) AS (" + " SELECT 1" + " UNION" + " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" + ")" + "INSERT INTO recovery.schema SELECT" + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" + " SELECT p FROM pages" + ") GROUP BY pgno, cell" + ); +} + +/* +** If this recover handle is not in SQL callback mode (i.e. was not created +** using sqlite3_recover_init_sql()) of if an error has already occurred, +** this function is a no-op. Otherwise, issue a callback with SQL statement +** zSql as the parameter. +** +** If the callback returns non-zero, set the recover handle error code to +** the value returned (so that the caller will abandon processing). +*/ +static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ + if( p->errCode==SQLITE_OK && p->xSql ){ + int res = p->xSql(p->pSqlCtx, zSql); + if( res ){ + recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); + } + } +} + +/* +** Transfer the following settings from the input database to the output +** database: +** +** + page-size, +** + auto-vacuum settings, +** + database encoding, +** + user-version (PRAGMA user_version), and +** + application-id (PRAGMA application_id), and +*/ +static void recoverTransferSettings(sqlite3_recover *p){ + const char *aPragma[] = { + "encoding", + "page_size", + "auto_vacuum", + "user_version", + "application_id" + }; + int ii; + + /* Truncate the output database to 0 pages in size. This is done by + ** opening a new, empty, temp db, then using the backup API to clobber + ** any existing output db with a copy of it. */ + if( p->errCode==SQLITE_OK ){ + sqlite3 *db2 = 0; + int rc = sqlite3_open("", &db2); + if( rc!=SQLITE_OK ){ + recoverDbError(p, db2); + return; + } + + for(ii=0; iidbIn, "PRAGMA %Q.%s", p->zDb, zPrag); + if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){ + const char *zArg = (const char*)sqlite3_column_text(p1, 0); + char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg); + recoverSqlCallback(p, z2); + recoverExec(p, db2, z2); + sqlite3_free(z2); + if( zArg==0 ){ + recoverError(p, SQLITE_NOMEM, 0); + } + } + recoverFinalize(p, p1); + } + recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;"); + + if( p->errCode==SQLITE_OK ){ + sqlite3 *db = p->dbOut; + sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); + if( pBackup ){ + sqlite3_backup_step(pBackup, -1); + p->errCode = sqlite3_backup_finish(pBackup); + }else{ + recoverDbError(p, db); + } + } + + sqlite3_close(db2); + } +} + /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in @@ -833,19 +947,8 @@ static int recoverOpenOutput(sqlite3_recover *p){ assert( p->dbOut==0 ); - if( p->errCode==SQLITE_OK ){ - if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ - recoverDbError(p, db); - }else{ - char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); - recoverExec(p, db, zSql); - recoverExec(p, db, - "PRAGMA writable_schema = 1;" - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" - "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" - ); - sqlite3_free(zSql); - } + if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){ + recoverDbError(p, db); } /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules. @@ -863,62 +966,22 @@ static int recoverOpenOutput(sqlite3_recover *p){ ); } - /* Truncate the output database to 0 pages in size. This is done by - ** opening a new, empty, temp db, then using the backup API to clobber - ** any existing output db with a copy of it. */ - if( p->errCode==SQLITE_OK ){ - sqlite3 *db2 = 0; - int rc = sqlite3_open("", &db2); - if( rc==SQLITE_OK ){ - sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main"); - if( pBackup ){ - sqlite3_backup_step(pBackup, -1); - p->errCode = sqlite3_backup_finish(pBackup); - }else{ - recoverDbError(p, db); - } - }else{ - recoverDbError(p, db2); - } - sqlite3_close(db2); - } - p->dbOut = db; return p->errCode; } -/* -** This function is a no-op if recover handle p already contains an error -** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in -** this case. -** -** Otherwise, attempt to populate temporary table "recovery.schema" with the -** parts of the database schema that can be extracted from the input database. -** -** If no error occurs, SQLITE_OK is returned. Otherwise, an error code -** and error message are left in the recover handle and a copy of the -** error code returned. It is not considered an error if part of all of -** the database schema cannot be recovered due to corruption. -*/ -static int recoverCacheSchema(sqlite3_recover *p){ - return recoverExec(p, p->dbOut, - "WITH RECURSIVE pages(p) AS (" - " SELECT 1" - " UNION" - " SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p" - ")" - "INSERT INTO recovery.schema SELECT" - " max(CASE WHEN field=0 THEN value ELSE NULL END)," - " max(CASE WHEN field=1 THEN value ELSE NULL END)," - " max(CASE WHEN field=2 THEN value ELSE NULL END)," - " max(CASE WHEN field=3 THEN value ELSE NULL END)," - " max(CASE WHEN field=4 THEN value ELSE NULL END)" - "FROM sqlite_dbdata('getpage()') WHERE pgno IN (" - " SELECT p FROM pages" - ") GROUP BY pgno, cell" +static void recoverOpenRecovery(sqlite3_recover *p){ + char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); + recoverExec(p, p->dbOut, zSql); + recoverExec(p, p->dbOut, + "PRAGMA writable_schema = 1;" + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" ); + sqlite3_free(zSql); } + /* ** This function is a no-op if recover handle p already contains an error ** (if p->errCode!=SQLITE_OK). @@ -1022,24 +1085,6 @@ static void recoverAddTable( } } -/* -** If this recover handle is not in SQL callback mode (i.e. was not created -** using sqlite3_recover_init_sql()) of if an error has already occurred, -** this function is a no-op. Otherwise, issue a callback with SQL statement -** zSql as the parameter. -** -** If the callback returns non-zero, set the recover handle error code to -** the value returned (so that the caller will abandon processing). -*/ -static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){ - if( p->errCode==SQLITE_OK && p->xSql ){ - int res = p->xSql(p->pSqlCtx, zSql); - if( res ){ - recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res); - } - } -} - /* ** This function is called after recoverCacheSchema() has cached those parts ** of the input database schema that could be recovered in temporary table @@ -2057,12 +2102,16 @@ static int recoverVfsDetectPagesize(sqlite3_file *pFd, i64 nSz, u32 *pPgsz){ int iBlk = 0; u8 *aPg = 0; u8 *aTmp = 0; + int nBlk = 0; aPg = (u8*)sqlite3_malloc(2*nMax); if( aPg==0 ) return SQLITE_NOMEM; aTmp = &aPg[nMax]; - for(iBlk=0; rc==SQLITE_OK && iBlk<((nSz+nMax-1)/nMax); iBlk++){ + nBlk = (nSz+nMax-1)/nMax; + if( nBlk>nMaxBlk ) nBlk = nMaxBlk; + + for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); memset(aPg, 0, nMax); rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); @@ -2313,24 +2362,27 @@ static void recoverStep(sqlite3_recover *p){ recoverSqlCallback(p, "BEGIN"); recoverSqlCallback(p, "PRAGMA writable_schema = on"); + sqlite3_mutex_enter( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + recoverInstallWrapper(p); + /* Open the output database. And register required virtual tables and ** user functions with the new handle. */ recoverOpenOutput(p); - sqlite3_mutex_enter( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); - recoverInstallWrapper(p); - /* Open transactions on both the input and output databases. */ recoverExec(p, p->dbIn, "PRAGMA writable_schema = on"); recoverExec(p, p->dbIn, "BEGIN"); if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1; - recoverExec(p, p->dbOut, "BEGIN"); - + recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema"); + recoverTransferSettings(p); + recoverOpenRecovery(p); recoverCacheSchema(p); recoverUninstallWrapper(p); sqlite3_mutex_leave( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + recoverExec(p, p->dbOut, "BEGIN"); + recoverWriteSchema1(p); p->eState = RECOVER_STATE_WRITING; break; diff --git a/manifest b/manifest index 4d01f35c31..bb0cc53623 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scode\sto\sdetermine\sthe\sdatabase\spage-size\sby\ssearching\sfor\swell-formed\spages. -D 2022-10-06T17:20:12.045 +C Ensure\sthe\spage-size,\sauto-vacuum\sand\sother\ssettings\sare\scopied\sinto\sthe\srecovered\sdatabase. +D 2022-10-06T21:00:23.727 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -386,7 +386,7 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a -F ext/recover/recover1.test d9844f37b2e6da021816d699424d4aaf85712f50a9a207bf5f85f8ea7b48eb9b +F ext/recover/recover1.test 6b59ef31e3c0beef8b66e112d50af160e6f7ad8df95bee10192a9e44169d03ec F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 F ext/recover/recovercorrupt.test 69af3d68aedc2cf1c3c41dbd6afb45b3cebadb57796d513550b5fd1e2a8b3fba @@ -395,9 +395,9 @@ F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 -F ext/recover/recoverslowidx.test 7e3889e50758c614b9b3e75187eeeea425c0ca8a2387441fc19ae749a96a7949 +F ext/recover/recoverslowidx.test f356bb9fba7ffd6fc50e045e419464f0129ac6e24decf6e919584f79c3493727 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 1b68d2593b83a130cea794c8921ad7af564cfafd848ccc80e4b9a0e893942c51 +F ext/recover/sqlite3recover.c dcd00bbb95150b3d2a1ae75abd194fa863e4ed86971a3589da226c716fb5a6c0 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2014,8 +2014,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2be0dba12fb9b98cd9632f1ff3f30aeb29661a2e121b68c2f3335617b027797b -R 428f4add1572ea8a473b960c2888f37e +P 0dbd0ccef532e076fa82c9fad4dc9fe632c0ef43e3483cc288cbb7eca336a9b9 +R c47f69833d79cdde413f6662a615cd06 U dan -Z 0bbefdeff86a57418ac328248f517762 +Z b397112aade8ffe2182cfeaa2f331f80 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 3fa25f41b4..5806dd4688 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dbd0ccef532e076fa82c9fad4dc9fe632c0ef43e3483cc288cbb7eca336a9b9 \ No newline at end of file +078520f2825bcb82d01b30209409e9bf8b6c99084c192345df41c9f531d08f51 \ No newline at end of file From 249efd04d783cdc2b5417d8d1f26ce61d0307b77 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Oct 2022 20:06:17 +0000 Subject: [PATCH 37/42] Add tests for detecting page size of databases. FossilOrigin-Name: 31f9e2369fcf59032b7c4c9f5bfc85e7ef7c174003b0b9e2757dad5a4c79b370 --- ext/recover/recoverpgsz.test | 103 +++++++++++++++++++++++++++++++++++ ext/recover/sqlite3recover.c | 21 ++++--- manifest | 13 +++-- manifest.uuid | 2 +- 4 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 ext/recover/recoverpgsz.test diff --git a/ext/recover/recoverpgsz.test b/ext/recover/recoverpgsz.test new file mode 100644 index 0000000000..71413b0ead --- /dev/null +++ b/ext/recover/recoverpgsz.test @@ -0,0 +1,103 @@ +# 2022 October 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source [file join [file dirname [info script]] recover_common.tcl] +source $testdir/tester.tcl + +db close +sqlite3_test_control_pending_byte 0x1000000 + +set testprefix recoverpgsz + +foreach {pgsz bOverflow} { + 512 0 1024 0 2048 0 4096 0 8192 0 16384 0 32768 0 65536 0 + 512 1 1024 1 2048 1 4096 1 8192 1 16384 1 32768 1 65536 1 +} { + reset_db + execsql "PRAGMA page_size = $pgsz" + do_execsql_test 1.$pgsz.$bOverflow.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(b, a, c); + INSERT INTO t1(a, b) VALUES(1, 2), (3, 4), (5, 6); + DELETE FROM t1 WHERE a=3; + } + if {$bOverflow} { + do_execsql_test 1.$pgsz.$bOverflow.1a { + UPDATE t1 SET c = randomblob(100000); + } + } + db close + + + set fd [open test.db] + fconfigure $fd -encoding binary -translation binary + seek $fd $pgsz + set pg1 [read $fd $pgsz] + set pg2 [read $fd $pgsz] + close $fd + + set fd2 [open test.db2 w] + fconfigure $fd2 -encoding binary -translation binary + seek $fd2 $pgsz + puts -nonewline $fd2 $pg1 + close $fd2 + + sqlite3 db2 test.db2 + do_test 1.$pgsz.$bOverflow.2 { + set R [sqlite3_recover_init db2 main test.db3] + $R run + $R finish + } {} + + sqlite3 db3 test.db3 + do_test 1.$pgsz.$bOverflow.3 { + db3 eval { SELECT * FROM sqlite_schema } + db3 eval { PRAGMA page_size } + } $pgsz + + db2 close + db3 close + + forcedelete test.db3 + forcedelete test.db2 + + set fd2 [open test.db2 w] + fconfigure $fd2 -encoding binary -translation binary + seek $fd2 $pgsz + puts -nonewline $fd2 $pg2 + close $fd2 + + sqlite3 db2 test.db2 + do_test 1.$pgsz.$bOverflow.4 { + set R [sqlite3_recover_init db2 main test.db3] + $R run + $R finish + } {} + + sqlite3 db3 test.db3 + do_test 1.$pgsz.$bOverflow.5 { + db3 eval { SELECT * FROM sqlite_schema } + db3 eval { PRAGMA page_size } + } $pgsz + + db2 close + db3 close +} + + +finish_test + + + diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 4b61d6b3ec..6e5db516c9 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -193,6 +193,7 @@ struct sqlite3_recover { int bSlowIndexes; /* SQLITE_RECOVER_SLOWINDEXES setting */ int pgsz; + int detected_pgsz; u8 *pPage1Disk; u8 *pPage1Cache; @@ -1980,7 +1981,7 @@ static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){ iNext = recoverGetU16(&a[iFree]); nByte = recoverGetU16(&a[iFree+2]); if( iFree+nByte>n ) return 0; - if( iNext>0) & 0x00FF; } -static int recoverVfsDetectPagesize(sqlite3_file *pFd, i64 nSz, u32 *pPgsz){ +static int recoverVfsDetectPagesize( + sqlite3_recover *p, + sqlite3_file *pFd, + i64 nSz +){ int rc = SQLITE_OK; const int nMin = 512; const int nMax = 65536; @@ -2129,7 +2134,7 @@ static int recoverVfsDetectPagesize(sqlite3_file *pFd, i64 nSz, u32 *pPgsz){ } } - *pPgsz = pgsz; + p->detected_pgsz = pgsz; sqlite3_free(aPg); return rc; } @@ -2192,12 +2197,12 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( pgsz==0x01 ) pgsz = 65536; rc = pFd->pMethods->xFileSize(pFd, &dbFileSize); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && p->detected_pgsz==0 ){ u32 pgsz2 = 0; - rc = recoverVfsDetectPagesize(pFd, dbFileSize, &pgsz2); - if( pgsz2!=0 ){ - pgsz = pgsz2; - } + rc = recoverVfsDetectPagesize(p, pFd, dbFileSize); + } + if( p->detected_pgsz ){ + pgsz = p->detected_pgsz; } if( pgsz ){ diff --git a/manifest b/manifest index bb0cc53623..9f328fce8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthe\spage-size,\sauto-vacuum\sand\sother\ssettings\sare\scopied\sinto\sthe\srecovered\sdatabase. -D 2022-10-06T21:00:23.727 +C Add\stests\sfor\sdetecting\spage\ssize\sof\sdatabases. +D 2022-10-13T20:06:17.168 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -394,10 +394,11 @@ F ext/recover/recovercorrupt2.test a131d8005337c092e2dfa3b84909ed67ae82d22399a8c F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b F ext/recover/recoverold.test 46e9d99b595fac583d4c67f74d7d89c20a435c752ef6eeb3e918b599940c88e0 +F ext/recover/recoverpgsz.test 93e970eab05e4e89f8fd6b1bd23f9ec137ea09857e66ba0d4d27a83cd1ba4a89 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test f356bb9fba7ffd6fc50e045e419464f0129ac6e24decf6e919584f79c3493727 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c dcd00bbb95150b3d2a1ae75abd194fa863e4ed86971a3589da226c716fb5a6c0 +F ext/recover/sqlite3recover.c 2e9244bcc552f84bb60ea67060b793af2eb050bcbb9c8caed13510308e2039fb F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -2014,8 +2015,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0dbd0ccef532e076fa82c9fad4dc9fe632c0ef43e3483cc288cbb7eca336a9b9 -R c47f69833d79cdde413f6662a615cd06 +P 078520f2825bcb82d01b30209409e9bf8b6c99084c192345df41c9f531d08f51 +R c9070f986a9313d4a9d177b6562a0478 U dan -Z b397112aade8ffe2182cfeaa2f331f80 +Z 7bf274fb579b6c3146f808450c95f374 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 5806dd4688..6462e520e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -078520f2825bcb82d01b30209409e9bf8b6c99084c192345df41c9f531d08f51 \ No newline at end of file +31f9e2369fcf59032b7c4c9f5bfc85e7ef7c174003b0b9e2757dad5a4c79b370 \ No newline at end of file From 3fa10f82f4c14a30574d64556e2a80e68be6735d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Oct 2022 15:46:19 +0000 Subject: [PATCH 38/42] Handle recovering databases with reserved bytes at the end of each page. FossilOrigin-Name: 52d0235ef3f21683daadf2343514f7b95a67c3589266f75c1e00e04f11dc6214 --- ext/misc/cksumvfs.c | 2 +- ext/recover/recover1.test | 24 +++++++++++++++ ext/recover/sqlite3recover.c | 60 ++++++++++++++++++++---------------- manifest | 16 +++++----- manifest.uuid | 2 +- 5 files changed, 67 insertions(+), 37 deletions(-) diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c index 8c340889fe..e7c2c9d5c0 100644 --- a/ext/misc/cksumvfs.c +++ b/ext/misc/cksumvfs.c @@ -47,7 +47,7 @@ ** ** sqlite3 *db; ** sqlite3_open(":memory:", &db); -** sqlite3_load_extention(db, "./cksumvfs"); +** sqlite3_load_extension(db, "./cksumvfs"); ** sqlite3_close(db); ** ** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and diff --git a/ext/recover/recover1.test b/ext/recover/recover1.test index 99ddc1f4b3..ed921b7e91 100644 --- a/ext/recover/recover1.test +++ b/ext/recover/recover1.test @@ -222,5 +222,29 @@ do_test 12.3 { $R finish } {} + + +#------------------------------------------------------------------------- +reset_db +file_control_reservebytes db 16 +do_execsql_test 12.1 { + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 45; + PRAGMA application_id = 22; + + CREATE TABLE u1(u, v); + CREATE UNIQUE INDEX i1 ON u1(u, v); + INSERT INTO u1 VALUES(1, 2), (3, 4); + + CREATE TABLE u2(u, v); + CREATE UNIQUE INDEX i2 ON u1(u, v); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(500)), hex(randomblob(1000))); + INSERT INTO u2 VALUES(hex(randomblob(50000)), hex(randomblob(20000))); +} + +do_recover_test 12 + finish_test diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 6e5db516c9..70f3386924 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -194,6 +194,7 @@ struct sqlite3_recover { int pgsz; int detected_pgsz; + int nReserve; u8 *pPage1Disk; u8 *pPage1Cache; @@ -667,20 +668,13 @@ static void recoverGetPage( if( pStmt ){ sqlite3_bind_int64(pStmt, 1, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - int bDone = 0; assert( p->errCode==SQLITE_OK ); - if( pgno==1 ){ - const u8 *aPg = sqlite3_column_blob(pStmt, 0); - int nPg = sqlite3_column_bytes(pStmt, 0); - if( nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ - sqlite3_result_blob(pCtx, p->pPage1Disk, nPg, SQLITE_STATIC); - bDone = 1; - } - } - - if( !bDone ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pStmt, 0)); + const u8 *aPg = sqlite3_column_blob(pStmt, 0); + int nPg = sqlite3_column_bytes(pStmt, 0); + if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){ + aPg = p->pPage1Disk; } + sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT); } recoverReset(p, pStmt); } @@ -2097,6 +2091,7 @@ static void recoverPutU32(u8 *a, u32 v){ static int recoverVfsDetectPagesize( sqlite3_recover *p, sqlite3_file *pFd, + u32 nReserve, /* Possible nReserve value */ i64 nSz ){ int rc = SQLITE_OK; @@ -2116,23 +2111,31 @@ static int recoverVfsDetectPagesize( nBlk = (nSz+nMax-1)/nMax; if( nBlk>nMaxBlk ) nBlk = nMaxBlk; - for(iBlk=0; rc==SQLITE_OK && iBlk=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); - memset(aPg, 0, nMax); - rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); - if( rc==SQLITE_OK ){ - int pgsz2; - for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ - int iOff; - for(iOff=0; iOff=((iBlk+1)*nMax)) ? nMax : (nSz % nMax); + memset(aPg, 0, nMax); + rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax); + if( rc==SQLITE_OK ){ + int pgsz2; + for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){ + int iOff; + for(iOff=0; iOffp->detected_pgsz ){ + p->detected_pgsz = pgsz; + p->nReserve = nReserve; + } + if( nReserve==0 ) break; + nReserve = 0; + }while( 1 ); p->detected_pgsz = pgsz; sqlite3_free(aPg); @@ -2188,6 +2191,7 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ u8 *a = (u8*)aBuf; u32 pgsz = recoverGetU16(&a[16]); + u32 nReserve = a[20]; u32 enc = recoverGetU32(&a[56]); u32 dbsz = 0; i64 dbFileSize = 0; @@ -2199,10 +2203,11 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ if( rc==SQLITE_OK && p->detected_pgsz==0 ){ u32 pgsz2 = 0; - rc = recoverVfsDetectPagesize(p, pFd, dbFileSize); + rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize); } if( p->detected_pgsz ){ pgsz = p->detected_pgsz; + nReserve = p->nReserve; } if( pgsz ){ @@ -2224,9 +2229,10 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ recoverPutU32(&aHdr[28], dbsz); recoverPutU32(&aHdr[56], enc); - recoverPutU16(&aHdr[105], pgsz); + recoverPutU16(&aHdr[105], pgsz-nReserve); if( pgsz==65536 ) pgsz = 1; recoverPutU16(&aHdr[16], pgsz); + aHdr[20] = nReserve; for(ii=0; ii Date: Sat, 15 Oct 2022 18:26:30 +0000 Subject: [PATCH 39/42] Add missing comments and fix other code issues. FossilOrigin-Name: 8ed4e4a80d8104d20b211a0b3e299abdb955cabdd36979bf6aae54266dc9778d --- ext/recover/sqlite3recover.c | 162 +++++++++++++++++++++++++++++++++-- manifest | 18 ++-- manifest.uuid | 2 +- 3 files changed, 164 insertions(+), 18 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 70f3386924..74b7f8a369 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -141,6 +141,11 @@ struct RecoverBitmap { u32 aElem[1]; /* Array of 32-bit bitmasks */ }; +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data for tables identified in the recovered schema (state +** RECOVER_STATE_WRITING). +*/ typedef struct RecoverStateW1 RecoverStateW1; struct RecoverStateW1 { sqlite3_stmt *pTbls; @@ -158,6 +163,11 @@ struct RecoverStateW1 { int iPrevCell; }; +/* +** State variables (part of the sqlite3_recover structure) used while +** recovering data destined for the lost and found table (states +** RECOVER_STATE_LOSTANDFOUND[123]). +*/ typedef struct RecoverStateLAF RecoverStateLAF; struct RecoverStateLAF { RecoverBitmap *pUsed; @@ -208,7 +218,7 @@ struct sqlite3_recover { /* Variables used with eState==RECOVER_STATE_WRITING */ RecoverStateW1 w1; - /* Variables used with states RECOVER_STATE_LOSTANDFOUND* */ + /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */ RecoverStateLAF laf; /* Fields used within sqlite3_recover_run() */ @@ -265,7 +275,6 @@ static RecoverGlobal recover_g; #define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2 - /* ** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid). */ @@ -965,6 +974,11 @@ static int recoverOpenOutput(sqlite3_recover *p){ return p->errCode; } +/* +** Attach the auxiliary database 'recovery' to the output database handle. +** This temporary database is used during the recovery process and then +** discarded. +*/ static void recoverOpenRecovery(sqlite3_recover *p){ char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb); recoverExec(p, p->dbOut, zSql); @@ -1424,6 +1438,15 @@ static sqlite3_stmt *recoverLostAndFoundInsert( return pRet; } +/* +** Input database page iPg contains data that will be written to the +** lost-and-found table of the output database. This function attempts +** to identify the root page of the tree that page iPg belonged to. +** If successful, it sets output variable (*piRoot) to the page number +** of the root page and returns SQLITE_OK. Otherwise, if an error occurs, +** an SQLite error code is returned and the final value of *piRoot +** undefined. +*/ static int recoverLostAndFoundFindRoot( sqlite3_recover *p, i64 iPg, @@ -1454,6 +1477,10 @@ static int recoverLostAndFoundFindRoot( return p->errCode; } +/* +** Recover data from page iPage of the input database and write it to +** the lost-and-found table in the output database. +*/ static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ RecoverStateLAF *pLaf = &p->laf; sqlite3_value **apVal = pLaf->apVal; @@ -1526,6 +1553,13 @@ static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){ } } +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found +** table of the output database is populated with recovered data that can +** not be assigned to any recovered schema object. +*/ static int recoverLostAndFound3Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; if( p->errCode==SQLITE_OK ){ @@ -1549,6 +1583,12 @@ static int recoverLostAndFound3Step(sqlite3_recover *p){ return SQLITE_OK; } +/* +** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 +** state - during which the lost-and-found table of the output database +** is populated with recovered data that can not be assigned to any +** recovered schema object. +*/ static void recoverLostAndFound3Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; @@ -1578,6 +1618,11 @@ static void recoverLostAndFound3Init(sqlite3_recover *p){ } } +/* +** Initialize resources required in RECOVER_STATE_WRITING state - during which +** tables recovered from the schema of the input database are populated with +** recovered data. +*/ static int recoverWriteDataInit(sqlite3_recover *p){ RecoverStateW1 *p1 = &p->w1; RecoverTable *pTbl = 0; @@ -1635,6 +1680,12 @@ static void recoverWriteDataCleanup(sqlite3_recover *p){ memset(p1, 0, sizeof(*p1)); } +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_WRITING state - during which tables recovered from the +** schema of the input database are populated with recovered data. +*/ static int recoverWriteDataStep(sqlite3_recover *p){ RecoverStateW1 *p1 = &p->w1; sqlite3_stmt *pSel = p1->pSel; @@ -1756,6 +1807,11 @@ static int recoverWriteDataStep(sqlite3_recover *p){ return p->errCode; } +/* +** Initialize resources required by sqlite3_recover_step() in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ static void recoverLostAndFound1Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; sqlite3_stmt *pStmt = 0; @@ -1802,6 +1858,12 @@ static void recoverLostAndFound1Init(sqlite3_recover *p){ pLaf->pUsedPages = pStmt; } +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not +** already allocated to a recovered schema element is determined. +*/ static int recoverLostAndFound1Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; int rc = p->errCode; @@ -1819,6 +1881,11 @@ static int recoverLostAndFound1Step(sqlite3_recover *p){ return rc; } +/* +** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 +** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1 +** are sorted into sets that likely belonged to the same database tree. +*/ static void recoverLostAndFound2Init(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; @@ -1843,6 +1910,13 @@ static void recoverLostAndFound2Init(sqlite3_recover *p){ ); } +/* +** Perform one step (sqlite3_recover_step()) of work for the connection +** passed as the only argument, which is guaranteed to be in +** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified +** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged +** to the same database tree. +*/ static int recoverLostAndFound2Step(sqlite3_recover *p){ RecoverStateLAF *pLaf = &p->laf; if( p->errCode==SQLITE_OK ){ @@ -1872,6 +1946,10 @@ static int recoverLostAndFound2Step(sqlite3_recover *p){ return p->errCode; } +/* +** Free all resources allocated as part of sqlite3_recover_step() calls +** in one of the RECOVER_STATE_LOSTANDFOUND[123] states. +*/ static void recoverLostAndFoundCleanup(sqlite3_recover *p){ recoverBitmapFree(p->laf.pUsed); p->laf.pUsed = 0; @@ -1895,6 +1973,9 @@ static void recoverLostAndFoundCleanup(sqlite3_recover *p){ p->laf.apVal = 0; } +/* +** Free all resources allocated as part of sqlite3_recover_step() calls. +*/ static void recoverFinalCleanup(sqlite3_recover *p){ RecoverTable *pTab = 0; RecoverTable *pNext = 0; @@ -1917,12 +1998,26 @@ static void recoverFinalCleanup(sqlite3_recover *p){ p->dbOut = 0; } +/* +** Decode and return an unsigned 16-bit big-endian integer value from +** buffer a[]. +*/ static u32 recoverGetU16(const u8 *a){ return (((u32)a[0])<<8) + ((u32)a[1]); } + +/* +** Decode and return an unsigned 32-bit big-endian integer value from +** buffer a[]. +*/ static u32 recoverGetU32(const u8 *a){ return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]); } + +/* +** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal) +** and return the number of bytes consumed. +*/ static int recoverGetVarint(const u8 *a, i64 *pVal){ sqlite3_int64 v = 0; int i; @@ -2077,10 +2172,17 @@ static int recoverVfsClose(sqlite3_file *pFd){ return pFd->pMethods->xClose(pFd); } +/* +** Write value v to buffer a[] as a 16-bit big-endian unsigned integer. +*/ static void recoverPutU16(u8 *a, u32 v){ a[0] = (v>>8) & 0x00FF; a[1] = (v>>0) & 0x00FF; } + +/* +** Write value v to buffer a[] as a 32-bit big-endian unsigned integer. +*/ static void recoverPutU32(u8 *a, u32 v){ a[0] = (v>>24) & 0x00FF; a[1] = (v>>16) & 0x00FF; @@ -2088,11 +2190,25 @@ static void recoverPutU32(u8 *a, u32 v){ a[3] = (v>>0) & 0x00FF; } +/* +** Detect the page-size of the database opened by file-handle pFd by +** searching the first part of the file for a well-formed SQLite b-tree +** page. If parameter nReserve is non-zero, then as well as searching for +** a b-tree page with zero reserved bytes, this function searches for one +** with nReserve reserved bytes at the end of it. +** +** If successful, set variable p->detected_pgsz to the detected page-size +** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page +** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or, +** if an error occurs (e.g. an IO or OOM error), then an SQLite error code +** is returned. The final value of p->detected_pgsz is undefined in this +** case. +*/ static int recoverVfsDetectPagesize( - sqlite3_recover *p, - sqlite3_file *pFd, + sqlite3_recover *p, /* Recover handle */ + sqlite3_file *pFd, /* File-handle open on input database */ u32 nReserve, /* Possible nReserve value */ - i64 nSz + i64 nSz /* Size of database file in bytes */ ){ int rc = SQLITE_OK; const int nMin = 512; @@ -2142,6 +2258,10 @@ static int recoverVfsDetectPagesize( return rc; } +/* +** The xRead() method of the wrapper VFS. This is used to intercept calls +** to read page 1 of the input database. +*/ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ int rc = SQLITE_OK; if( pFd->pMethods==&recover_methods ){ @@ -2252,6 +2372,9 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ return rc; } +/* +** Used to make sqlite3_io_methods wrapper methods less verbose. +*/ #define RECOVER_VFS_WRAPPER(code) \ int rc = SQLITE_OK; \ if( pFd->pMethods==&recover_methods ){ \ @@ -2263,6 +2386,12 @@ static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){ } \ return rc; +/* +** Methods of the wrapper VFS. All methods except for xRead() and xClose() +** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent +** method on the lower level VFS, then reinstall the wrapper before returning. +** Those that return an integer value use the RECOVER_VFS_WRAPPER macro. +*/ static int recoverVfsWrite( sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff ){ @@ -2275,7 +2404,6 @@ static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){ pFd->pMethods->xTruncate(pFd, size) ); } - static int recoverVfsSync(sqlite3_file *pFd, int flags){ RECOVER_VFS_WRAPPER ( pFd->pMethods->xSync(pFd, flags) @@ -2343,9 +2471,15 @@ static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){ ); } +/* +** Install the VFS wrapper around the file-descriptor open on the input +** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held +** when this function is called. +*/ static void recoverInstallWrapper(sqlite3_recover *p){ sqlite3_file *pFd = 0; assert( recover_g.pMethods==0 ); + assert( sqlite3_mutex_held( sqlite3_mutex_alloc(RECOVER_MUTEX_ID) ) ); sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd); if( pFd ){ recover_g.pMethods = pFd->pMethods; @@ -2353,7 +2487,14 @@ static void recoverInstallWrapper(sqlite3_recover *p){ pFd->pMethods = &recover_methods; } } + +/* +** Uninstall the VFS wrapper that was installed around the file-descriptor open +** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be +** held when this function is called. +*/ static void recoverUninstallWrapper(sqlite3_recover *p){ + assert( sqlite3_mutex_held( sqlite3_mutex_alloc(RECOVER_MUTEX_ID) ) ); if( recover_g.pMethods ){ sqlite3_file *pFd = 0; sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd); @@ -2364,6 +2505,11 @@ static void recoverUninstallWrapper(sqlite3_recover *p){ } } +/* +** This function does the work of a single sqlite3_recover_step() call. It +** is guaranteed that the handle is not in an error state when this +** function is called. +*/ static void recoverStep(sqlite3_recover *p){ assert( p && p->errCode==SQLITE_OK ); switch( p->eState ){ @@ -2373,7 +2519,7 @@ static void recoverStep(sqlite3_recover *p){ recoverSqlCallback(p, "BEGIN"); recoverSqlCallback(p, "PRAGMA writable_schema = on"); - sqlite3_mutex_enter( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + sqlite3_mutex_enter( sqlite3_mutex_alloc(RECOVER_MUTEX_ID) ); recoverInstallWrapper(p); /* Open the output database. And register required virtual tables and @@ -2390,7 +2536,7 @@ static void recoverStep(sqlite3_recover *p){ recoverCacheSchema(p); recoverUninstallWrapper(p); - sqlite3_mutex_leave( sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2) ); + sqlite3_mutex_leave( sqlite3_mutex_alloc(RECOVER_MUTEX_ID) ); recoverExec(p, p->dbOut, "BEGIN"); diff --git a/manifest b/manifest index 6ea8d64d5c..a8752032f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2022-10-15T15:39:46.578 +C Add\smissing\scomments\sand\sfix\sother\scode\sissues. +D 2022-10-15T18:26:30.696 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -398,7 +398,7 @@ F ext/recover/recoverpgsz.test 93e970eab05e4e89f8fd6b1bd23f9ec137ea09857e66ba0d4 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test f356bb9fba7ffd6fc50e045e419464f0129ac6e24decf6e919584f79c3493727 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c 3ab2017a947fa8c9286aa6257d0b19443ec04e557af9d989227b89f36ff50dd6 +F ext/recover/sqlite3recover.c db8f2166dbbdfc09a58a85f51f8e848858858ca8b4915a2f9c8cd5fd5ca536a8 F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -486,7 +486,7 @@ F ext/session/test_session.c f433f68a8a8c64b0f5bc74dc725078f12483301ad4ae8375205 F ext/userauth/sqlite3userauth.h 7f3ea8c4686db8e40b0a0e7a8e0b00fac13aa7a3 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 F ext/userauth/userauth.c 7f00cded7dcaa5d47f54539b290a43d2e59f4b1eb5f447545fa865f002fc80cb -F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c w ext/wasm/EXPORTED_FUNCTIONS.fiddle +F ext/wasm/EXPORTED_FUNCTIONS.fiddle.in 27450c8b8c70875a260aca55435ec927068b34cef801a96205adb81bdcefc65c F ext/wasm/EXPORTED_RUNTIME_METHODS.fiddle 0e88c8cfc3719e4b7e74980d9da664c709e68acf863e48386cda376edfd3bfb0 F ext/wasm/GNUmakefile 4ec270532b921c7c4b437fbdb06f6a0ce41f3bd874395ce70dbc933c8553efa9 F ext/wasm/README.md 1e5b28158b74ab3ffc9d54fcbc020f0bbeb82c2ff8bbd904214c86c70e8a3066 @@ -503,7 +503,7 @@ F ext/wasm/api/sqlite3-api-glue.js 842dc03783aecc951a543a209523343a6fda9af258fb0 F ext/wasm/api/sqlite3-api-oo1.js 00f5cfce0989d2e08d7b21765d703c69234245d03a0cce8fcb32ccfcd53ffdbb F ext/wasm/api/sqlite3-api-opfs.js 5a8ab3b76880c8ada8710ca9ba1ca5b160872edfd8bd5322e4f179a7f41cc616 F ext/wasm/api/sqlite3-api-prologue.js b7c82a22d50658a48463fa646a23135273bc2cfa843aedda32627ff281c12e4d -F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8 w ext/wasm/api/sqlite3-api-worker.js +F ext/wasm/api/sqlite3-api-worker1.js 7f4f46cb6b512a48572d7567233896e6a9c46570c44bdc3d13419730c7c221c8 F ext/wasm/api/sqlite3-wasi.h 25356084cfe0d40458a902afb465df8c21fc4152c1d0a59b563a3fba59a068f9 F ext/wasm/api/sqlite3-wasm.c 4c131945ced4b08a694d287abcdb066b896d961ef79ee5241805ecc37e83d63a F ext/wasm/batch-runner.html cf1a410c92bad50fcec2ddc71390b4e9df63a6ea1bef12a5163a66a0af4d78d9 @@ -536,7 +536,7 @@ F ext/wasm/sql/000-mandelbrot.sql 775337a4b80938ac8146aedf88808282f04d02d983d826 F ext/wasm/sql/001-sudoku.sql 35b7cb7239ba5d5f193bc05ec379bcf66891bce6f2a5b3879f2f78d0917299b5 F ext/wasm/sqlite3-opfs-async-proxy.js 206ce6bbc3c30ad51a37d9c25e3a2712e70b586e0f9a2cf8cb0b9619017c2671 F ext/wasm/sqlite3-worker1-promiser.js 307d7837420ca6a9d3780dfc81194f1c0715637e6d9540e935514086b96913d8 -F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf w ext/wasm/api/sqlite3-worker.js +F ext/wasm/sqlite3-worker1.js 466e9bd39409ab03f3e00999887aaffc11e95b416e2689596e3d7f1516673fdf F ext/wasm/test-opfs-vfs.html eb69dda21eb414b8f5e3f7c1cc0f774103cc9c0f87b2d28a33419e778abfbab5 F ext/wasm/test-opfs-vfs.js 56c3d725044c668fa7910451e96c1195d25ad95825f9ac79f747a7759d1973d0 F ext/wasm/tester1-worker.html 0af7a22025ff1da72a84765d64f8f221844a57c6e6e314acf3a30f176101fd3f @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 52d0235ef3f21683daadf2343514f7b95a67c3589266f75c1e00e04f11dc6214 ed14863dd72e35fa3a23320c3d5a8166515faea39a555c28a27b2d35e701eac4 -R 67f4b2d0a660b4df4f4f8911e9f69960 +P 42255ead8e44b24a8abc04aca49f170c55f3b52f42366a0d8ef248ee65fcc7ce +R ba398a54428773327d217095fdca83a1 U dan -Z 2f155b3b54b3254ee16d85f44719ac5e +Z 9c11c46950d6b0a76e0307ada9a3c264 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index c99ce61654..49018171fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42255ead8e44b24a8abc04aca49f170c55f3b52f42366a0d8ef248ee65fcc7ce \ No newline at end of file +8ed4e4a80d8104d20b211a0b3e299abdb955cabdd36979bf6aae54266dc9778d \ No newline at end of file From 8e62d370cf57ce7b74d0a1999ae5ca1c6bf936ff Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 15 Oct 2022 19:18:10 +0000 Subject: [PATCH 40/42] Exclude slow test recovercorrupt.test from veryquick.test. FossilOrigin-Name: ff9a733532fe11bb253246b45a94681a174c976aafee5c780af571893e8ce694 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/permutations.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a8752032f6..48bd10c08c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\scomments\sand\sfix\sother\scode\sissues. -D 2022-10-15T18:26:30.696 +C Exclude\sslow\stest\srecovercorrupt.test\sfrom\sveryquick.test. +D 2022-10-15T19:18:10.184 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1381,7 +1381,7 @@ F test/parser1.test 6ccdf5e459a5dc4673d3273dc311a7e9742ca952dd0551a6a6320d27035c F test/pcache.test c8acbedd3b6fd0f9a7ca887a83b11d24a007972b F test/pcache2.test af7f3deb1a819f77a6d0d81534e97d1cf62cd442 F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff -F test/permutations.test 847df2d81f0172ab7032e55145f0f3da460dd65759ac2b02864e385add1947d5 +F test/permutations.test 650d89ab5aad0c9fab9325b11deca8662cb5e72f43e005073d35f12ad00eaca2 F test/pg_common.tcl 3b27542224db1e713ae387459b5d117c836a5f6e328846922993b6d2b7640d9f F test/pragma.test cae534c12a033a5c319ccc94f50b32811acdef9f67bf19a82ff42697caccd69f F test/pragma2.test e5d5c176360c321344249354c0c16aec46214c9f @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 42255ead8e44b24a8abc04aca49f170c55f3b52f42366a0d8ef248ee65fcc7ce -R ba398a54428773327d217095fdca83a1 +P 8ed4e4a80d8104d20b211a0b3e299abdb955cabdd36979bf6aae54266dc9778d +R 845d52da8da6984141e3c82e5a9ffafe U dan -Z 9c11c46950d6b0a76e0307ada9a3c264 +Z c4bdc57e0746f05859841f5f12f56c5e # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 49018171fa..531c1cb665 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8ed4e4a80d8104d20b211a0b3e299abdb955cabdd36979bf6aae54266dc9778d \ No newline at end of file +ff9a733532fe11bb253246b45a94681a174c976aafee5c780af571893e8ce694 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 8ab9dc8755..8a5484944a 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -130,6 +130,7 @@ set allquicktests [test_set $alltests -exclude { fts4merge5.test fts3cov.test fts3snippet.test fts3corrupt2.test fts3an.test fts3defer.test fts4langid.test fts3sort.test fts5unicode.test + recovercorrupt.test rtree4.test sessionbig.test From 361fb98b5d1ad2d9787e2b42cc849ec594a0fd51 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Oct 2022 18:03:39 +0000 Subject: [PATCH 41/42] Move file /ext/misc/dbdata.c to the /ext/recover/ directory. FossilOrigin-Name: f6d5ac807efcf6140016a8b8f0c74566c0bfc98ad92bd8fca03c5cc80ebf3dec --- Makefile.in | 4 ++-- Makefile.msc | 4 ++-- ext/{misc => recover}/dbdata.c | 0 ext/recover/recovercorrupt.test | 4 ++-- main.mk | 4 ++-- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/shell.c.in | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) rename ext/{misc => recover}/dbdata.c (100%) diff --git a/Makefile.in b/Makefile.in index d5000549f4..e0eef2b09e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -435,7 +435,7 @@ TESTSRC = \ $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/ext/session/test_session.c \ $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ $(TOP)/ext/rbu/test_rbu.c @@ -1111,7 +1111,7 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c diff --git a/Makefile.msc b/Makefile.msc index 9265cd641e..eff104eb56 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1590,7 +1590,7 @@ TESTEXT = \ $(TOP)\ext\rtree\test_rtreedoc.c \ $(TOP)\ext\recover\sqlite3recover.c \ $(TOP)\ext\recover\test_recover.c \ - $(TOP)\ext\misc\dbdata.c \ + $(TOP)\ext\recover\dbdata.c \ fts5.c # If use of zlib is enabled, add the "zipfile.c" source file. @@ -2231,7 +2231,7 @@ SHELL_SRC = \ $(TOP)\ext\expert\sqlite3expert.c \ $(TOP)\ext\expert\sqlite3expert.h \ $(TOP)\ext\misc\memtrace.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)\src\test_windirent.c diff --git a/ext/misc/dbdata.c b/ext/recover/dbdata.c similarity index 100% rename from ext/misc/dbdata.c rename to ext/recover/dbdata.c diff --git a/ext/recover/recovercorrupt.test b/ext/recover/recovercorrupt.test index 290ba63b0c..40859f3d1c 100644 --- a/ext/recover/recovercorrupt.test +++ b/ext/recover/recovercorrupt.test @@ -35,8 +35,8 @@ do_execsql_test 1.0 { } do_test 1.1 { - file size test.db -} {5120} + expr [file size test.db]>3072 +} {1} proc toggle_bit {blob bit} { set byte [expr {$bit / 8}] diff --git a/main.mk b/main.mk index 087f7f1b36..14ce23b01d 100644 --- a/main.mk +++ b/main.mk @@ -447,7 +447,7 @@ TESTSRC2 = \ $(TOP)/ext/session/sqlite3session.c \ $(TOP)/ext/session/test_session.c \ $(TOP)/ext/recover/sqlite3recover.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/test_recover.c \ fts5.c @@ -764,7 +764,7 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ - $(TOP)/ext/misc/dbdata.c \ + $(TOP)/ext/recover/dbdata.c \ $(TOP)/ext/recover/sqlite3recover.c \ $(TOP)/ext/recover/sqlite3recover.h \ $(TOP)/src/test_windirent.c diff --git a/manifest b/manifest index 48bd10c08c..38b47d48a8 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Exclude\sslow\stest\srecovercorrupt.test\sfrom\sveryquick.test. -D 2022-10-15T19:18:10.184 +C Move\sfile\s/ext/misc/dbdata.c\sto\sthe\s/ext/recover/\sdirectory. +D 2022-10-19T18:03:39.271 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 -F Makefile.in 64ccee54a3027ae4b2333dd28c680be45eda1002360273c03a6447b9da2bb113 +F Makefile.in b3d8ebf01cb78204988ee21e6b0d2d28c87f7d1f2b47ac23ed7a89802e68cb40 F Makefile.linux-gcc f609543700659711fbd230eced1f01353117621dccae7b9fb70daa64236c5241 -F Makefile.msc 4986d28d183af7b0660e82aae250385cfd3c3d006de7ed6ff553bb6cf6367757 +F Makefile.msc ea790f1db3c02b62cb13416a680fa909b54bc1e0533cca222db0dae3f7af3db9 F README.md 8b8df9ca852aeac4864eb1e400002633ee6db84065bd01b78c33817f97d31f5e F VERSION 8868ddfa6e1eee218286021a94b3e22d13e550c76c72d878857547ca001de24a F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -298,7 +298,6 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c 6dafd7f4348eecc7be9e920d4b419d1fb2af75d938cd9c59a20cfe8beb2f22b9 F ext/misc/compress.c 3354c77a7c8e86e07d849916000cdac451ed96500bfb5bd83b20eb61eee012c9 F ext/misc/csv.c ca8d6dafc5469639de81937cb66ae2e6b358542aba94c4f791910d355a8e7f73 -F ext/misc/dbdata.c ca7b235fa2396e8fc2e950826872f820f31268ac2cb51368b0d655bb71568f07 F ext/misc/dbdump.c b8592f6f2da292c62991a13864a60d6c573c47a9cc58362131b9e6a64f823e01 F ext/misc/decimal.c 09f967dcf4a1ee35a76309829308ec278d3648168733f4a1147820e11ebefd12 F ext/misc/eval.c 04bc9aada78c888394204b4ed996ab834b99726fb59603b0ee3ed6e049755dc1 @@ -386,10 +385,11 @@ F ext/rbu/rbuvacuum4.test a78898e438a44803eb2bc897ba3323373c9f277418e2d6d76e90f2 F ext/rbu/sqlite3rbu.c 8737cabdfbee84bb25a7851ecef8b1312be332761238da9be6ddb10c62ad4291 F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9cd389812 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a +F ext/recover/dbdata.c ca7b235fa2396e8fc2e950826872f820f31268ac2cb51368b0d655bb71568f07 w ext/misc/dbdata.c F ext/recover/recover1.test 93acc42f95259f8b34050ad75873685a305da76d6cb1727d003f45157a4a6402 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c F ext/recover/recoverclobber.test 294dcc894124ab4ca3a7b35766630742a3d25810fceac22220beb64f70a33a60 -F ext/recover/recovercorrupt.test 69af3d68aedc2cf1c3c41dbd6afb45b3cebadb57796d513550b5fd1e2a8b3fba +F ext/recover/recovercorrupt.test 6540aae95e17398dd70b44518367fd56588c44962cb276d2623a0fedba9efe9e F ext/recover/recovercorrupt2.test a131d8005337c092e2dfa3b84909ed67ae82d22399a8cfb8c984b2939969ca42 F ext/recover/recoverfault.test 3a0a32b9fc216592b97775d69220695b0926980c0f7424b7a59144e47d7cb568 F ext/recover/recoverfault2.test 321036336af23e778a87f148c4cc4407f88fbdab1fd72ddb661669be9020d36b @@ -551,7 +551,7 @@ F ext/wasm/wasmfs.make 3cce1820006196de140f90f2da4b4ea657083fb5bfee7d125be43f7a8 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 845d815ae761459c346ffd8a1ffc243195701b800955f94db257b8ce71a4ab38 +F main.mk d5fdf519235d657a3b87d70324f8ee1f057207626e5ab4daf23a0199d20a25ad F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -633,7 +633,7 @@ F src/random.c 546d6feb15ec69c1aafe9bb351a277cbb498fd5410e646add673acb805714960 F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92 F src/select.c b795f31a326564ffc448bb26f863c64f0796b2a5e0585a3358aff7877b72ed82 -F src/shell.c.in 831dbd94e910215fffd67c0cfdbf2d38f3906e60eaf3c9647d69af0bfe12abd6 +F src/shell.c.in d24655392a5be8143b8d29226856ba6528339355702b52e5a4c165e16c01ec8f F src/sqlite.h.in d9c8a6243fc0a1c270d69db33758e34b810af3462f9bc5b4af113b347e07c69d F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 5336beea1868d99d2f62e628dbea55e97267dbff8193291ab175e960c5df9141 @@ -2046,8 +2046,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8ed4e4a80d8104d20b211a0b3e299abdb955cabdd36979bf6aae54266dc9778d -R 845d52da8da6984141e3c82e5a9ffafe +P ff9a733532fe11bb253246b45a94681a174c976aafee5c780af571893e8ce694 +R 0b6e256547f11180eef1e8e83664e1a7 U dan -Z c4bdc57e0746f05859841f5f12f56c5e +Z 01393ff26cde3c9365efd46ef5c6323b # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 531c1cb665..609d1f6daf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff9a733532fe11bb253246b45a94681a174c976aafee5c780af571893e8ce694 \ No newline at end of file +f6d5ac807efcf6140016a8b8f0c74566c0bfc98ad92bd8fca03c5cc80ebf3dec \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index e9b028b349..cd0abfbe71 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1046,7 +1046,7 @@ INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) -INCLUDE ../ext/misc/dbdata.c +INCLUDE ../ext/recover/dbdata.c INCLUDE ../ext/recover/sqlite3recover.h INCLUDE ../ext/recover/sqlite3recover.c #endif From 50632afa6e10df30ecd22b7bfcc582c73ea142ab Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Oct 2022 18:29:19 +0000 Subject: [PATCH 42/42] Remove an undefined left-shift operation from the recover extension. FossilOrigin-Name: a67082357a2cc348faf8236aafa7f39eb5cb673b1d114a594c6d5bb257f85b73 --- ext/recover/sqlite3recover.c | 10 +++++----- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/recover/sqlite3recover.c b/ext/recover/sqlite3recover.c index 74b7f8a369..5e51bce9d4 100644 --- a/ext/recover/sqlite3recover.c +++ b/ext/recover/sqlite3recover.c @@ -2019,14 +2019,14 @@ static u32 recoverGetU32(const u8 *a){ ** and return the number of bytes consumed. */ static int recoverGetVarint(const u8 *a, i64 *pVal){ - sqlite3_int64 v = 0; + sqlite3_uint64 u = 0; int i; for(i=0; i<8; i++){ - v = (v<<7) + (a[i]&0x7f); - if( (a[i]&0x80)==0 ){ *pVal = v; return i+1; } + u = (u<<7) + (a[i]&0x7f); + if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; } } - v = (v<<8) + (a[i]&0xff); - *pVal = v; + u = (u<<8) + (a[i]&0xff); + *pVal = (sqlite3_int64)u; return 9; } diff --git a/manifest b/manifest index e9680fd7ee..848d7b0c4e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfurther\schanges\sfrom\strunk,\sincluding\sfix\sto\sdbdata.c. -D 2022-10-26T18:22:22.756 +C Remove\san\sundefined\sleft-shift\soperation\sfrom\sthe\srecover\sextension. +D 2022-10-26T18:29:19.827 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -398,7 +398,7 @@ F ext/recover/recoverpgsz.test 93e970eab05e4e89f8fd6b1bd23f9ec137ea09857e66ba0d4 F ext/recover/recoverrowid.test 1694a1a5526d825f71279f3d02ab02a1ee4c5265de18858bf54cb8ec54487ac8 F ext/recover/recoverslowidx.test f356bb9fba7ffd6fc50e045e419464f0129ac6e24decf6e919584f79c3493727 F ext/recover/recoversql.test f9872ff2114e13ffd8ee31e1de06919f62b9b48bc080191b5bd076d10becb60f -F ext/recover/sqlite3recover.c db8f2166dbbdfc09a58a85f51f8e848858858ca8b4915a2f9c8cd5fd5ca536a8 +F ext/recover/sqlite3recover.c 537bfad4bf9f14fd43d138b988365bb38548fabec3252d3bff719524d907a1da F ext/recover/sqlite3recover.h f698ccc94bd4da38761035415ad08c4549a408491ff9fd5f52d34d2214f64e36 F ext/recover/test_recover.c 61ec931e47abca6b2210f46239cafd9f3060741605e3d3c45a7c7a53f63dd957 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 @@ -530,7 +530,7 @@ F ext/wasm/fiddle.make 68abe5dcfdd6fdf8dc1b715b94b96fae771b5b28bc1843997b9b1ec79 F ext/wasm/fiddle/emscripten.css 3d253a6fdb8983a2ac983855bfbdd4b6fa1ff267c28d69513dd6ef1f289ada3f F ext/wasm/fiddle/fiddle-worker.js 41b9796d689fabbef1ad9447496d14645eeb4101b534e54951fa5259c877c341 F ext/wasm/fiddle/fiddle.js 974b995119ac443685d7d94d3b3c58c6a36540e9eb3fed7069d5653284071715 -F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 w ext/wasm/fiddle/fiddle.html +F ext/wasm/fiddle/index.html 5daf54e8f3d7777cbb1ca4f93affe28858dbfff25841cb4ab81d694efed28ec2 F ext/wasm/index-dist.html cb0da16cba0f21cda2c25724c5869102d48eb0af04446acd3cd0ca031f80ed19 F ext/wasm/index.html 6ac073d45d7602e1d8a5b493ce4204dc4a6c5594fdeeb12dec38c4e36ce473ad F ext/wasm/jaccwabyt/jaccwabyt.js 0d7f32817456a0f3937fcfd934afeb32154ca33580ab264dab6c285e6dbbd215 @@ -2052,8 +2052,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 757e3f585959f4f113fee48fe3d504f037604c53a95c8d47ce5c2bccfb2af8ff 4eef562a00ae988f2426c9af51f4165c0e4cbccd601061664a0c54c19b9cc70f -R fd8351a8bbdf0dfc78a1c53b0189bb2d +P bcf6b48d52c1ce656899f50bd508c0920cae8cf2ef3c0758a7633981f0f0f484 +R 1abad855e0472232f0e7862b9a7dd438 U dan -Z 99bd3f3c58c53208f300189c934ad348 +Z 4fb1e189272de9078a5ba7b4ef187b9a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 2bca4f1203..af7804ec05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcf6b48d52c1ce656899f50bd508c0920cae8cf2ef3c0758a7633981f0f0f484 \ No newline at end of file +a67082357a2cc348faf8236aafa7f39eb5cb673b1d114a594c6d5bb257f85b73 \ No newline at end of file