Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c.

FossilOrigin-Name: f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562
This commit is contained in:
dan 2019-04-22 20:52:12 +00:00
parent 68cb86ef23
commit b40af49d73
5 changed files with 201 additions and 35 deletions

View File

@ -506,6 +506,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
nRem -= nCopy;
pgnoOvfl = get_uint32(aOvfl);
sqlite3_free(aOvfl);
}
}

View File

@ -1,5 +1,5 @@
C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork.
D 2019-04-20T20:57:28.136
C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c.
D 2019-04-22T20:52:12.850
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8
F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f
F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189
F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb
F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054
F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8
F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336
F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e
F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f
@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8
F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49
F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680
F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33
F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5
@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee
F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8
F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373
F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10
F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4
F test/dbdata.test c8d97bafd1b2efb1e445871c4641208dcd91e686d2dfbb6463d83934adbd1ac5
F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf
F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee
@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32
R 9a7e323fe9ebfe9eb0759e1c70136795
P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da
R 14050e9b3d7e7a784dc8a127e6275222
U dan
Z 100e3ccaddf1d19d0bacd74d773a93b7
Z abcbf2aebf4cbfcbf65f71e558364487

View File

@ -1 +1 @@
7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da
f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562

View File

@ -6073,9 +6073,9 @@ typedef struct RecoverTable RecoverTable;
struct RecoverTable {
char *zName; /* Name of table */
char *zQuoted; /* Quoted version of zName */
char *zCreate; /* SQL to create table in default schema */
int nCol; /* Number of columns in table */
char **azlCol; /* Array of column lists */
int iPk;
};
/*
@ -6085,7 +6085,6 @@ static void recoverFreeTable(RecoverTable *pTab){
if( pTab ){
sqlite3_free(pTab->zName);
sqlite3_free(pTab->zQuoted);
sqlite3_free(pTab->zCreate);
if( pTab->azlCol ){
int i;
for(i=0; i<pTab->nCol; i++){
@ -6097,17 +6096,141 @@ static void recoverFreeTable(RecoverTable *pTab){
}
}
static void recoverOldTable(
int *pRc, /* IN/OUT: Error code */
RecoverTable *pTab,
const char *zName, /* Name of table */
const char *zSql, /* CREATE TABLE statement */
int bIntkey,
int nCol
){
sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */
int rc = *pRc;
if( rc==SQLITE_OK ){
int nSqlCol = 0;
int bSqlIntkey = 0;
sqlite3_stmt *pStmt = 0;
rc = sqlite3_open("", &dbtmp);
if( rc==SQLITE_OK ){
rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0);
if( rc==SQLITE_ERROR ){
rc = SQLITE_OK;
goto finished;
}
}
shellPreparePrintf(dbtmp, &rc, &pStmt,
"SELECT count(*) FROM pragma_table_info(%Q)", zName
);
if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
nSqlCol = sqlite3_column_int(pStmt, 0);
}
shellFinalize(&rc, pStmt);
if( rc!=SQLITE_OK || nSqlCol<nCol ){
goto finished;
}
shellPreparePrintf(dbtmp, &rc, &pStmt,
"SELECT ("
" SELECT substr(data,1,1)==X'0D' FROM sqlite_dbpage WHERE pgno=rootpage"
") FROM sqlite_master WHERE name = %Q", zName
);
if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
bSqlIntkey = sqlite3_column_int(pStmt, 0);
}
shellFinalize(&rc, pStmt);
if( bIntkey==bSqlIntkey ){
const char *zPk = "_rowid_";
sqlite3_stmt *pPkFinder = 0;
shellPreparePrintf(dbtmp, &rc, &pPkFinder,
"SELECT cid, name FROM pragma_table_info(%Q) "
" WHERE pk=1 AND type='integer' COLLATE nocase"
" AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)",
zName, zName
);
if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){
pTab->iPk = sqlite3_column_int(pPkFinder, 0);
zPk = (const char*)sqlite3_column_text(pPkFinder, 1);
}
pTab->zName = shellMPrintf(&rc, "%s", zName);
pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName);
pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol);
pTab->nCol = nSqlCol;
if( nSqlCol==1 && pTab->iPk==0 ){
pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk);
}else{
shellPreparePrintf(dbtmp, &rc, &pStmt,
"SELECT -1+row_number() OVER (ORDER BY cid),"
" %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) "
" OVER (ORDER BY cid) "
"FROM pragma_table_info(%Q)",
(bIntkey ? zPk : ""), (bIntkey ? ", " : ""),
pTab->iPk, zName
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int idx = sqlite3_column_int(pStmt, 0);
const char *zText = (const char*)sqlite3_column_text(pStmt, 1);
pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText);
}
shellFinalize(&rc, pStmt);
}
shellFinalize(&rc, pPkFinder);
}
}
finished:
sqlite3_close(dbtmp);
*pRc = rc;
}
static RecoverTable *recoverNewTable(
ShellState *pState,
int *pRc,
int iRoot,
int bIntkey,
int nCol
){
sqlite3_stmt *pStmt = 0;
RecoverTable *pRet = 0;
int bNoop = 0;
const char *zSql = 0;
const char *zName = 0;
pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable));
if( pRet ){
if( pRet ) pRet->iPk = -2;
/* Search the recovered schema for an object with root page iRoot. */
shellPreparePrintf(pState->db, pRc, &pStmt,
"SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot
);
while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zType = (const char*)sqlite3_column_text(pStmt, 0);
if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){
bNoop = 1;
break;
}
if( sqlite3_stricmp(zType, "table")==0 ){
zName = (const char*)sqlite3_column_text(pStmt, 1);
zSql = (const char*)sqlite3_column_text(pStmt, 2);
recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol);
break;
}
}
shellFinalize(pRc, pStmt);
if( bNoop ){
sqlite3_free(pRet);
return 0;
}
if( pRet && pRet->zName==0 ){
sqlite3_stmt *pStmt = 0;
pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot);
pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName);
pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol);
@ -6117,8 +6240,8 @@ static RecoverTable *recoverNewTable(
"WITH s(i) AS ("
" SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d"
")"
"SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s",
nCol
"SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s",
nCol, (bIntkey ? "id, " : "")
);
while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
int idx = sqlite3_column_int(pStmt, 0);
@ -6127,9 +6250,15 @@ static RecoverTable *recoverNewTable(
}
shellFinalize(pRc, pStmt);
pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)",
if( *pRc==SQLITE_OK ){
char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)",
pRet->zName, pRet->azlCol[nCol-1]
);
);
if( zCreate ){
raw_printf(pState->out, "%s;\n", zCreate);
sqlite3_free(zCreate);
}
}
}
if( *pRc!=SQLITE_OK ){
@ -6146,7 +6275,6 @@ static RecoverTable *recoverNewTable(
** on stream pState->out.
*/
static int recoverDatabaseCmd(ShellState *pState){
const char *zSql;
int rc = SQLITE_OK;
sqlite3_stmt *pLoop = 0; /* Loop through all root pages */
@ -6173,7 +6301,9 @@ static int recoverDatabaseCmd(ShellState *pState){
/* Create the "map" table that will (eventually) contain instructions
** for dealing with each page in the db that contains one or more
** records. */
"CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);"
"CREATE TABLE recovery.map("
"pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT"
");"
/* Populate table [map]. If there are circular loops of pages in the
** database, the following adds all pages in such a loop to the map
@ -6182,10 +6312,12 @@ static int recoverDatabaseCmd(ShellState *pState){
" SELECT page_count, max(field+1) "
" FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count"
" UNION ALL"
" SELECT * FROM (SELECT i-1, max(field+1)"
" FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)"
" SELECT i-1, ("
" SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1"
" ) FROM pages WHERE i>=2"
")"
"INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, ("
"INSERT INTO recovery.map(pgno, maxlen, intkey, root) "
" SELECT i, maxlen, NULL, ("
" WITH p(orig, pgno, parent) AS ("
" SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)"
" UNION ALL"
@ -6195,6 +6327,9 @@ static int recoverDatabaseCmd(ShellState *pState){
" SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)"
") "
"FROM pages WHERE maxlen > 0;"
"UPDATE recovery.map AS o SET intkey = ("
" SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno"
");"
/* Extract data from page 1 and any linked pages into table
** recovery.schema. With the same schema as an sqlite_master table. */
@ -6221,36 +6356,53 @@ static int recoverDatabaseCmd(ShellState *pState){
return rc;
#endif
/* Open a transaction, then print out all non-virtual, non-"sqlite_%"
** CREATE TABLE statements that extracted from the existing schema. */
if( rc==SQLITE_OK ){
sqlite3_stmt *pStmt = 0;
raw_printf(pState->out, "BEGIN;\n");
shellPrepare(pState->db, &rc,
"SELECT sql FROM recovery.schema "
"WHERE type='table' "
" AND length(sql)>6"
" AND sql LIKE 'create table%'"
" AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){
const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0);
raw_printf(pState->out, "%s;\n", zCreateTable);
}
shellFinalize(&rc, pStmt);
}
/* Loop through each root page. */
zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root";
shellPrepare(pState->db, &rc, zSql, &pLoop);
shellPrepare(pState->db, &rc,
"SELECT root, intkey, max(maxlen) FROM recovery.map"
" WHERE root>1 GROUP BY root, intkey", &pLoop
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){
int iRoot = sqlite3_column_int(pLoop, 0);
int nCol = sqlite3_column_int(pLoop, 1);
int bIntkey = sqlite3_column_int(pLoop, 1);
int nCol = sqlite3_column_int(pLoop, 2);
RecoverTable *pTab;
pTab = recoverNewTable(pState, &rc, iRoot, nCol);
pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol);
if( pTab ){
sqlite3_stmt *pData = 0;
raw_printf(pState->out, "%s;\n", pTab->zCreate);
shellPreparePrintf(pState->db, &rc, &pData,
"SELECT max(field), group_concat(quote(value), ', ') "
"FROM sqlite_dbdata WHERE pgno IN ("
" SELECT pgno FROM recovery.map WHERE root=%d"
")"
"GROUP BY pgno, cell;", iRoot
" AND field!=%d "
"GROUP BY pgno, cell;", iRoot, pTab->iPk
);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){
int iMax = sqlite3_column_int(pData, 0);
const char *zVal = (const char*)sqlite3_column_text(pData, 1);
if( iMax+1==pTab->nCol ){
raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n",
pTab->zQuoted, zVal);
}else{
raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n",
pTab->zQuoted, pTab->azlCol[iMax], zVal
);
}
raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n",
pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal
);
}
shellFinalize(&rc, pData);
}
@ -6258,6 +6410,9 @@ static int recoverDatabaseCmd(ShellState *pState){
}
shellFinalize(&rc, pLoop);
if( rc==SQLITE_OK ){
raw_printf(pState->out, "COMMIT;\n");
}
sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0);
return rc;
}

View File

@ -67,6 +67,16 @@ do_execsql_test 1.3 {
SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1;
} $big
do_execsql_test 1.4 {
DELETE FROM t1;
INSERT INTO t1 VALUES(NULL, randomblob(5050));
}
do_test 1.5 {
execsql {
SELECT quote(value) FROM sqlite_dbdata WHERE pgno=2 AND cell=0 AND field=1;
}
} [db one {SELECT quote(b) FROM t1}]
#-------------------------------------------------------------------------
reset_db
db enable_load_extension 1