From f1173b69b1c3bfc68ea2877e6fabf15902b7bf72 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Apr 2019 21:17:22 +0000 Subject: [PATCH 01/16] Add the experimental dbdata extension. FossilOrigin-Name: a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb --- ext/misc/dbdata.c | 620 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- test/dbdata.test | 54 ++++ 4 files changed, 686 insertions(+), 7 deletions(-) create mode 100644 ext/misc/dbdata.c create mode 100644 test/dbdata.test diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c new file mode 100644 index 0000000000..c0ad8c1f9e --- /dev/null +++ b/ext/misc/dbdata.c @@ -0,0 +1,620 @@ +/* +** 2019-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the eponymous "sqlite_dbdata" +** virtual table. sqlite_dbdata is used to extract data directly from a +** database b-tree page and its associated overflow pages, bypassing the b-tree +** layer. The table schema is equivalent to: +** +** CREATE TABLE sqlite_dbdata( +** pgno INTEGER, +** cell INTEGER, +** field INTEGER, +** value ANY, +** schema TEXT HIDDEN +** ); +** +** Each page of the database is inspected. If it cannot be interpreted as a +** b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the table +** contains one row for each field in the record associated with each +** cell on the page. For intkey b-trees, the key value is stored in field -1. +** +** For example, for the database: +** +** CREATE TABLE t1(a, b); -- root page is page 2 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: +** +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES +** (2, 0, -1, 5 ), +** (2, 0, 0, 'v' ), +** (2, 0, 1, 'five'), +** (2, 1, -1, 10 ), +** (2, 1, 0, 'x' ), +** (2, 1, 1, 'ten' ); +** +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. +** +** This module requires that the "sqlite_dbpage" eponymous virtual table be +** available. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" + +typedef unsigned char u8; +typedef unsigned int u32; + +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +typedef struct DbdataTable DbdataTable; +typedef struct DbdataCursor DbdataCursor; + +/* A cursor for the sqlite_dbdata table */ +struct DbdataCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + + int iPgno; /* Current page number */ + u8 *aPage; /* Buffer containing page */ + int nPage; /* Size of aPage[] in bytes */ + int nCell; /* Number of cells on aPage[] */ + int iCell; /* Current cell number */ + u8 *pRec; /* Buffer containing current record */ + int nRec; /* Size of pRec[] in bytes */ + int nField; /* Number of fields in pRec */ + int iField; /* Current field number */ + sqlite3_int64 iIntkey; /* Integer key value */ + + sqlite3_int64 iRowid; +}; + +/* The sqlite_dbdata table */ +struct DbdataTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection */ +}; + +#define DBDATA_COLUMN_PGNO 0 +#define DBDATA_COLUMN_CELL 1 +#define DBDATA_COLUMN_FIELD 2 +#define DBDATA_COLUMN_VALUE 3 +#define DBDATA_COLUMN_SCHEMA 4 + +#define DBDATA_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " cell INTEGER," \ + " field INTEGER," \ + " value ANY," \ + " schema TEXT HIDDEN" \ + ")" + +/* +** Connect to the sqlite_dbdata virtual table. +*/ +static int dbdataConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbdataTable *pTab = 0; + int rc = sqlite3_declare_vtab(db, DBDATA_SCHEMA); + + if( rc==SQLITE_OK ){ + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(DbdataTable)); + pTab->db = db; + } + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbdata virtual table. +*/ +static int dbdataDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** +** This function interprets two types of constraints: +** +** schema=? +** pgno=? +** +** If neither are present, idxNum is set to 0. If schema=? is present, +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit +** in idxNum is set. +** +** If both parameters are present, schema is in position 0 and pgno in +** position 1. +*/ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + int iSchema = -1; + int iPgno = -1; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( p->iColumn==DBDATA_COLUMN_SCHEMA ){ + if( p->usable==0 ) return SQLITE_CONSTRAINT; + iSchema = i; + } + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ + iPgno = i; + } + } + } + + if( iSchema>=0 ){ + pIdxInfo->aConstraintUsage[iSchema].argvIndex = 1; + pIdxInfo->aConstraintUsage[iSchema].omit = 1; + } + if( iPgno>=0 ){ + pIdxInfo->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdxInfo->aConstraintUsage[iPgno].omit = 1; + } + pIdxInfo->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + return SQLITE_OK; +} + +/* +** Open a new dbdata cursor. +*/ +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbdataCursor *pCsr; + + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(DbdataCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +static void dbdataResetCursor(DbdataCursor *pCsr){ + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + pCsr->iPgno = 1; + pCsr->iCell = 0; + pCsr->iField = 0; +} + +/* +** Close a dbdata cursor. +*/ +static int dbdataClose(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + dbdataResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* Decode big-endian integers */ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +static int dbdataLoadPage( + DbdataCursor *pCsr, + u32 pgno, + u8 **ppPage, + int *pnPage +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = pCsr->pStmt; + + *ppPage = 0; + *pnPage = 0; + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + u8 *pPage = (u8*)sqlite3_malloc64(nCopy); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + *ppPage = pPage; + *pnPage = nCopy; + } + } + rc2 = sqlite3_reset(pStmt); + if( *ppPage==0 ) rc = rc2; + + return rc; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Move a dbdata cursor to the next entry in the file. +*/ +static int dbdataNext(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor *)pCursor; + + pCsr->iRowid++; + while( 1 ){ + int rc; + + if( pCsr->aPage==0 ){ + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + pCsr->iCell = 0; + pCsr->nCell = get_uint16(&pCsr->aPage[pCsr->iPgno==1 ? 103 : 3]); + } + + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int iOff = (pCsr->iPgno==1 ? 100 : 0); + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + pCsr->iCell = pCsr->nCell; + break; + } + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + return SQLITE_OK; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; + }else{ + X = ((U-12)*64/255)-23; + } + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } + } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + sqlite3_free(aOvfl); + } + } + + /* Figure out how many fields in the record */ + pCsr->nField = 0; + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + while( iHdrpRec[iHdr], &iDummy); + pCsr->nField++; + } + + pCsr->iField = (bHasRowid ? -2 : -1); + } + + pCsr->iField++; + if( pCsr->iFieldnField ) return SQLITE_OK; + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; + } + + assert( !"can't get here" ); + return SQLITE_OK; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int dbdataEof(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + return pCsr->aPage==0; +} + +/* Position a cursor back to the beginning. +*/ +static int dbdataFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + int rc; + const char *zSchema = "main"; + + dbdataResetCursor(pCsr); + assert( pCsr->iPgno==1 ); + if( idxNum & 0x01 ){ + zSchema = sqlite3_value_text(argv[0]); + } + if( idxNum & 0x02 ){ + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + } + + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + } + if( rc==SQLITE_OK ){ + rc = dbdataNext(pCursor); + } + return rc; +} + +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + return ((eType-12) / 2); + } +} + +static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + sqlite3_result_text(pCtx, pData, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } +} + +/* Return a column for the sqlite_dbdata table */ +static int dbdataColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + int iHdr; + sqlite3_int64 iType; + sqlite3_int64 iOff; + int i; + iHdr = dbdataGetVarint(pCsr->pRec, &iOff); + for(i=0; iiField; i++){ + iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + iOff += dbdataValueBytes(iType); + } + dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + + dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + } + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_dbdata table */ +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + + +/* +** Invoke this routine to register the "sqlite_dbdata" virtual table module +*/ +static int sqlite3DbdataRegister(sqlite3 *db){ + static sqlite3_module dbdata_module = { + 0, /* iVersion */ + 0, /* xCreate */ + dbdataConnect, /* xConnect */ + dbdataBestIndex, /* xBestIndex */ + dbdataDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + dbdataOpen, /* xOpen - open a cursor */ + dbdataClose, /* xClose - close a cursor */ + dbdataFilter, /* xFilter - configure scan constraints */ + dbdataNext, /* xNext - advance a cursor */ + dbdataEof, /* xEof - check for end of scan */ + dbdataColumn, /* xColumn - read data */ + dbdataRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + return sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3DbdataRegister(db); +} diff --git a/manifest b/manifest index b6c54ac76e..59bdc64972 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\sand\ssmaller\simplementation\sof\ssqlite3StrICmp(). -D 2019-04-17T11:34:44.568 +C Add\sthe\sexperimental\sdbdata\sextension. +D 2019-04-17T21:17:22.795 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,6 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb +F ext/misc/dbdata.c 436a7883a7f1455c5d2853bd927c5ac31826b21c6e16ca11a21d7262d25ff838 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -786,6 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 +F test/dbdata.test 6e791619d18e0cff2c79392de980ca0594368cdaa05326f043f866beb0c82614 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1818,7 +1820,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 884b4b7e502b4e991677b53971277adfaf0a04a284f8e483e2553d0f83156b50 -R dcc66ec8a55d4cba7e12163fe5fbfbcf -U drh -Z 40a9f6f7fce76c8f72a2cb9229b22090 +P 7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d +R 9b65ccecf461800383c04a268416aaa8 +T *branch * dbdata +T *sym-dbdata * +T -sym-trunk * +U dan +Z f99e8e6e33bab5d12ccf1d6f1a7cae6f diff --git a/manifest.uuid b/manifest.uuid index c8e27168f5..dc49ef98d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d \ No newline at end of file +a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test new file mode 100644 index 0000000000..c4412a5de9 --- /dev/null +++ b/test/dbdata.test @@ -0,0 +1,54 @@ +# 2019-04-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_dbpage virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix dbdata + +ifcapable !vtab||!compound { + finish_test + return +} +db enable_load_extension 1 +if { [catch { db eval { SELECT load_extension('../dbdata') } }] } { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE T1(a, b); + INSERT INTO t1(rowid, a ,b) VALUES(5, 'v', 'five'); + INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +} + +do_execsql_test 1.1 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata WHERE pgno=2; +} { + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.2 { + INSERT INTO t1 VALUES(NULL, $big); + SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; +} $big + + + +finish_test From 3b412ac2473707fc7bc4ce29466d7b7dc4988905 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Apr 2019 21:14:11 +0000 Subject: [PATCH 02/16] Add the sqlite_dbptr virtual table to the dbdata extension. For querying the links between b-tree pages. FossilOrigin-Name: 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 --- ext/misc/dbdata.c | 347 ++++++++++++++++++++++++++++------------------ manifest | 17 +-- manifest.uuid | 2 +- test/dbdata.test | 52 ++++++- 4 files changed, 270 insertions(+), 148 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index c0ad8c1f9e..eb2bc01600 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -23,6 +23,9 @@ ** schema TEXT HIDDEN ** ); ** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND "schema". +** ** Each page of the database is inspected. If it cannot be interpreted as a ** b-tree page, or if it is a b-tree page containing 0 entries, the ** sqlite_dbdata table contains no rows for that page. Otherwise, the table @@ -52,6 +55,13 @@ ** ** This module requires that the "sqlite_dbpage" eponymous virtual table be ** available. +** +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); */ #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -67,6 +77,7 @@ SQLITE_EXTENSION_INIT1 typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; + /* A cursor for the sqlite_dbdata table */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ @@ -77,19 +88,22 @@ struct DbdataCursor { int nPage; /* Size of aPage[] in bytes */ int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + sqlite3_int64 iRowid; + + /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ int nRec; /* Size of pRec[] in bytes */ int nField; /* Number of fields in pRec */ int iField; /* Current field number */ sqlite3_int64 iIntkey; /* Integer key value */ - - sqlite3_int64 iRowid; }; /* The sqlite_dbdata table */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ + int bPtr; /* True for sqlite3_dbptr table */ }; #define DBDATA_COLUMN_PGNO 0 @@ -98,6 +112,10 @@ struct DbdataTable { #define DBDATA_COLUMN_VALUE 3 #define DBDATA_COLUMN_SCHEMA 4 +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 + #define DBDATA_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -107,6 +125,13 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" + /* ** Connect to the sqlite_dbdata virtual table. */ @@ -118,7 +143,7 @@ static int dbdataConnect( char **pzErr ){ DbdataTable *pTab = 0; - int rc = sqlite3_declare_vtab(db, DBDATA_SCHEMA); + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); @@ -127,6 +152,7 @@ static int dbdataConnect( }else{ memset(pTab, 0, sizeof(DbdataTable)); pTab->db = db; + pTab->bPtr = (pAux!=0); } } @@ -157,14 +183,16 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ ** position 1. */ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - if( p->iColumn==DBDATA_COLUMN_SCHEMA ){ + if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; iSchema = i; } @@ -210,6 +238,7 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iPgno = 1; pCsr->iCell = 0; pCsr->iField = 0; + pCsr->bOnePage = 0; } /* @@ -281,132 +310,152 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ ** Move a dbdata cursor to the next entry in the file. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor *)pCursor; + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; pCsr->iRowid++; while( 1 ){ int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); if( pCsr->aPage==0 ){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK ) return rc; - pCsr->iCell = 0; - pCsr->nCell = get_uint16(&pCsr->aPage[pCsr->iPgno==1 ? 103 : 3]); + if( rc!=SQLITE_OK || pCsr->aPage==0 ) return rc; + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } - /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ - int iOff = (pCsr->iPgno==1 ? 100 : 0); - int bHasRowid = 0; - int nPointer = 0; - sqlite3_int64 nPayload = 0; - sqlite3_int64 nHdr = 0; - int iHdr; - int U, X; - int nLocal; - - switch( pCsr->aPage[iOff] ){ - case 0x02: - nPointer = 4; - break; - case 0x0a: - break; - case 0x0d: - bHasRowid = 1; - break; - default: - pCsr->iCell = pCsr->nCell; - break; + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; } + pCsr->iCell++; if( pCsr->iCell>=pCsr->nCell ){ sqlite3_free(pCsr->aPage); pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ return SQLITE_OK; } + }else{ + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } - iOff += 8 + nPointer + pCsr->iCell*2; - iOff = get_uint16(&pCsr->aPage[iOff]); - - /* For an interior node cell, skip past the child-page number */ - iOff += nPointer; - - /* Load the "byte of payload including overflow" field */ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); - - /* If this is a leaf intkey cell, load the rowid */ - if( bHasRowid ){ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); - } - - /* Allocate space for payload */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - pCsr->nRec = nPayload; - - U = pCsr->nPage; - if( bHasRowid ){ - X = U-35; - }else{ - X = ((U-12)*64/255)-23; - } - if( nPayload<=X ){ - nLocal = nPayload; - }else{ - int M, K; - M = ((U-12)*32/255)-23; - K = M+((nPayload-M)%(U-4)); - if( K<=X ){ - nLocal = K; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + continue; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; }else{ - nLocal = M; + X = ((U-12)*64/255)-23; } - } - - /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); - iOff += nLocal; - - /* Load content from overflow pages */ - if( nPayload>nLocal ){ - sqlite3_int64 nRem = nPayload - nLocal; - u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); - while( nRem>0 ){ - u8 *aOvfl = 0; - int nOvfl = 0; - int nCopy; - rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); - assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); - if( rc!=SQLITE_OK ) return rc; - - nCopy = U-4; - if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); - nRem -= nCopy; - - sqlite3_free(aOvfl); + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + sqlite3_free(aOvfl); + } + } + + /* Figure out how many fields in the record */ + pCsr->nField = 0; + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + while( iHdrpRec[iHdr], &iDummy); + pCsr->nField++; + } + + pCsr->iField = (bHasRowid ? -2 : -1); } - - /* Figure out how many fields in the record */ - pCsr->nField = 0; - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - while( iHdrpRec[iHdr], &iDummy); - pCsr->nField++; - } - - pCsr->iField = (bHasRowid ? -2 : -1); + + pCsr->iField++; + if( pCsr->iFieldnField ) return SQLITE_OK; + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; } - - pCsr->iField++; - if( pCsr->iFieldnField ) return SQLITE_OK; - - /* Advance to the next cell. The next iteration of the loop will load - ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; - pCsr->iCell++; } assert( !"can't get here" ); @@ -440,6 +489,7 @@ static int dbdataFilter( } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; } rc = sqlite3_prepare_v2(pTab->db, @@ -533,34 +583,54 @@ static int dbdataColumn( int i ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; - switch( i ){ - case DBDATA_COLUMN_PGNO: - sqlite3_result_int64(ctx, pCsr->iPgno); - break; - case DBDATA_COLUMN_CELL: - sqlite3_result_int(ctx, pCsr->iCell); - break; - case DBDATA_COLUMN_FIELD: - sqlite3_result_int(ctx, pCsr->iField); - break; - case DBDATA_COLUMN_VALUE: { - if( pCsr->iField<0 ){ - sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ - int iHdr; - sqlite3_int64 iType; - sqlite3_int64 iOff; - int i; - iHdr = dbdataGetVarint(pCsr->pRec, &iOff); - for(i=0; iiField; i++){ - iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - iOff += dbdataValueBytes(iType); + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); } - dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - - dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + int iHdr; + sqlite3_int64 iType; + sqlite3_int64 iOff; + int i; + iHdr = dbdataGetVarint(pCsr->pRec, &iOff); + for(i=0; iiField; i++){ + iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + iOff += dbdataValueBytes(iType); + } + dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + + dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + } + break; } - break; } } return SQLITE_OK; @@ -604,7 +674,12 @@ static int sqlite3DbdataRegister(sqlite3 *db){ 0, /* xRollbackTo */ 0 /* xShadowName */ }; - return sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); + } + return rc; } #ifdef _WIN32 diff --git a/manifest b/manifest index 59bdc64972..d6a2b045e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\sdbdata\sextension. -D 2019-04-17T21:17:22.795 +C Add\sthe\ssqlite_dbptr\svirtual\stable\sto\sthe\sdbdata\sextension.\sFor\squerying\sthe\slinks\sbetween\sb-tree\spages. +D 2019-04-18T21:14:11.260 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 436a7883a7f1455c5d2853bd927c5ac31826b21c6e16ca11a21d7262d25ff838 +F ext/misc/dbdata.c 20d85d7d7503817a5f9ac7d93c4ae7bf5fcc40570a77adc417acab0500058c99 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 6e791619d18e0cff2c79392de980ca0594368cdaa05326f043f866beb0c82614 +F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,10 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d -R 9b65ccecf461800383c04a268416aaa8 -T *branch * dbdata -T *sym-dbdata * -T -sym-trunk * +P a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb +R 901e9e476021499846a29c847f49c578 U dan -Z f99e8e6e33bab5d12ccf1d6f1a7cae6f +Z e0c2ac18f5bd53328c7a0485e8c7ef97 diff --git a/manifest.uuid b/manifest.uuid index dc49ef98d8..6ee3f97f4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb \ No newline at end of file +3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test index c4412a5de9..5a0df5dc86 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -43,12 +43,62 @@ do_execsql_test 1.1 { 2 1 1 'ten' } -set big [string repeat big 2000] +breakpoint do_execsql_test 1.2 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata; +} { + 1 0 -1 1 + 1 0 0 'table' + 1 0 1 'T1' + 1 0 2 'T1' + 1 0 3 2 + 1 0 4 {'CREATE TABLE T1(a, b)'} + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.3 { INSERT INTO t1 VALUES(NULL, $big); SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +#------------------------------------------------------------------------- +reset_db +db enable_load_extension 1 +db eval { SELECT load_extension('../dbdata') } + +do_execsql_test 2.0 { + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT randomblob(900) FROM s; +} + +do_execsql_test 2.1 { + SELECT * FROM sqlite_dbptr WHERE pgno=2; +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 +} + +do_execsql_test 2.2 { + SELECT * FROM sqlite_dbptr WHERE pgno=3; +} { + 3 24 3 23 +} + +do_execsql_test 2.3 { + SELECT * FROM sqlite_dbptr +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 + 3 24 3 23 +} finish_test From 68cb86ef23e3bc9e6362fec4937968a65ec025b5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Apr 2019 20:57:28 +0000 Subject: [PATCH 03/16] Add the ".recovery" command to the shell tool. For recovering the maximum amount data from corrupt databases. Still needs work. FossilOrigin-Name: 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da --- ext/misc/dbdata.c | 194 ++++++++++++++++---------------- main.mk | 1 + manifest | 18 +-- manifest.uuid | 2 +- src/shell.c.in | 277 +++++++++++++++++++++++++++++++++++++++++++--- tool/mkshellc.tcl | 8 +- 6 files changed, 375 insertions(+), 125 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index eb2bc01600..aae862fc7f 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -63,11 +63,11 @@ ** schema TEXT HIDDEN ** ); */ -#if !defined(SQLITEINT_H) +#if !defined(SQLITEINT_H) #include "sqlite3ext.h" typedef unsigned char u8; -typedef unsigned int u32; +typedef unsigned long u32; #endif SQLITE_EXTENSION_INIT1 @@ -94,8 +94,11 @@ struct DbdataCursor { /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ int nRec; /* Size of pRec[] in bytes */ - int nField; /* Number of fields in pRec */ + int nHdr; /* Size of header in bytes */ int iField; /* Current field number */ + u8 *pHdrPtr; + u8 *pPtr; + sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -306,6 +309,78 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ return 9; } +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + return ((eType-12) / 2); + } +} + +static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } +} + + /* ** Move a dbdata cursor to the next entry in the file. */ @@ -435,20 +510,23 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } } - /* Figure out how many fields in the record */ - pCsr->nField = 0; iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - while( iHdrpRec[iHdr], &iDummy); - pCsr->nField++; + pCsr->nHdr = nHdr; + pCsr->pHdrPtr = &pCsr->pRec[iHdr]; + pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->iField = (bHasRowid ? -1 : 0); + }else{ + pCsr->iField++; + if( pCsr->iField>0 ){ + sqlite3_int64 iType; + pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); + pCsr->pPtr += dbdataValueBytes(iType); } - - pCsr->iField = (bHasRowid ? -2 : -1); } - - pCsr->iField++; - if( pCsr->iFieldnField ) return SQLITE_OK; + + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + return SQLITE_OK; + } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ @@ -485,7 +563,7 @@ static int dbdataFilter( dbdataResetCursor(pCsr); assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ - zSchema = sqlite3_value_text(argv[0]); + zSchema = (const char*)sqlite3_value_text(argv[0]); } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); @@ -498,6 +576,8 @@ static int dbdataFilter( ); if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + }else{ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); @@ -505,77 +585,6 @@ static int dbdataFilter( return rc; } -static int dbdataValueBytes(int eType){ - switch( eType ){ - case 0: case 8: case 9: - case 10: case 11: - return 0; - case 1: - return 1; - case 2: - return 2; - case 3: - return 3; - case 4: - return 4; - case 5: - return 6; - case 6: - case 7: - return 8; - default: - return ((eType-12) / 2); - } -} - -static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); - } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - sqlite3_result_text(pCtx, pData, n, SQLITE_TRANSIENT); - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); - } - } - } -} - /* Return a column for the sqlite_dbdata table */ static int dbdataColumn( sqlite3_vtab_cursor *pCursor, @@ -616,18 +625,9 @@ static int dbdataColumn( if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); }else{ - int iHdr; sqlite3_int64 iType; - sqlite3_int64 iOff; - int i; - iHdr = dbdataGetVarint(pCsr->pRec, &iOff); - for(i=0; iiField; i++){ - iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - iOff += dbdataValueBytes(iType); - } - dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - - dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + dbdataGetVarint(pCsr->pHdrPtr, &iType); + dbdataValue(ctx, iType, pCsr->pPtr); } break; } diff --git a/main.mk b/main.mk index f418eec68a..508554d733 100644 --- a/main.mk +++ b/main.mk @@ -738,6 +738,7 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/dbdata.c \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/manifest b/manifest index d6a2b045e8..c103da1624 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite_dbptr\svirtual\stable\sto\sthe\sdbdata\sextension.\sFor\squerying\sthe\slinks\sbetween\sb-tree\spages. -D 2019-04-18T21:14:11.260 +C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork. +D 2019-04-20T20:57:28.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 20d85d7d7503817a5f9ac7d93c4ae7bf5fcc40570a77adc417acab0500058c99 +F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -441,7 +441,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 23d3660f7053d196aef76938bf78b10fc3ce1831a85d96bd71565758788f34d4 +F main.mk 125adda36bb32c99dc3a11340bd029ef373b9523eac2b2af76087bfe82d4fdf8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c186796810 +F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1754,7 +1754,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 49039adedafbc430d2959400da2e0e8f20ef8dcf6898e447c946e7d50ef5906b -F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5 +F tool/mkshellc.tcl d18a6769f766bd5a0fd5fd60e6bbde38ed6d1a41fced5fe32ac01ab9d2e0d5ec F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f @@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb -R 901e9e476021499846a29c847f49c578 +P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 +R 9a7e323fe9ebfe9eb0759e1c70136795 U dan -Z e0c2ac18f5bd53328c7a0485e8c7ef97 +Z 100e3ccaddf1d19d0bacd74d773a93b7 diff --git a/manifest.uuid b/manifest.uuid index 6ee3f97f4e..0b854fa40a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 \ No newline at end of file +7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index d8c57b481e..c38bfb64b9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -948,6 +948,8 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +INCLUDE ../ext/misc/dbdata.c + #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session @@ -1102,6 +1104,7 @@ struct ShellState { #define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ #define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ +#define SHFLG_Recover 0x00000080 /* .dump is --recover */ /* ** Macros for testing and setting shellFlgs @@ -3999,6 +4002,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_dbdata_init(p->db, 0, 0); #ifdef SQLITE_HAVE_ZLIB sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); @@ -6026,6 +6030,238 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + char *zErr = 0; + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", zErr); + } + *pRc = rc; + } +} + +static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +static char *shellMPrintf(int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return z; +} + +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + char *zName; /* Name of table */ + char *zQuoted; /* Quoted version of zName */ + char *zCreate; /* SQL to create table in default schema */ + int nCol; /* Number of columns in table */ + char **azlCol; /* Array of column lists */ +}; + +/* +** Free a RecoverTable object allocated by recoverNewTable() +*/ +static void recoverFreeTable(RecoverTable *pTab){ + if( pTab ){ + sqlite3_free(pTab->zName); + sqlite3_free(pTab->zQuoted); + sqlite3_free(pTab->zCreate); + if( pTab->azlCol ){ + int i; + for(i=0; inCol; i++){ + sqlite3_free(pTab->azlCol[i]); + } + sqlite3_free(pTab->azlCol); + } + sqlite3_free(pTab); + } +} + +static RecoverTable *recoverNewTable( + ShellState *pState, + int *pRc, + int iRoot, + int nCol +){ + RecoverTable *pRet = 0; + + pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pRet ){ + sqlite3_stmt *pStmt = 0; + pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); + pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); + pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); + pRet->nCol = nCol; + + shellPreparePrintf(pState->db, pRc, &pStmt, + "WITH s(i) AS (" + " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" + ")" + "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", + nCol + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int idx = sqlite3_column_int(pStmt, 0); + const char *zText = (const char*)sqlite3_column_text(pStmt, 1); + pRet->azlCol[idx] = shellMPrintf(pRc, "%s", zText); + } + shellFinalize(pRc, pStmt); + + pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", + pRet->zName, pRet->azlCol[nCol-1] + ); + } + + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pRet); + pRet = 0; + } + + return pRet; +} + +/* +** This function is called to recover data from the database. A script +** to construct a new database containing all recovered data is output +** on stream pState->out. +*/ +static int recoverDatabaseCmd(ShellState *pState){ + const char *zSql; + int rc = SQLITE_OK; + sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + + shellExec(pState->db, &rc, + /* Attach an in-memory database named 'recovery'. Create an indexed + ** cache of the sqlite_dbptr virtual table. */ + "ATTACH '' AS recovery;" + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO dbptr(pgno, child) SELECT * FROM sqlite_dbptr;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + + /* Create the "map" table that will (eventually) contain instructions + ** for dealing with each page in the db that contains one or more + ** records. */ + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);" + + /* Populate table [map]. If there are circular loops of pages in the + ** database, the following adds all pages in such a loop to the map + ** as individual root pages. This could be handled better. */ + "WITH pages(i, maxlen) AS (" + " SELECT page_count, max(field+1) " + " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" + " UNION ALL" + " SELECT * FROM (SELECT i-1, max(field+1)" + " FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)" + ")" + "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, (" + " WITH p(orig, pgno, parent) AS (" + " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" + " UNION ALL" + " SELECT i, p.parent, " + " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" + " )" + " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" + ") " + "FROM pages WHERE maxlen > 0;" + + /* Extract data from page 1 and any linked pages into table + ** recovery.schema. With the same schema as an sqlite_master table. */ + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + "INSERT INTO recovery.schema SELECT " + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=1" + ")" + "GROUP BY pgno, cell;" + ); + +#if 0 + zSql = "SELECT type ||','|| name ||','|| tbl_name ||','|| rootpage ||','|| sql FROM recovery.schema;"; + shellPrepare(pState->db, &rc, zSql, &pLoop); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + raw_printf(pState->out, "%s\n", (const char*)sqlite3_column_text(pLoop, 0)); + } + shellFinalize(&rc, pLoop); + return rc; +#endif + + /* Loop through each root page. */ + zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root"; + shellPrepare(pState->db, &rc, zSql, &pLoop); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + int iRoot = sqlite3_column_int(pLoop, 0); + int nCol = sqlite3_column_int(pLoop, 1); + RecoverTable *pTab; + + pTab = recoverNewTable(pState, &rc, iRoot, nCol); + if( pTab ){ + sqlite3_stmt *pData = 0; + raw_printf(pState->out, "%s;\n", pTab->zCreate); + shellPreparePrintf(pState->db, &rc, &pData, + "SELECT max(field), group_concat(quote(value), ', ') " + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=%d" + ")" + "GROUP BY pgno, cell;", iRoot + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ + int iMax = sqlite3_column_int(pData, 0); + const char *zVal = (const char*)sqlite3_column_text(pData, 1); + if( iMax+1==pTab->nCol ){ + raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", + pTab->zQuoted, zVal); + }else{ + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax], zVal + ); + } + } + shellFinalize(&rc, pData); + } + recoverFreeTable(pTab); + } + shellFinalize(&rc, pLoop); + + sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; +} + /* ** If an input line begins with "." then invoke this routine to @@ -6313,6 +6549,11 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = shell_dbinfo_command(p, nArg, azArg); }else + if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + open_db(p, 0); + rc = recoverDatabaseCmd(p); + }else + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0; int i; @@ -6350,7 +6591,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zLike = azArg[i]; } } + open_db(p, 0); + /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ @@ -6365,30 +6608,30 @@ static int do_meta_command(char *zLine, ShellState *p){ p->nErr = 0; if( zLike==0 ){ run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" + ); run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE name=='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE name=='sqlite_sequence'" + ); run_table_dump_query(p, - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 - ); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); }else{ char *zSql; zSql = sqlite3_mprintf( - "SELECT name, type, sql FROM sqlite_master " - "WHERE tbl_name LIKE %Q AND type=='table'" - " AND sql NOT NULL", zLike); + "SELECT name, type, sql FROM sqlite_master " + "WHERE tbl_name LIKE %Q AND type=='table'" + " AND sql NOT NULL", zLike); run_schema_dump_query(p,zSql); sqlite3_free(zSql); zSql = sqlite3_mprintf( - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL" - " AND type IN ('index','trigger','view')" - " AND tbl_name LIKE %Q", zLike); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name LIKE %Q", zLike); run_table_dump_query(p, zSql, 0); sqlite3_free(zSql); } @@ -6398,7 +6641,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); - raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n"); + raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; }else diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 534ac6156a..73eacdb9da 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -40,16 +40,21 @@ proc omit_redundant_typedefs {line} { } return $line } +set iLine 0 while {1} { set lx [omit_redundant_typedefs [gets $in]] if {[eof $in]} break; + incr iLine if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] puts $out "/************************* Begin $cfile ******************/" + puts $out "#line 1 \"$cfile\"" set in2 [open $topdir/src/$cfile rb] while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] - if {[regexp {^#include "sqlite} $lx]} continue + if {[regexp {^#include "sqlite} $lx]} { + set lx "/* $lx */" + } if {[regexp {^# *include "test_windirent.h"} $lx]} { set lx "/* $lx */" } @@ -58,6 +63,7 @@ while {1} { } close $in2 puts $out "/************************* End $cfile ********************/" + puts $out "#line [expr $iLine+1] \"shell.c.in\"" continue } puts $out $lx From b40af49d735f7ea8111bf37dab5c3f041e549290 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Apr 2019 20:52:12 +0000 Subject: [PATCH 04/16] Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c. FossilOrigin-Name: f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 --- ext/misc/dbdata.c | 1 + manifest | 16 ++-- manifest.uuid | 2 +- src/shell.c.in | 207 ++++++++++++++++++++++++++++++++++++++++------ test/dbdata.test | 10 +++ 5 files changed, 201 insertions(+), 35 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index aae862fc7f..7662417ae5 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -506,6 +506,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; + pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } } diff --git a/manifest b/manifest index c103da1624..b830ce2b1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork. -D 2019-04-20T20:57:28.136 +C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c. +D 2019-04-22T20:52:12.850 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054 +F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680 +F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 +F test/dbdata.test c8d97bafd1b2efb1e445871c4641208dcd91e686d2dfbb6463d83934adbd1ac5 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 -R 9a7e323fe9ebfe9eb0759e1c70136795 +P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da +R 14050e9b3d7e7a784dc8a127e6275222 U dan -Z 100e3ccaddf1d19d0bacd74d773a93b7 +Z abcbf2aebf4cbfcbf65f71e558364487 diff --git a/manifest.uuid b/manifest.uuid index 0b854fa40a..65b5fdd250 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da \ No newline at end of file +f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c38bfb64b9..abba5b0168 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6073,9 +6073,9 @@ typedef struct RecoverTable RecoverTable; struct RecoverTable { char *zName; /* Name of table */ char *zQuoted; /* Quoted version of zName */ - char *zCreate; /* SQL to create table in default schema */ int nCol; /* Number of columns in table */ char **azlCol; /* Array of column lists */ + int iPk; }; /* @@ -6085,7 +6085,6 @@ static void recoverFreeTable(RecoverTable *pTab){ if( pTab ){ sqlite3_free(pTab->zName); sqlite3_free(pTab->zQuoted); - sqlite3_free(pTab->zCreate); if( pTab->azlCol ){ int i; for(i=0; inCol; i++){ @@ -6097,17 +6096,141 @@ static void recoverFreeTable(RecoverTable *pTab){ } } +static void recoverOldTable( + int *pRc, /* IN/OUT: Error code */ + RecoverTable *pTab, + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + + pTab->zName = shellMPrintf(&rc, "%s", zName); + pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol); + pTab->nCol = nSqlCol; + + if( nSqlCol==1 && pTab->iPk==0 ){ + pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); + }else{ + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT -1+row_number() OVER (ORDER BY cid)," + " %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) " + " OVER (ORDER BY cid) " + "FROM pragma_table_info(%Q)", + (bIntkey ? zPk : ""), (bIntkey ? ", " : ""), + pTab->iPk, zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int idx = sqlite3_column_int(pStmt, 0); + const char *zText = (const char*)sqlite3_column_text(pStmt, 1); + pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText); + } + shellFinalize(&rc, pStmt); + } + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; +} + static RecoverTable *recoverNewTable( ShellState *pState, int *pRc, int iRoot, + int bIntkey, int nCol ){ + sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pRet ){ + if( pRet ) pRet->iPk = -2; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol); + break; + } + } + shellFinalize(pRc, pStmt); + if( bNoop ){ + sqlite3_free(pRet); + return 0; + } + + if( pRet && pRet->zName==0 ){ sqlite3_stmt *pStmt = 0; + pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); @@ -6117,8 +6240,8 @@ static RecoverTable *recoverNewTable( "WITH s(i) AS (" " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" ")" - "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", - nCol + "SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", + nCol, (bIntkey ? "id, " : "") ); while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int idx = sqlite3_column_int(pStmt, 0); @@ -6127,9 +6250,15 @@ static RecoverTable *recoverNewTable( } shellFinalize(pRc, pStmt); - pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", + if( *pRc==SQLITE_OK ){ + char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)", pRet->zName, pRet->azlCol[nCol-1] - ); + ); + if( zCreate ){ + raw_printf(pState->out, "%s;\n", zCreate); + sqlite3_free(zCreate); + } + } } if( *pRc!=SQLITE_OK ){ @@ -6146,7 +6275,6 @@ static RecoverTable *recoverNewTable( ** on stream pState->out. */ static int recoverDatabaseCmd(ShellState *pState){ - const char *zSql; int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ @@ -6173,7 +6301,9 @@ static int recoverDatabaseCmd(ShellState *pState){ /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);" + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" /* Populate table [map]. If there are circular loops of pages in the ** database, the following adds all pages in such a loop to the map @@ -6182,10 +6312,12 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT page_count, max(field+1) " " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" " UNION ALL" - " SELECT * FROM (SELECT i-1, max(field+1)" - " FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" ")" - "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, (" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" " WITH p(orig, pgno, parent) AS (" " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" " UNION ALL" @@ -6195,6 +6327,9 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ") " "FROM pages WHERE maxlen > 0;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" /* Extract data from page 1 and any linked pages into table ** recovery.schema. With the same schema as an sqlite_master table. */ @@ -6221,36 +6356,53 @@ static int recoverDatabaseCmd(ShellState *pState){ return rc; #endif + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + raw_printf(pState->out, "BEGIN;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' " + " AND length(sql)>6" + " AND sql LIKE 'create table%'" + " AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "%s;\n", zCreateTable); + } + shellFinalize(&rc, pStmt); + } + /* Loop through each root page. */ - zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root"; - shellPrepare(pState->db, &rc, zSql, &pLoop); + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey", &pLoop + ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ int iRoot = sqlite3_column_int(pLoop, 0); - int nCol = sqlite3_column_int(pLoop, 1); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, nCol); + pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ sqlite3_stmt *pData = 0; - raw_printf(pState->out, "%s;\n", pTab->zCreate); shellPreparePrintf(pState->db, &rc, &pData, "SELECT max(field), group_concat(quote(value), ', ') " "FROM sqlite_dbdata WHERE pgno IN (" " SELECT pgno FROM recovery.map WHERE root=%d" ")" - "GROUP BY pgno, cell;", iRoot + " AND field!=%d " + "GROUP BY pgno, cell;", iRoot, pTab->iPk ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ int iMax = sqlite3_column_int(pData, 0); const char *zVal = (const char*)sqlite3_column_text(pData, 1); - if( iMax+1==pTab->nCol ){ - raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", - pTab->zQuoted, zVal); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax], zVal - ); - } + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + ); } shellFinalize(&rc, pData); } @@ -6258,6 +6410,9 @@ static int recoverDatabaseCmd(ShellState *pState){ } shellFinalize(&rc, pLoop); + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "COMMIT;\n"); + } sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); return rc; } diff --git a/test/dbdata.test b/test/dbdata.test index 5a0df5dc86..391b278025 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -67,6 +67,16 @@ do_execsql_test 1.3 { SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +do_execsql_test 1.4 { + DELETE FROM t1; + INSERT INTO t1 VALUES(NULL, randomblob(5050)); +} +do_test 1.5 { + execsql { + SELECT quote(value) FROM sqlite_dbdata WHERE pgno=2 AND cell=0 AND field=1; + } +} [db one {SELECT quote(b) FROM t1}] + #------------------------------------------------------------------------- reset_db db enable_load_extension 1 From 38f9c7194df5d9c86cbeaa04c22bdee56c3c5635 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Apr 2019 18:03:02 +0000 Subject: [PATCH 05/16] Fixes for the ".recover" shell command. FossilOrigin-Name: 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 --- manifest | 13 +++---- manifest.uuid | 2 +- src/shell.c.in | 47 +++++++++++++++++++++---- test/recover.test | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 test/recover.test diff --git a/manifest b/manifest index b830ce2b1e..509212bede 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c. -D 2019-04-22T20:52:12.850 +C Fixes\sfor\sthe\s".recover"\sshell\scommand. +D 2019-04-23T18:03:02.318 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33 +F src/shell.c.in 6c02cbd1de3b878abc04b08f2f74cc4d65d0a8e52dd3e78194db2a79d686b841 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,6 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 +F test/recover.test a8fed5acf2742268e366abb76a01de6ddce278aae01658488b950c968ebef638 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1820,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da -R 14050e9b3d7e7a784dc8a127e6275222 +P f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 +R 73bcd5cc8a48367ff5a018a5ccb0a79d U dan -Z abcbf2aebf4cbfcbf65f71e558364487 +Z e408dc486ac689ec1d61c890dc79708f diff --git a/manifest.uuid b/manifest.uuid index 65b5fdd250..680d92aac8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 \ No newline at end of file +8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index abba5b0168..8ac87fbfa2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6113,6 +6113,9 @@ static void recoverOldTable( sqlite3_stmt *pStmt = 0; rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); if( rc==SQLITE_ERROR ){ @@ -6361,16 +6364,16 @@ static int recoverDatabaseCmd(ShellState *pState){ if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt = 0; raw_printf(pState->out, "BEGIN;\n"); - shellPrepare(pState->db, &rc, + raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); + shellPrepare(pState->db, &rc, "SELECT sql FROM recovery.schema " - "WHERE type='table' " - " AND length(sql)>6" - " AND sql LIKE 'create table%'" - " AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt + "WHERE type='table' AND sql LIKE 'create table%'", &pStmt ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "%s;\n", zCreateTable); + raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", + &zCreateTable[12] + ); } shellFinalize(&rc, pStmt); } @@ -6378,7 +6381,9 @@ static int recoverDatabaseCmd(ShellState *pState){ /* Loop through each root page. */ shellPrepare(pState->db, &rc, "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey", &pLoop + " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" + " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" + ")", &pLoop ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ int iRoot = sqlite3_column_int(pLoop, 0); @@ -6389,6 +6394,9 @@ static int recoverDatabaseCmd(ShellState *pState){ pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ sqlite3_stmt *pData = 0; + if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } shellPreparePrintf(pState->db, &rc, &pData, "SELECT max(field), group_concat(quote(value), ', ') " "FROM sqlite_dbdata WHERE pgno IN (" @@ -6410,7 +6418,32 @@ static int recoverDatabaseCmd(ShellState *pState){ } shellFinalize(&rc, pLoop); + /* The rest of the schema */ if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + shellPrepare(pState->db, &rc, + "SELECT sql, name FROM recovery.schema " + "WHERE (type='table' AND sql LIKE 'create table%') IS NOT TRUE", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + char *zPrint = shellMPrintf(&rc, + "INSERT INTO sqlite_master VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + ); + raw_printf(pState->out, "%s;\n", zPrint); + sqlite3_free(zPrint); + }else{ + raw_printf(pState->out, "%s;\n", zSql); + } + } + shellFinalize(&rc, pStmt); + } + + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); raw_printf(pState->out, "COMMIT;\n"); } sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); diff --git a/test/recover.test b/test/recover.test new file mode 100644 index 0000000000..6ee35054f8 --- /dev/null +++ b/test/recover.test @@ -0,0 +1,88 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the shell tool ".ar" command. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix recover + +ifcapable !vtab { + finish_test; return +} +set CLI [test_find_cli] + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn} { + set fd [open "|$::CLI test.db .recover"] + fconfigure $fd -encoding binary + fconfigure $fd -translation binary + set sql [read $fd] + close $fd + + forcedelete test.db2 + sqlite3 db2 test.db2 + breakpoint + execsql $sql db2 + uplevel [list do_test $tn [list compare_dbs db db2] {}] + db2 close +} + +set doc { + hello + world +} +do_execsql_test 1.1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 4, X'1234567800'); + INSERT INTO t1 VALUES(2, 'test', 8.1); + INSERT INTO t1 VALUES(3, $doc, 8.4); +} +do_recover_test 1.1.2 + +do_execsql_test 1.2.1 " + DELETE FROM t1; + INSERT INTO t1 VALUES(13, 'hello\r\nworld', 13); +" +do_recover_test 1.2.2 + +do_execsql_test 1.3.1 " + CREATE TABLE t2(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t2 VALUES(NULL, 1, 2); + INSERT INTO t2 VALUES(NULL, 3, 4); + INSERT INTO t2 VALUES(NULL, 5, 6); + CREATE TABLE t3(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t3 VALUES(NULL, 1, 2); + INSERT INTO t3 VALUES(NULL, 3, 4); + INSERT INTO t3 VALUES(NULL, 5, 6); + DELETE FROM t2; +" +do_recover_test 1.3.2 + +finish_test From b182588c8e0cdbafc5987df0a919d29021d1ba58 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Apr 2019 20:48:32 +0000 Subject: [PATCH 06/16] Have ".recover" handle "\r" and "\n" in the same way as ".dump". FossilOrigin-Name: f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 --- manifest | 12 +++--- manifest.uuid | 2 +- src/shell.c.in | 106 +++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 509212bede..e3ab13575f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sthe\s".recover"\sshell\scommand. -D 2019-04-23T18:03:02.318 +C Have\s".recover"\shandle\s"\\r"\sand\s"\\n"\sin\sthe\ssame\sway\sas\s".dump". +D 2019-04-23T20:48:32.366 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 6c02cbd1de3b878abc04b08f2f74cc4d65d0a8e52dd3e78194db2a79d686b841 +F src/shell.c.in 6e56c60640410885a8c3c264971619407ae5c7f46165e2bfe5fb1ca1a6a58ad0 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 -R 73bcd5cc8a48367ff5a018a5ccb0a79d +P 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 +R b9c18dd1095e3f7b028bdaea88f9fcb0 U dan -Z e408dc486ac689ec1d61c890dc79708f +Z edabfc3afbc2656b0ce1a8fb6b1e492b diff --git a/manifest.uuid b/manifest.uuid index 680d92aac8..fdcc68f9a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 \ No newline at end of file +f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8ac87fbfa2..7989c9dfde 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3934,6 +3934,96 @@ readHexDb_error: } #endif /* SQLITE_ENABLE_DESERIALIZE */ +/* +** Scalar function "shell_escape_crnl" used by the .recover command. +** The argument passed to this function is the output of built-in +** function quote(). If the first character of the input is "'", +** indicating that the value passed to quote() was a text value, +** then this function searches the input for "\n" and "\r" characters +** and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void shellEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+12)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -4013,6 +4103,8 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, + shellEscapeCrnl, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); @@ -6349,16 +6441,6 @@ static int recoverDatabaseCmd(ShellState *pState){ "GROUP BY pgno, cell;" ); -#if 0 - zSql = "SELECT type ||','|| name ||','|| tbl_name ||','|| rootpage ||','|| sql FROM recovery.schema;"; - shellPrepare(pState->db, &rc, zSql, &pLoop); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - raw_printf(pState->out, "%s\n", (const char*)sqlite3_column_text(pLoop, 0)); - } - shellFinalize(&rc, pLoop); - return rc; -#endif - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" ** CREATE TABLE statements that extracted from the existing schema. */ if( rc==SQLITE_OK ){ @@ -6398,7 +6480,7 @@ static int recoverDatabaseCmd(ShellState *pState){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } shellPreparePrintf(pState->db, &rc, &pData, - "SELECT max(field), group_concat(quote(value), ', ') " + "SELECT max(field), group_concat(shell_escape_crnl(quote(value)),', ')" "FROM sqlite_dbdata WHERE pgno IN (" " SELECT pgno FROM recovery.map WHERE root=%d" ")" @@ -6423,7 +6505,7 @@ static int recoverDatabaseCmd(ShellState *pState){ sqlite3_stmt *pStmt = 0; shellPrepare(pState->db, &rc, "SELECT sql, name FROM recovery.schema " - "WHERE (type='table' AND sql LIKE 'create table%') IS NOT TRUE", &pStmt + "WHERE sql NOT LIKE 'create table%'", &pStmt ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); From efa363b84c43d63c706e7997085a2e5977aac64d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Apr 2019 20:48:55 +0000 Subject: [PATCH 07/16] Improve the performance of the .recover command. FossilOrigin-Name: a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 --- ext/misc/dbdata.c | 60 +++++++++++++++++++++++++++++++++++------------ manifest | 14 +++++------ manifest.uuid | 2 +- src/shell.c.in | 42 ++++++++++++++++++++------------- 4 files changed, 79 insertions(+), 39 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 7662417ae5..13bb51f08b 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -106,6 +106,7 @@ struct DbdataCursor { struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ + sqlite3_stmt *pStmt; /* For fetching database pages */ int bPtr; /* True for sqlite3_dbptr table */ }; @@ -167,7 +168,11 @@ static int dbdataConnect( ** Disconnect from or destroy a dbdata virtual table. */ static int dbdataDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); + DbdataTable *pTab = (DbdataTable*)pVtab; + if( pTab ){ + sqlite3_finalize(pTab->pStmt); + sqlite3_free(pVtab); + } return SQLITE_OK; } @@ -185,15 +190,15 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ ** If both parameters are present, schema is in position 0 and pgno in ** position 1. */ -static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; @@ -206,14 +211,29 @@ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } if( iSchema>=0 ){ - pIdxInfo->aConstraintUsage[iSchema].argvIndex = 1; - pIdxInfo->aConstraintUsage[iSchema].omit = 1; + pIdx->aConstraintUsage[iSchema].argvIndex = 1; + pIdx->aConstraintUsage[iSchema].omit = 1; } if( iPgno>=0 ){ - pIdxInfo->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); - pIdxInfo->aConstraintUsage[iPgno].omit = 1; + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdx->aConstraintUsage[iPgno].omit = 1; + pIdx->estimatedCost = 100; + pIdx->estimatedRows = 50; + + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ + int iCol = pIdx->aOrderBy[0].iColumn; + if( pIdx->nOrderBy==1 ){ + pIdx->orderByConsumed = (iCol==0 || iCol==1); + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); + } + } + + }else{ + pIdx->estimatedCost = 100000000; + pIdx->estimatedRows = 1000000000; } - pIdxInfo->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); return SQLITE_OK; } @@ -236,7 +256,12 @@ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ } static void dbdataResetCursor(DbdataCursor *pCsr){ - sqlite3_finalize(pCsr->pStmt); + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); + if( pTab->pStmt==0 ){ + pTab->pStmt = pCsr->pStmt; + }else{ + sqlite3_finalize(pCsr->pStmt); + } pCsr->pStmt = 0; pCsr->iPgno = 1; pCsr->iCell = 0; @@ -558,7 +583,7 @@ static int dbdataFilter( ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; - int rc; + int rc = SQLITE_OK; const char *zSchema = "main"; dbdataResetCursor(pCsr); @@ -571,10 +596,15 @@ static int dbdataFilter( pCsr->bOnePage = 1; } - rc = sqlite3_prepare_v2(pTab->db, - "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, - &pCsr->pStmt, 0 - ); + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); }else{ diff --git a/manifest b/manifest index e3ab13575f..039fa63685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\s".recover"\shandle\s"\\r"\sand\s"\\n"\sin\sthe\ssame\sway\sas\s".dump". -D 2019-04-23T20:48:32.366 +C Improve\sthe\sperformance\sof\sthe\s.recover\scommand. +D 2019-04-24T20:48:55.802 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8 +F ext/misc/dbdata.c 0c80f0757c3a1b5c57027328ab0ccfe30fcef1a4875b41dd9ddc0f53c7087e97 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 6e56c60640410885a8c3c264971619407ae5c7f46165e2bfe5fb1ca1a6a58ad0 +F src/shell.c.in 569ee1f79236f19b44fb4fcb8a0a2f0b2dce9a3d856d41a196aaaaa5f0d37704 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 -R b9c18dd1095e3f7b028bdaea88f9fcb0 +P f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 +R 0e7ee563b8599ed64c48e30038f5f99f U dan -Z edabfc3afbc2656b0ce1a8fb6b1e492b +Z f093e1d7a99537670d8620da5d0b299b diff --git a/manifest.uuid b/manifest.uuid index fdcc68f9a9..3f61303065 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 \ No newline at end of file +a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 7989c9dfde..05ce02d605 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6372,6 +6372,8 @@ static RecoverTable *recoverNewTable( static int recoverDatabaseCmd(ShellState *pState){ int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ + sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ shellExec(pState->db, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed @@ -6460,6 +6462,15 @@ static int recoverDatabaseCmd(ShellState *pState){ shellFinalize(&rc, pStmt); } + shellPrepare(pState->db, &rc, + "SELECT pgno FROM recovery.map WHERE root=?", &pPages + ); + shellPrepare(pState->db, &rc, + "SELECT max(field), group_concat(shell_escape_crnl(quote(value)), ', ')" + "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" + "GROUP BY cell", &pCells + ); + /* Loop through each root page. */ shellPrepare(pState->db, &rc, "SELECT root, intkey, max(maxlen) FROM recovery.map" @@ -6475,30 +6486,29 @@ static int recoverDatabaseCmd(ShellState *pState){ pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ - sqlite3_stmt *pData = 0; if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } - shellPreparePrintf(pState->db, &rc, &pData, - "SELECT max(field), group_concat(shell_escape_crnl(quote(value)),', ')" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=%d" - ")" - " AND field!=%d " - "GROUP BY pgno, cell;", iRoot, pTab->iPk - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ - int iMax = sqlite3_column_int(pData, 0); - const char *zVal = (const char*)sqlite3_column_text(pData, 1); - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal - ); + sqlite3_bind_int(pPages, 1, iRoot); + sqlite3_bind_int(pCells, 2, pTab->iPk); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + sqlite3_bind_int(pCells, 1, sqlite3_column_int(pPages, 0)); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int iMax = sqlite3_column_int(pCells, 0); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + ); + } + shellReset(&rc, pCells); } - shellFinalize(&rc, pData); + shellReset(&rc, pPages); } recoverFreeTable(pTab); } shellFinalize(&rc, pLoop); + shellFinalize(&rc, pPages); + shellFinalize(&rc, pCells); /* The rest of the schema */ if( rc==SQLITE_OK ){ From b9b71dbfd4066df5d62fd59c1f5350faca74fcd4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2019 16:20:40 +0000 Subject: [PATCH 08/16] Fix a bug preventing .recover from working on databases where the final page of the db is corrupt. FossilOrigin-Name: 959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 039fa63685..2701068663 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\s.recover\scommand. -D 2019-04-24T20:48:55.802 +C Fix\sa\sbug\spreventing\s.recover\sfrom\sworking\son\sdatabases\swhere\sthe\sfinal\spage\sof\sthe\sdb\sis\scorrupt. +D 2019-04-25T16:20:40.499 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 569ee1f79236f19b44fb4fcb8a0a2f0b2dce9a3d856d41a196aaaaa5f0d37704 +F src/shell.c.in 2e9b6b05fd202499bbe233632f9e5a913d2afcd645b59a114bfeb440c3e5cd38 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 -R 0e7ee563b8599ed64c48e30038f5f99f +P a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 +R 9da7af39576825c9e597192b24527e0d U dan -Z f093e1d7a99537670d8620da5d0b299b +Z 0de6a10b348dda1bb213a1119e4cc094 diff --git a/manifest.uuid b/manifest.uuid index 3f61303065..e18c3e1cac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 \ No newline at end of file +959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 05ce02d605..73bf9dff97 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6369,7 +6369,7 @@ static RecoverTable *recoverNewTable( ** to construct a new database containing all recovered data is output ** on stream pState->out. */ -static int recoverDatabaseCmd(ShellState *pState){ +static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ @@ -6406,8 +6406,9 @@ static int recoverDatabaseCmd(ShellState *pState){ ** database, the following adds all pages in such a loop to the map ** as individual root pages. This could be handled better. */ "WITH pages(i, maxlen) AS (" - " SELECT page_count, max(field+1) " - " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" + " SELECT page_count, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" + " ) FROM pragma_page_count" " UNION ALL" " SELECT i-1, (" " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" @@ -6831,7 +6832,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); - rc = recoverDatabaseCmd(p); + rc = recoverDatabaseCmd(p, nArg, azArg); }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ From 9c014f8b0c63b3b8c8c2ae30dd9c9cbe1b883952 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2019 19:23:15 +0000 Subject: [PATCH 09/16] Unless the "--freelist-corrupt" option is specified, do not have the .recover command attempt to recover data from pages that are on the database free-list. FossilOrigin-Name: 8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd --- manifest | 12 ++++---- manifest.uuid | 2 +- src/shell.c.in | 78 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2701068663..f7a1f6f81c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\spreventing\s.recover\sfrom\sworking\son\sdatabases\swhere\sthe\sfinal\spage\sof\sthe\sdb\sis\scorrupt. -D 2019-04-25T16:20:40.499 +C Unless\sthe\s"--freelist-corrupt"\soption\sis\sspecified,\sdo\snot\shave\sthe\s.recover\scommand\sattempt\sto\srecover\sdata\sfrom\spages\sthat\sare\son\sthe\sdatabase\sfree-list. +D 2019-04-25T19:23:15.197 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 2e9b6b05fd202499bbe233632f9e5a913d2afcd645b59a114bfeb440c3e5cd38 +F src/shell.c.in a3b1bb23c3c3dde9aab1823dc37cb0f3e81debfd3937e40ee8fa69fe1f0e4c16 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 -R 9da7af39576825c9e597192b24527e0d +P 959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 +R d86f5b916c733bd261b32efe2b58444b U dan -Z 0de6a10b348dda1bb213a1119e4cc094 +Z 3b2dfcc08d011cfeb383f50fbff24d3e diff --git a/manifest.uuid b/manifest.uuid index e18c3e1cac..37c6cf3c7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 \ No newline at end of file +8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 73bf9dff97..95c18ae4d1 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3934,6 +3934,35 @@ readHexDb_error: } #endif /* SQLITE_ENABLE_DESERIALIZE */ +/* +** Scalar function "shell_int32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void shellInt32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + + ((sqlite3_int64)a[1]<<16) + + ((sqlite3_int64)a[2]<< 8) + + ((sqlite3_int64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + /* ** Scalar function "shell_escape_crnl" used by the .recover command. ** The argument passed to this function is the output of built-in @@ -4105,6 +4134,8 @@ static void open_db(ShellState *p, int openFlags){ shellPutsFunc, 0, 0); sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, shellEscapeCrnl, 0, 0); + sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, + shellInt32, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); @@ -6374,6 +6405,25 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + int i; + + int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + for(i=1; idb, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed @@ -6395,6 +6445,32 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" ");" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);" + ); + + if( bFreelist ){ + shellExec(pState->db, &rc, + "WITH trunk(pgno) AS (" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " + " WHERE x>0" + " UNION" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " + " FROM trunk WHERE x>0" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, shell_int32(data, 1)-1, t.pgno " + " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" + " UNION ALL" + " SELECT data, n-1, shell_int32(data, 2+n) " + " FROM freelist WHERE n>=0" + ")" + "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" + ); + } + + shellExec(pState->db, &rc, /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ @@ -6424,7 +6500,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " )" " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ") " - "FROM pages WHERE maxlen > 0;" + "FROM pages WHERE maxlen > 0 AND i NOT IN freelist;" "UPDATE recovery.map AS o SET intkey = (" " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" ");" From c0b42437ab69d01daa61bd77f74badb95c82bd40 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 15:14:53 +0000 Subject: [PATCH 10/16] Fix a locking-page related problem with the ".recover" command. FossilOrigin-Name: afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 --- ext/misc/dbdata.c | 66 +++++++++++++++++++++++++++++++++++------------ manifest | 14 +++++----- manifest.uuid | 2 +- src/shell.c.in | 42 ++++++++++++++++++++++++------ 4 files changed, 91 insertions(+), 33 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 13bb51f08b..a27f19dc7a 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -89,6 +89,7 @@ struct DbdataCursor { int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ int bOnePage; /* True to stop after one page */ + int szDb; sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ @@ -303,18 +304,21 @@ static int dbdataLoadPage( sqlite3_bind_int64(pStmt, 2, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ int nCopy = sqlite3_column_bytes(pStmt, 0); - u8 *pPage = (u8*)sqlite3_malloc64(nCopy); - if( pPage==0 ){ - rc = SQLITE_NOMEM; - }else{ - const u8 *pCopy = sqlite3_column_blob(pStmt, 0); - memcpy(pPage, pCopy, nCopy); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + } *ppPage = pPage; *pnPage = nCopy; } } rc2 = sqlite3_reset(pStmt); - if( *ppPage==0 ) rc = rc2; + if( rc==SQLITE_OK ) rc = rc2; return rc; } @@ -419,8 +423,13 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ int iOff = (pCsr->iPgno==1 ? 100 : 0); if( pCsr->aPage==0 ){ - rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK || pCsr->aPage==0 ) return rc; + while( 1 ){ + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + if( pCsr->aPage ) break; + pCsr->iPgno++; + } pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } @@ -574,6 +583,24 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; + char *zSql = 0; + int rc, rc2; + sqlite3_stmt *pStmt = 0; + + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + pCsr->szDb = sqlite3_column_int(pStmt, 0); + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + /* Position a cursor back to the beginning. */ static int dbdataFilter( @@ -594,16 +621,21 @@ static int dbdataFilter( if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; + }else{ + pCsr->nPage = dbdataDbsize(pCsr, zSchema); + rc = dbdataDbsize(pCsr, zSchema); } - if( pTab->pStmt ){ - pCsr->pStmt = pTab->pStmt; - pTab->pStmt = 0; - }else{ - rc = sqlite3_prepare_v2(pTab->db, - "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, - &pCsr->pStmt, 0 - ); + if( rc==SQLITE_OK ){ + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); diff --git a/manifest b/manifest index c6b5a05076..d361dc7ebc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2019-04-25T20:06:34.069 +C Fix\sa\slocking-page\srelated\sproblem\swith\sthe\s".recover"\scommand. +D 2019-04-26T15:14:53.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 0c80f0757c3a1b5c57027328ab0ccfe30fcef1a4875b41dd9ddc0f53c7087e97 +F ext/misc/dbdata.c b7547f43906f9296e43be807ca78e2ef3f335ca26d6e91d178df30cd2fd46572 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 78004b100ed486695f4ff8bbc40b99c2e4d7b6d06278ce64b80ac1900bf356a3 +F src/shell.c.in 63b1075817997806d7f09c2a0433ffac854b6dee47836ef938d51531cdb8042f F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd 7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf -R af342ccd780cc18bf46aaaf7890063fa +P 1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 +R 981714fd52a70e51cf4586f924a49fdc U dan -Z 147b326af09d7b23ccf3f83204577057 +Z 303a0ea3c8fcc86a61370f3e12d83ec5 diff --git a/manifest.uuid b/manifest.uuid index 0d49487589..b5f67ee780 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 \ No newline at end of file +afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 55f7964319..1884e7bfe2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6165,6 +6165,22 @@ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ } } +static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellExec(db, pRc, z); + } + sqlite3_free(z); + } +} + static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ @@ -6405,6 +6421,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ int i; int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ @@ -6416,23 +6433,32 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){ bFreelist = 0; } + if( n<=12 && memcmp("-recovery-db", z, n)==0 && i<(nArg-1) ){ + i++; + zRecoveryDb = azArg[i]; + } else{ - raw_printf(stderr, - "unexpected option: %s - expected \"--freelist-corrupt\"\n", - azArg[i] - ); + raw_printf(stderr, "unexpected option: %s\n", azArg[i]); + raw_printf(stderr, "options are:\n"); + raw_printf(stderr, " --freelist-corrupt\n"); + raw_printf(stderr, " --recovery-db DATABASE\n"); return 1; } } - shellExec(pState->db, &rc, + shellExecPrintf(pState->db, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed ** cache of the sqlite_dbptr virtual table. */ - "ATTACH '' AS recovery;" + "ATTACH %Q AS recovery;" + "DROP TABLE IF EXISTS recovery.dbptr;" + "DROP TABLE IF EXISTS recovery.freelist;" + "DROP TABLE IF EXISTS recovery.map;" + "DROP TABLE IF EXISTS recovery.schema;" "CREATE TABLE recovery.dbptr(" " pgno, child, PRIMARY KEY(child, pgno)" ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO dbptr(pgno, child) SELECT * FROM sqlite_dbptr;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr;" /* Delete any pointer to page 1. This ensures that page 1 is considered ** a root page, regardless of how corrupt the db is. */ @@ -6445,7 +6471,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" ");" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb ); if( bFreelist ){ From ca424382613ed62e80b54734231a1d1d8dd916bd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 15:40:27 +0000 Subject: [PATCH 11/16] Fix another problem with database freelist handling in the ".recover" command. FossilOrigin-Name: bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 35 ++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index d361dc7ebc..2ea5b39962 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\slocking-page\srelated\sproblem\swith\sthe\s".recover"\scommand. -D 2019-04-26T15:14:53.514 +C Fix\sanother\sproblem\swith\sdatabase\sfreelist\shandling\sin\sthe\s".recover"\scommand. +D 2019-04-26T15:40:27.902 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 63b1075817997806d7f09c2a0433ffac854b6dee47836ef938d51531cdb8042f +F src/shell.c.in 44dce9f5b3c68c07e30db740aa4e635819ab6fa01a229a21356ddd4fa8f1e47e F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 -R 981714fd52a70e51cf4586f924a49fdc +P afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 +R 5b76306d65317b1b0fee3a74fc0359d7 U dan -Z 303a0ea3c8fcc86a61370f3e12d83ec5 +Z 068a6ba7f1f2eaa523c5f432cd2b8c0c diff --git a/manifest.uuid b/manifest.uuid index b5f67ee780..4c5a229461 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 \ No newline at end of file +bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1884e7bfe2..c417160d52 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6454,23 +6454,6 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ "DROP TABLE IF EXISTS recovery.freelist;" "DROP TABLE IF EXISTS recovery.map;" "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb ); @@ -6497,6 +6480,24 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } shellExec(pState->db, &rc, + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr" + " WHERE pgno NOT IN freelist AND child NOT IN freelist;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ From 98c5ad309fce1eeea6cfe624e1dfd341abca947a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 21:11:37 +0000 Subject: [PATCH 12/16] Have .recover store all orphaned rows in a single table, with extra columns to indicate the orphaned page and sub-tree they were discovered within. FossilOrigin-Name: 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d --- manifest | 12 +-- manifest.uuid | 2 +- src/shell.c.in | 209 +++++++++++++++++++++++++++++-------------------- 3 files changed, 133 insertions(+), 90 deletions(-) diff --git a/manifest b/manifest index 2ea5b39962..8404fd782b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sproblem\swith\sdatabase\sfreelist\shandling\sin\sthe\s".recover"\scommand. -D 2019-04-26T15:40:27.902 +C Have\s.recover\sstore\sall\sorphaned\srows\sin\sa\ssingle\stable,\swith\sextra\scolumns\sto\sindicate\sthe\sorphaned\spage\sand\ssub-tree\sthey\swere\sdiscovered\swithin. +D 2019-04-26T21:11:37.615 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 44dce9f5b3c68c07e30db740aa4e635819ab6fa01a229a21356ddd4fa8f1e47e +F src/shell.c.in 146dbc2708a314556710f8e8b13226d647fbb70778c973bb6fd655186811ce63 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 -R 5b76306d65317b1b0fee3a74fc0359d7 +P bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd +R 52eb405413df1bbf1e7b369ddce62695 U dan -Z 068a6ba7f1f2eaa523c5f432cd2b8c0c +Z 87ebd735a2a8d2115b37082d2935a00e diff --git a/manifest.uuid b/manifest.uuid index 4c5a229461..3cfd1fa139 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd \ No newline at end of file +7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c417160d52..0737a347a0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6226,7 +6226,7 @@ static void recoverFreeTable(RecoverTable *pTab){ sqlite3_free(pTab->zQuoted); if( pTab->azlCol ){ int i; - for(i=0; inCol; i++){ + for(i=0; i<=pTab->nCol; i++){ sqlite3_free(pTab->azlCol[i]); } sqlite3_free(pTab->azlCol); @@ -6235,9 +6235,8 @@ static void recoverFreeTable(RecoverTable *pTab){ } } -static void recoverOldTable( +static RecoverTable *recoverOldTable( int *pRc, /* IN/OUT: Error code */ - RecoverTable *pTab, const char *zName, /* Name of table */ const char *zSql, /* CREATE TABLE statement */ int bIntkey, @@ -6245,12 +6244,14 @@ static void recoverOldTable( ){ sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ int rc = *pRc; + RecoverTable *pTab = 0; + pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); if( rc==SQLITE_OK ){ int nSqlCol = 0; int bSqlIntkey = 0; sqlite3_stmt *pStmt = 0; - + rc = sqlite3_open("", &dbtmp); if( rc==SQLITE_OK ){ rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); @@ -6285,43 +6286,48 @@ static void recoverOldTable( shellFinalize(&rc, pStmt); if( bIntkey==bSqlIntkey ){ + int i; const char *zPk = "_rowid_"; sqlite3_stmt *pPkFinder = 0; - shellPreparePrintf(dbtmp, &rc, &pPkFinder, + pTab->iPk = -2; + if( bIntkey ){ + shellPreparePrintf(dbtmp, &rc, &pPkFinder, "SELECT cid, name FROM pragma_table_info(%Q) " " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)", - zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" + , zName, zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ + pTab->iPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } } pTab->zName = shellMPrintf(&rc, "%s", zName); pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); pTab->nCol = nSqlCol; - if( nSqlCol==1 && pTab->iPk==0 ){ + if( bIntkey ){ pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); }else{ - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT -1+row_number() OVER (ORDER BY cid)," - " %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) " - " OVER (ORDER BY cid) " - "FROM pragma_table_info(%Q)", - (bIntkey ? zPk : ""), (bIntkey ? ", " : ""), - pTab->iPk, zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int idx = sqlite3_column_int(pStmt, 0); - const char *zText = (const char*)sqlite3_column_text(pStmt, 1); - pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText); - } - shellFinalize(&rc, pStmt); + pTab->azlCol[0] = shellMPrintf(&rc, ""); } + i = 1; + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT %Q || group_concat(name, ', ') " + " FILTER (WHERE cid!=%d) OVER (ORDER BY cid) " + "FROM pragma_table_info(%Q)", + bIntkey ? ", " : "", pTab->iPk, zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zText = (const char*)sqlite3_column_text(pStmt, 0); + pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); + i++; + } + shellFinalize(&rc, pStmt); + shellFinalize(&rc, pPkFinder); } } @@ -6329,6 +6335,11 @@ static void recoverOldTable( finished: sqlite3_close(dbtmp); *pRc = rc; + if( rc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + } + return pTab; } static RecoverTable *recoverNewTable( @@ -6336,7 +6347,8 @@ static RecoverTable *recoverNewTable( int *pRc, int iRoot, int bIntkey, - int nCol + int nCol, + int *pbNoop ){ sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; @@ -6344,8 +6356,6 @@ static RecoverTable *recoverNewTable( const char *zSql = 0; const char *zName = 0; - pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pRet ) pRet->iPk = -2; /* Search the recovered schema for an object with root page iRoot. */ shellPreparePrintf(pState->db, pRc, &pStmt, @@ -6360,55 +6370,56 @@ static RecoverTable *recoverNewTable( if( sqlite3_stricmp(zType, "table")==0 ){ zName = (const char*)sqlite3_column_text(pStmt, 1); zSql = (const char*)sqlite3_column_text(pStmt, 2); - recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol); + pRet = recoverOldTable(pRc, zName, zSql, bIntkey, nCol); break; } } + shellFinalize(pRc, pStmt); - if( bNoop ){ - sqlite3_free(pRet); - return 0; - } + *pbNoop = bNoop; + return pRet; +} - if( pRet && pRet->zName==0 ){ - sqlite3_stmt *pStmt = 0; - - pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); - pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); - pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); - pRet->nCol = nCol; - - shellPreparePrintf(pState->db, pRc, &pStmt, - "WITH s(i) AS (" - " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" - ")" - "SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", - nCol, (bIntkey ? "id, " : "") +static RecoverTable *recoverOrphanTable( + ShellState *pState, + int *pRc, + int nCol +){ + RecoverTable *pTab = 0; + if( nCol>=0 && *pRc==SQLITE_OK ){ + int i; + raw_printf(pState->out, + "CREATE TABLE recover_orphan(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER" ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int idx = sqlite3_column_int(pStmt, 0); - const char *zText = (const char*)sqlite3_column_text(pStmt, 1); - pRet->azlCol[idx] = shellMPrintf(pRc, "%s", zText); + for(i=0; iout, ", c%d", i); } - shellFinalize(pRc, pStmt); + raw_printf(pState->out, ");\n"); - if( *pRc==SQLITE_OK ){ - char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)", - pRet->zName, pRet->azlCol[nCol-1] - ); - if( zCreate ){ - raw_printf(pState->out, "%s;\n", zCreate); - sqlite3_free(zCreate); + pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pTab ){ + pTab->zName = shellMPrintf(pRc, "%s", "recover_orphan"); + pTab->zQuoted = shellMPrintf(pRc, "%Q", pTab->zName); + pTab->nCol = nCol; + pTab->iPk = -2; + if( nCol>0 ){ + pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); + if( pTab->azlCol ){ + pTab->azlCol[nCol] = shellMPrintf(pRc, ""); + for(i=nCol-1; i>=0; i--){ + pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); + } + } } } - } - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pRet); - pRet = 0; + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + } } - - return pRet; + return pTab; } /* @@ -6423,6 +6434,8 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ const char *zRecoveryDb = ""; /* Name of "recovery" database */ int i; + int nOrphan = -1; + RecoverTable *pOrphan = 0; int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ for(i=1; idb, &rc, + "SELECT coalesce(max(maxlen), -2) FROM recovery.map" + " WHERE root>1 AND root NOT IN (SELECT rootpage FROM recovery.schema)" + , &pLoop + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + nOrphan = sqlite3_column_int(pLoop, 0); + } + shellFinalize(&rc, pLoop); + pLoop = 0; + pOrphan = recoverOrphanTable(pState, &rc, nOrphan); + shellPrepare(pState->db, &rc, "SELECT pgno FROM recovery.map WHERE root=?", &pPages ); @@ -6586,33 +6614,48 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int iRoot = sqlite3_column_int(pLoop, 0); int bIntkey = sqlite3_column_int(pLoop, 1); int nCol = sqlite3_column_int(pLoop, 2); + int bNoop = 0; RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); - if( pTab ){ - if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); - } - sqlite3_bind_int(pPages, 1, iRoot); - sqlite3_bind_int(pCells, 2, pTab->iPk); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - sqlite3_bind_int(pCells, 1, sqlite3_column_int(pPages, 0)); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int iMax = sqlite3_column_int(pCells, 0); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + if( bNoop || rc ) continue; + if( pTab==0 ) pTab = pOrphan; + + if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } + sqlite3_bind_int(pPages, 1, iRoot); + sqlite3_bind_int(pCells, 2, pTab->iPk); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + int iPgno = sqlite3_column_int(pPages, 0); + sqlite3_bind_int(pCells, 1, iPgno); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int nField = sqlite3_column_int(pCells, 0); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + + nField = nField+1; + if( pTab==pOrphan ){ + raw_printf(pState->out, + "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", + pTab->zQuoted, iRoot, iPgno, nField, + bIntkey ? "" : "NULL, ", zVal, pTab->azlCol[nField] + ); + }else{ raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + pTab->zQuoted, pTab->azlCol[nField], zVal ); } - shellReset(&rc, pCells); } - shellReset(&rc, pPages); + shellReset(&rc, pCells); } - recoverFreeTable(pTab); + shellReset(&rc, pPages); + if( pTab!=pOrphan ) recoverFreeTable(pTab); } shellFinalize(&rc, pLoop); shellFinalize(&rc, pPages); shellFinalize(&rc, pCells); + recoverFreeTable(pOrphan); /* The rest of the schema */ if( rc==SQLITE_OK ){ From f57bea31bad6081613945a4a344e6859cd189de1 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 15:35:45 +0000 Subject: [PATCH 13/16] Fix a problem in the .recover command with recovering WITHOUT ROWID tables where the PK columns are not the leftmost in the CREATE TABLE statement. FossilOrigin-Name: 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 11 +++++++++-- test/recover.test | 11 +++++++++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8404fd782b..94b2e05ed0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\s.recover\sstore\sall\sorphaned\srows\sin\sa\ssingle\stable,\swith\sextra\scolumns\sto\sindicate\sthe\sorphaned\spage\sand\ssub-tree\sthey\swere\sdiscovered\swithin. -D 2019-04-26T21:11:37.615 +C Fix\sa\sproblem\sin\sthe\s.recover\scommand\swith\srecovering\sWITHOUT\sROWID\stables\swhere\sthe\sPK\scolumns\sare\snot\sthe\sleftmost\sin\sthe\sCREATE\sTABLE\sstatement. +D 2019-04-27T15:35:45.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 146dbc2708a314556710f8e8b13226d647fbb70778c973bb6fd655186811ce63 +F src/shell.c.in 3701177f3821330c8eb2af96f60123245cf42273abdae472bcb96bb120dcba8f F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,7 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test a8fed5acf2742268e366abb76a01de6ddce278aae01658488b950c968ebef638 +F test/recover.test bfeb5ab4574f9a264b3893ce0e41f04c2052b72b174e5dd9d847b6e7b8f4d15c F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd -R 52eb405413df1bbf1e7b369ddce62695 +P 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d +R e71b5bc56d773e53b87820c68d2be836 U dan -Z 87ebd735a2a8d2115b37082d2935a00e +Z d098c243e647f537ccde95cbf714168f diff --git a/manifest.uuid b/manifest.uuid index 3cfd1fa139..6b5834e3bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d \ No newline at end of file +91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0737a347a0..f906205bff 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6290,6 +6290,11 @@ static RecoverTable *recoverOldTable( const char *zPk = "_rowid_"; sqlite3_stmt *pPkFinder = 0; + /* If this is an intkey table and there is an INTEGER PRIMARY KEY, + ** set zPk to the name of the PK column, and pTab->iPk to the index + ** of the column, where columns are 0-numbered from left to right. + ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, + ** leave zPk as "_rowid_" and pTab->iPk at -2. */ pTab->iPk = -2; if( bIntkey ){ shellPreparePrintf(dbtmp, &rc, &pPkFinder, @@ -6317,9 +6322,11 @@ static RecoverTable *recoverOldTable( i = 1; shellPreparePrintf(dbtmp, &rc, &pStmt, "SELECT %Q || group_concat(name, ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY cid) " + " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, zName + bIntkey ? ", " : "", pTab->iPk, + bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", + zName ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zText = (const char*)sqlite3_column_text(pStmt, 0); diff --git a/test/recover.test b/test/recover.test index 6ee35054f8..4de7f503a5 100644 --- a/test/recover.test +++ b/test/recover.test @@ -85,4 +85,15 @@ do_execsql_test 1.3.1 " " do_recover_test 1.3.2 +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} + +do_recover_test 2.1.1 + finish_test From 42ebb01e9fd94c94d4f90b0a59b46194aea28400 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 18:47:03 +0000 Subject: [PATCH 14/16] Add the "--lost-and-found" option to the ".recover" command. For setting the name of the orphaned rows table. FossilOrigin-Name: 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 --- ext/misc/dbdata.c | 131 ++++++++++++++++++++++++++-------------- manifest | 16 ++--- manifest.uuid | 2 +- src/shell.c.in | 150 ++++++++++++++++++++++++++++++++-------------- test/recover.test | 36 ++++++++++- 5 files changed, 234 insertions(+), 101 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index a27f19dc7a..786369ce0a 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -10,10 +10,14 @@ ** ****************************************************************************** ** -** This file contains an implementation of the eponymous "sqlite_dbdata" -** virtual table. sqlite_dbdata is used to extract data directly from a -** database b-tree page and its associated overflow pages, bypassing the b-tree -** layer. The table schema is equivalent to: +** This file contains an implementation of two eponymous virtual tables, +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the +** "sqlite_dbpage" eponymous virtual table be available. +** +** SQLITE_DBDATA: +** sqlite_dbdata is used to extract data directly from a database b-tree +** page and its associated overflow pages, bypassing the b-tree layer. +** The table schema is equivalent to: ** ** CREATE TABLE sqlite_dbdata( ** pgno INTEGER, @@ -23,23 +27,25 @@ ** schema TEXT HIDDEN ** ); ** -** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE -** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND "schema". +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND +** "schema". ** -** Each page of the database is inspected. If it cannot be interpreted as a -** b-tree page, or if it is a b-tree page containing 0 entries, the -** sqlite_dbdata table contains no rows for that page. Otherwise, the table -** contains one row for each field in the record associated with each -** cell on the page. For intkey b-trees, the key value is stored in field -1. +** Each page of the database is inspected. If it cannot be interpreted as +** a b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the +** table contains one row for each field in the record associated with +** each cell on the page. For intkey b-trees, the key value is stored in +** field -1. ** -** For example, for the database: +** For example, for the database: ** ** CREATE TABLE t1(a, b); -- root page is page 2 ** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); ** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); ** -** the sqlite_dbdata table contains, as well as from entries related to -** page 1, content equivalent to: +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: ** ** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES ** (2, 0, -1, 5 ), @@ -49,19 +55,21 @@ ** (2, 1, 0, 'x' ), ** (2, 1, 1, 'ten' ); ** -** If database corruption is encountered, this module does not report an -** error. Instead, it attempts to extract as much data as possible and -** ignores the corruption. -** -** This module requires that the "sqlite_dbpage" eponymous virtual table be -** available. +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. ** +** SQLITE_DBPTR: +** The sqlite_dbptr table has the following schema: ** ** CREATE TABLE sqlite_dbptr( ** pgno INTEGER, ** child INTEGER, ** schema TEXT HIDDEN ** ); +** +** It contains one entry for each b-tree pointer between a parent and +** child page in the database. */ #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -77,8 +85,7 @@ SQLITE_EXTENSION_INIT1 typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; - -/* A cursor for the sqlite_dbdata table */ +/* Cursor object */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ sqlite3_stmt *pStmt; /* For fetching database pages */ @@ -103,7 +110,7 @@ struct DbdataCursor { sqlite3_int64 iIntkey; /* Integer key value */ }; -/* The sqlite_dbdata table */ +/* Table object */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ @@ -111,16 +118,12 @@ struct DbdataTable { int bPtr; /* True for sqlite3_dbptr table */ }; +/* Column and schema definitions for sqlite_dbdata */ #define DBDATA_COLUMN_PGNO 0 #define DBDATA_COLUMN_CELL 1 #define DBDATA_COLUMN_FIELD 2 #define DBDATA_COLUMN_VALUE 3 #define DBDATA_COLUMN_SCHEMA 4 - -#define DBPTR_COLUMN_PGNO 0 -#define DBPTR_COLUMN_CHILD 1 -#define DBPTR_COLUMN_SCHEMA 2 - #define DBDATA_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -130,6 +133,10 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +/* Column and schema definitions for sqlite_dbptr */ +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 #define DBPTR_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -138,7 +145,8 @@ struct DbdataTable { ")" /* -** Connect to the sqlite_dbdata virtual table. +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual +** table. */ static int dbdataConnect( sqlite3 *db, @@ -166,7 +174,7 @@ static int dbdataConnect( } /* -** Disconnect from or destroy a dbdata virtual table. +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. */ static int dbdataDisconnect(sqlite3_vtab *pVtab){ DbdataTable *pTab = (DbdataTable*)pVtab; @@ -178,7 +186,6 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ } /* -** ** This function interprets two types of constraints: ** ** schema=? @@ -239,7 +246,7 @@ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ } /* -** Open a new dbdata cursor. +** Open a new sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ DbdataCursor *pCsr; @@ -256,6 +263,10 @@ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* +** Restore a cursor object to the state it was in when first allocated +** by dbdataOpen(). +*/ static void dbdataResetCursor(DbdataCursor *pCsr){ DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); if( pTab->pStmt==0 ){ @@ -271,7 +282,7 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ } /* -** Close a dbdata cursor. +** Close an sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -280,8 +291,9 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } - -/* Decode big-endian integers */ +/* +** Utility methods to decode 16 and 32-bit big-endian unsigned integers. +*/ static unsigned int get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } @@ -289,11 +301,21 @@ static unsigned int get_uint32(unsigned char *a){ return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; } +/* +** Load page pgno from the database via the sqlite_dbpage virtual table. +** If successful, set (*ppPage) to point to a buffer containing the page +** data, (*pnPage) to the size of that buffer in bytes and return +** SQLITE_OK. In this case it is the responsibility of the caller to +** eventually free the buffer using sqlite3_free(). +** +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and +** return an SQLite error code. +*/ static int dbdataLoadPage( - DbdataCursor *pCsr, - u32 pgno, - u8 **ppPage, - int *pnPage + DbdataCursor *pCsr, /* Cursor object */ + u32 pgno, /* Page number of page to load */ + u8 **ppPage, /* OUT: pointer to page buffer */ + int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ int rc2; int rc = SQLITE_OK; @@ -338,6 +360,10 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ return 9; } +/* +** Return the number of bytes of space used by an SQLite value of type +** eType. +*/ static int dbdataValueBytes(int eType){ switch( eType ){ case 0: case 8: case 9: @@ -361,6 +387,10 @@ static int dbdataValueBytes(int eType){ } } +/* +** Load a value of type eType from buffer pData and use it to set the +** result of context object pCtx. +*/ static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ switch( eType ){ case 0: @@ -411,7 +441,7 @@ static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ /* -** Move a dbdata cursor to the next entry in the file. +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -575,14 +605,20 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -/* We have reached EOF if previous sqlite3_step() returned -** anything other than SQLITE_ROW; +/* +** Return true if the cursor is at EOF. */ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; return pCsr->aPage==0; } +/* +** Determine the size in pages of database zSchema (where zSchema is +** "main", "temp" or the name of an attached database) and set +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, +** an SQLite error code. +*/ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; @@ -601,7 +637,8 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } -/* Position a cursor back to the beginning. +/* +** xFilter method for sqlite_dbdata and sqlite_dbptr. */ static int dbdataFilter( sqlite3_vtab_cursor *pCursor, @@ -648,7 +685,9 @@ static int dbdataFilter( return rc; } -/* Return a column for the sqlite_dbdata table */ +/* +** Return a column for the sqlite_dbdata or sqlite_dbptr table. +*/ static int dbdataColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, @@ -699,7 +738,9 @@ static int dbdataColumn( return SQLITE_OK; } -/* Return the ROWID for the sqlite_dbdata table */ +/* +** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +*/ static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; *pRowid = pCsr->iRowid; diff --git a/manifest b/manifest index 94b2e05ed0..2f2f86a8fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\s.recover\scommand\swith\srecovering\sWITHOUT\sROWID\stables\swhere\sthe\sPK\scolumns\sare\snot\sthe\sleftmost\sin\sthe\sCREATE\sTABLE\sstatement. -D 2019-04-27T15:35:45.828 +C Add\sthe\s"--lost-and-found"\soption\sto\sthe\s".recover"\scommand.\sFor\ssetting\sthe\sname\sof\sthe\sorphaned\srows\stable. +D 2019-04-27T18:47:03.466 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c b7547f43906f9296e43be807ca78e2ef3f335ca26d6e91d178df30cd2fd46572 +F ext/misc/dbdata.c fe978dad2df13dd4b377b5d38f4883282801b18711220a229d0fd266a5deab26 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 3701177f3821330c8eb2af96f60123245cf42273abdae472bcb96bb120dcba8f +F src/shell.c.in 51f027f6b48fab39e0745915d979747880b7c35827798535726ac44366a291f1 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,7 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test bfeb5ab4574f9a264b3893ce0e41f04c2052b72b174e5dd9d847b6e7b8f4d15c +F test/recover.test 52609c8cc24e72d3d8a20fb8bc32ba2ce8ca2093a7f4573bd4f2969f78f6d2b4 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d -R e71b5bc56d773e53b87820c68d2be836 +P 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 +R a9542f620433fc5d37a56caa32ea75c3 U dan -Z d098c243e647f537ccde95cbf714168f +Z 07c2469e7eeced685ce45289494bc94e diff --git a/manifest.uuid b/manifest.uuid index 6b5834e3bb..e6d0f63ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 \ No newline at end of file +67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f906205bff..8c967a83d8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1104,7 +1104,6 @@ struct ShellState { #define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ #define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ -#define SHFLG_Recover 0x00000080 /* .dump is --recover */ /* ** Macros for testing and setting shellFlgs @@ -3577,6 +3576,7 @@ static const char *(azHelp[]) = { ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", ".read FILE Read input from FILE", + ".recover Recover as much data as possible from corrupt db.", ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save FILE Write in-memory database into FILE", ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", @@ -6153,6 +6153,12 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +/* +** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, the SQL statement or statements in zSql are executed using +** database connection db and the error code written to *pRc before +** this function returns. +*/ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ int rc = *pRc; if( rc==SQLITE_OK ){ @@ -6165,6 +6171,9 @@ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ } } +/* +** Like shellExec(), except that zFmt is a printf() style format string. +*/ static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ char *z = 0; if( *pRc==SQLITE_OK ){ @@ -6181,6 +6190,12 @@ static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ } } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, an attempt is made to allocate, zero and return a pointer +** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set +** to SQLITE_NOMEM and NULL returned. +*/ static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ @@ -6194,6 +6209,17 @@ static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ return pRet; } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, zFmt is treated as a printf() style string. The result of +** formatting it along with any trailing arguments is written into a +** buffer obtained from sqlite3_malloc(), and pointer to which is returned. +** It is the responsibility of the caller to eventually free this buffer +** using a call to sqlite3_free(). +** +** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL +** pointer returned. +*/ static char *shellMPrintf(int *pRc, const char *zFmt, ...){ char *z = 0; if( *pRc==SQLITE_OK ){ @@ -6208,21 +6234,25 @@ static char *shellMPrintf(int *pRc, const char *zFmt, ...){ return z; } +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ typedef struct RecoverTable RecoverTable; struct RecoverTable { - char *zName; /* Name of table */ - char *zQuoted; /* Quoted version of zName */ + char *zQuoted; /* Quoted version of table name */ int nCol; /* Number of columns in table */ char **azlCol; /* Array of column lists */ - int iPk; + int iPk; /* Index of IPK column */ }; /* -** Free a RecoverTable object allocated by recoverNewTable() +** Free a RecoverTable object allocated by recoverFindTable() or +** recoverOrphanTable(). */ static void recoverFreeTable(RecoverTable *pTab){ if( pTab ){ - sqlite3_free(pTab->zName); sqlite3_free(pTab->zQuoted); if( pTab->azlCol ){ int i; @@ -6235,7 +6265,14 @@ static void recoverFreeTable(RecoverTable *pTab){ } } -static RecoverTable *recoverOldTable( +/* +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. +** Otherwise, it allocates and returns a RecoverTable object based on the +** final four arguments passed to this function. It is the responsibility +** of the caller to eventually free the returned object using +** recoverFreeTable(). +*/ +static RecoverTable *recoverNewTable( int *pRc, /* IN/OUT: Error code */ const char *zName, /* Name of table */ const char *zSql, /* CREATE TABLE statement */ @@ -6309,8 +6346,7 @@ static RecoverTable *recoverOldTable( } } - pTab->zName = shellMPrintf(&rc, "%s", zName); - pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); + pTab->zQuoted = shellMPrintf(&rc, "%Q", zName); pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); pTab->nCol = nSqlCol; @@ -6349,7 +6385,7 @@ static RecoverTable *recoverOldTable( return pTab; } -static RecoverTable *recoverNewTable( +static RecoverTable *recoverFindTable( ShellState *pState, int *pRc, int iRoot, @@ -6363,7 +6399,6 @@ static RecoverTable *recoverNewTable( const char *zSql = 0; const char *zName = 0; - /* Search the recovered schema for an object with root page iRoot. */ shellPreparePrintf(pState->db, pRc, &pStmt, "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot @@ -6377,7 +6412,7 @@ static RecoverTable *recoverNewTable( if( sqlite3_stricmp(zType, "table")==0 ){ zName = (const char*)sqlite3_column_text(pStmt, 1); zSql = (const char*)sqlite3_column_text(pStmt, 2); - pRet = recoverOldTable(pRc, zName, zSql, bIntkey, nCol); + pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); break; } } @@ -6390,24 +6425,35 @@ static RecoverTable *recoverNewTable( static RecoverTable *recoverOrphanTable( ShellState *pState, int *pRc, + const char *zLostAndFound, int nCol ){ RecoverTable *pTab = 0; if( nCol>=0 && *pRc==SQLITE_OK ){ int i; - raw_printf(pState->out, - "CREATE TABLE recover_orphan(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER" + + /* This block determines the name of the orphan table. The prefered + ** name is zLostAndFound. But if that clashes with another name + ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 + ** and so on until a non-clashing name is found. */ + int iTab = 0; + char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); + sqlite3_stmt *pTest = 0; + shellPrepare(pState->db, pRc, + "SELECT 1 FROM recovery.schema WHERE name=?", &pTest ); - for(i=0; iout, ", c%d", i); + if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ + shellReset(pRc, pTest); + sqlite3_free(zTab); + zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); + sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); } - raw_printf(pState->out, ");\n"); + shellFinalize(pRc, pTest); pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); if( pTab ){ - pTab->zName = shellMPrintf(pRc, "%s", "recover_orphan"); - pTab->zQuoted = shellMPrintf(pRc, "%Q", pTab->zName); + pTab->zQuoted = shellMPrintf(pRc, "%Q", zTab); pTab->nCol = nCol; pTab->iPk = -2; if( nCol>0 ){ @@ -6419,12 +6465,22 @@ static RecoverTable *recoverOrphanTable( } } } - } - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + }else{ + raw_printf(pState->out, + "CREATE TABLE %s(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted + ); + for(i=0; iout, ", c%d", i); + } + raw_printf(pState->out, ");\n"); + } } + sqlite3_free(zTab); } return pTab; } @@ -6440,6 +6496,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ const char *zRecoveryDb = ""; /* Name of "recovery" database */ + const char *zLostAndFound = "lost_and_found"; int i; int nOrphan = -1; RecoverTable *pOrphan = 0; @@ -6452,16 +6509,21 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ n = strlen(z); if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){ bFreelist = 0; - } + }else if( n<=12 && memcmp("-recovery-db", z, n)==0 && i<(nArg-1) ){ i++; zRecoveryDb = azArg[i]; + }else + if( n<=15 && memcmp("-lost-and-found", z, n)==0 && i<(nArg-1) ){ + i++; + zLostAndFound = azArg[i]; } else{ raw_printf(stderr, "unexpected option: %s\n", azArg[i]); raw_printf(stderr, "options are:\n"); raw_printf(stderr, " --freelist-corrupt\n"); raw_printf(stderr, " --recovery-db DATABASE\n"); + raw_printf(stderr, " --lost-and-found TABLE-NAME\n"); return 1; } } @@ -6599,7 +6661,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } shellFinalize(&rc, pLoop); pLoop = 0; - pOrphan = recoverOrphanTable(pState, &rc, nOrphan); + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); shellPrepare(pState->db, &rc, "SELECT pgno FROM recovery.map WHERE root=?", &pPages @@ -6624,11 +6686,11 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int bNoop = 0; RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); if( bNoop || rc ) continue; if( pTab==0 ) pTab = pOrphan; - if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + if( 0==sqlite3_stricmp(pTab->zQuoted, "'sqlite_sequence'") ){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } sqlite3_bind_int(pPages, 1, iRoot); @@ -7042,30 +7104,30 @@ static int do_meta_command(char *zLine, ShellState *p){ p->nErr = 0; if( zLike==0 ){ run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" + ); run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE name=='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE name=='sqlite_sequence'" + ); run_table_dump_query(p, - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 - ); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); }else{ char *zSql; zSql = sqlite3_mprintf( - "SELECT name, type, sql FROM sqlite_master " - "WHERE tbl_name LIKE %Q AND type=='table'" - " AND sql NOT NULL", zLike); + "SELECT name, type, sql FROM sqlite_master " + "WHERE tbl_name LIKE %Q AND type=='table'" + " AND sql NOT NULL", zLike); run_schema_dump_query(p,zSql); sqlite3_free(zSql); zSql = sqlite3_mprintf( - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL" - " AND type IN ('index','trigger','view')" - " AND tbl_name LIKE %Q", zLike); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name LIKE %Q", zLike); run_table_dump_query(p, zSql, 0); sqlite3_free(zSql); } diff --git a/test/recover.test b/test/recover.test index 4de7f503a5..4dad0d646d 100644 --- a/test/recover.test +++ b/test/recover.test @@ -39,7 +39,7 @@ proc compare_dbs {db1 db2} { } } -proc do_recover_test {tn} { +proc do_recover_test {tn {tsql {}} {res {}}} { set fd [open "|$::CLI test.db .recover"] fconfigure $fd -encoding binary fconfigure $fd -translation binary @@ -48,9 +48,12 @@ proc do_recover_test {tn} { forcedelete test.db2 sqlite3 db2 test.db2 - breakpoint execsql $sql db2 - uplevel [list do_test $tn [list compare_dbs db db2] {}] + if {$tsql==""} { + uplevel [list do_test $tn [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn $tsql $res] + } db2 close } @@ -96,4 +99,31 @@ do_execsql_test 2.1.0 { do_recover_test 2.1.1 +do_execsql_test 2.2.0 { + PRAGMA writable_schema = 1; + DELETE FROM sqlite_master WHERE name='t1'; +} +do_recover_test 2.2.1 { + SELECT name FROM sqlite_master +} {lost_and_found} + +do_execsql_test 2.3.0 { + CREATE TABLE lost_and_found(a, b, c); +} +do_recover_test 2.3.1 { + SELECT name FROM sqlite_master +} {lost_and_found lost_and_found_0} + +do_execsql_test 2.4.0 { + CREATE TABLE lost_and_found_0(a, b, c); +} +do_recover_test 2.4.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + finish_test From 0aa01ee42cd9616fdca75f97b88e861651481d3c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 19:36:49 +0000 Subject: [PATCH 15/16] Add comments and fix formatting issues in new code in shell.c.in. FossilOrigin-Name: b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 2f2f86a8fc..c4c8fd8087 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"--lost-and-found"\soption\sto\sthe\s".recover"\scommand.\sFor\ssetting\sthe\sname\sof\sthe\sorphaned\srows\stable. -D 2019-04-27T18:47:03.466 +C Add\scomments\sand\sfix\sformatting\sissues\sin\snew\scode\sin\sshell.c.in. +D 2019-04-27T19:36:49.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 51f027f6b48fab39e0745915d979747880b7c35827798535726ac44366a291f1 +F src/shell.c.in 2316b9ee7a43e6f127e5abe8d74d69b92f6bbae1be38826ffb58384aab832013 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 -R a9542f620433fc5d37a56caa32ea75c3 +P 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 +R 1520b643c98e19ff415724ed970cd8b1 U dan -Z 07c2469e7eeced685ce45289494bc94e +Z 3d67e0e6129b0deb0a86f6c6a9eda8ce diff --git a/manifest.uuid b/manifest.uuid index e6d0f63ea7..73bfbb6ae3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 \ No newline at end of file +b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8c967a83d8..1fc08662db 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6385,13 +6385,28 @@ static RecoverTable *recoverNewTable( return pTab; } +/* +** This function is called to search the schema recovered from the +** sqlite_master table of the (possibly) corrupt database as part +** of a ".recover" command. Specifically, for a table with root page +** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the +** table must be a WITHOUT ROWID table, or if non-zero, not one of +** those. +** +** If a table is found, a (RecoverTable*) object is returned. Or, if +** no such table is found, but bIntkey is false and iRoot is the +** root page of an index in the recovered schema, then (*pbNoop) is +** set to true and NULL returned. Or, if there is no such table or +** index, NULL is returned and (*pbNoop) set to 0, indicating that +** the caller should write data to the orphans table. +*/ static RecoverTable *recoverFindTable( - ShellState *pState, - int *pRc, - int iRoot, - int bIntkey, - int nCol, - int *pbNoop + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + int iRoot, /* Root page of table */ + int bIntkey, /* True for an intkey table */ + int nCol, /* Number of columns in table */ + int *pbNoop /* OUT: True if iRoot is root of index */ ){ sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; @@ -6422,11 +6437,14 @@ static RecoverTable *recoverFindTable( return pRet; } +/* +** Return a RecoverTable object representing the orphans table. +*/ static RecoverTable *recoverOrphanTable( - ShellState *pState, - int *pRc, - const char *zLostAndFound, - int nCol + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + const char *zLostAndFound, /* Base name for orphans table */ + int nCol /* Number of user data columns */ ){ RecoverTable *pTab = 0; if( nCol>=0 && *pRc==SQLITE_OK ){ From 1b16216f9eb297ddebc36fd83d3c3816baad0163 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 20:15:15 +0000 Subject: [PATCH 16/16] Fix building the shell with SQLITE_OMIT_VIRTUAL_TABLE. And without SQLITE_ENABLE_DBPAGE_VTAB. FossilOrigin-Name: 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 --- ext/misc/dbdata.c | 5 ++--- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 21 +++++++++++++++++---- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 786369ce0a..0e7f7d59cc 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -75,7 +75,6 @@ #include "sqlite3ext.h" typedef unsigned char u8; -typedef unsigned long u32; #endif SQLITE_EXTENSION_INIT1 @@ -313,7 +312,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - u32 pgno, /* Page number of page to load */ + unsigned int pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -556,7 +555,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; diff --git a/manifest b/manifest index c4c8fd8087..1abc30e6f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sand\sfix\sformatting\sissues\sin\snew\scode\sin\sshell.c.in. -D 2019-04-27T19:36:49.564 +C Fix\sbuilding\sthe\sshell\swith\sSQLITE_OMIT_VIRTUAL_TABLE.\sAnd\swithout\sSQLITE_ENABLE_DBPAGE_VTAB. +D 2019-04-27T20:15:15.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c fe978dad2df13dd4b377b5d38f4883282801b18711220a229d0fd266a5deab26 +F ext/misc/dbdata.c 1b3751b02d8f575d25c6bda358670d2e39ace368a0d05595989c308a10c615f6 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 2316b9ee7a43e6f127e5abe8d74d69b92f6bbae1be38826ffb58384aab832013 +F src/shell.c.in 104bbae904a2b67bc6c0c95337447544d15d0594dc46468608aae769d5f51da9 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 -R 1520b643c98e19ff415724ed970cd8b1 +P b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 +R dcbe5105dfc58103275280fa56f0f458 U dan -Z 3d67e0e6129b0deb0a86f6c6a9eda8ce +Z a7824dfb7c5c59a3dfb17eb398274a3e diff --git a/manifest.uuid b/manifest.uuid index 73bfbb6ae3..3985d129a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 \ No newline at end of file +425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1fc08662db..2f9a2456a1 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -948,7 +948,9 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) INCLUDE ../ext/misc/dbdata.c +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -3576,7 +3578,9 @@ static const char *(azHelp[]) = { ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", ".read FILE Read input from FILE", +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ".recover Recover as much data as possible from corrupt db.", +#endif ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save FILE Write in-memory database into FILE", ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", @@ -4121,7 +4125,9 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) sqlite3_dbdata_init(p->db, 0, 0); +#endif #ifdef SQLITE_HAVE_ZLIB sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); @@ -5390,10 +5396,7 @@ static int lintDotCommand( return SQLITE_ERROR; } -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) -/********************************************************************************* -** The ".archive" or ".ar" command. -*/ +#if !defined SQLITE_OMIT_VIRTUALTABLE static void shellPrepare( sqlite3 *db, int *pRc, @@ -5464,6 +5467,12 @@ static void shellReset( *pRc = rc; } } +#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +/********************************************************************************* +** The ".archive" or ".ar" command. +*/ /* ** Structure representing a single ".ar" command. */ @@ -6153,6 +6162,7 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) /* ** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, the SQL statement or statements in zSql are executed using @@ -6775,6 +6785,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); return rc; } +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ /* @@ -7063,10 +7074,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = shell_dbinfo_command(p, nArg, azArg); }else +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); rc = recoverDatabaseCmd(p, nArg, azArg); }else +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0;