Take the freelist into account when recovering data that is not linked in to any tree associated with a schema entry.

FossilOrigin-Name: dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a

ext/recover/recoverold.test

@@ -131,6 +131,23 @@ do_recover_test 2.4.1 {
   2 2 3 {} 8 9 7
 }
 
+do_execsql_test 2.5 {
+  CREATE TABLE x1(a, b, c);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
+  )
+  INSERT INTO x1 SELECT i, i, hex(randomblob(500)) FROM s;
+  DROP TABLE x1;
+}
+do_recover_test 2.5.1 {
+  SELECT name FROM sqlite_master;
+  SELECT * FROM lost_and_found_1;
+} {lost_and_found lost_and_found_0 lost_and_found_1
+  2 2 3 {} 2 3 1
+  2 2 3 {} 5 6 4
+  2 2 3 {} 8 9 7
+}
+
 #-------------------------------------------------------------------------
 breakpoint
 reset_db

ext/recover/sqlite3recover.c

@@ -107,7 +107,7 @@ static int recoverStrlen(const char *zStr){
   return nRet;
 }
 
-static void *recoverMalloc(sqlite3_recover *p, sqlite3_int64 nByte){
+static void *recoverMalloc(sqlite3_recover *p, i64 nByte){
   void *pRet = 0;
   assert( nByte>0 );
   if( p->errCode==SQLITE_OK ){
@@ -276,6 +276,36 @@ static i64 recoverPageCount(sqlite3_recover *p){
   return nPg;
 }
 
+/*
+** Scalar function "read_i32". The first argument to this function
+** must be a blob. The second a non-negative integer. This function
+** reads and returns a 32-bit big-endian integer from byte
+** offset (4*<arg2>) of the blob.
+*/
+static void recoverReadI32(
+  sqlite3_context *context, 
+  int argc, 
+  sqlite3_value **argv
+){
+  const unsigned char *pBlob;
+  int nBlob;
+  int iInt;
+
+  assert( argc==2 );
+  nBlob = sqlite3_value_bytes(argv[0]);
+  pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]);
+  iInt = sqlite3_value_int(argv[1]);
+
+  if( iInt>=0 && (iInt+1)*4<=nBlob ){
+    const unsigned char *a = &pBlob[iInt*4];
+    i64 iVal = ((i64)a[0]<<24)
+             + ((i64)a[1]<<16)
+             + ((i64)a[2]<< 8)
+             + ((i64)a[3]<< 0);
+    sqlite3_result_int64(context, iVal);
+  }
+}
+
 /*
 ** SELECT page_is_used(pgno);
 */
@@ -285,7 +315,7 @@ static void recoverPageIsUsed(
   sqlite3_value **apArg
 ){
   sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
-  sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]);
+  i64 pgno = sqlite3_value_int64(apArg[0]);
   sqlite3_stmt *pStmt = 0;
   int bRet;
 
@@ -314,7 +344,7 @@ static void recoverGetPage(
   sqlite3_value **apArg
 ){
   sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
-  sqlite3_int64 pgno = sqlite3_value_int64(apArg[0]);
+  i64 pgno = sqlite3_value_int64(apArg[0]);
   sqlite3_stmt *pStmt = 0;
 
   assert( nArg==1 );
@@ -397,6 +427,11 @@ static int recoverOpenOutput(sqlite3_recover *p){
           db, "page_is_used", 1, SQLITE_UTF8, (void*)p, recoverPageIsUsed, 0, 0
       );
     }
+    if( rc==SQLITE_OK ){
+      rc = sqlite3_create_function(
+          db, "read_i32", 2, SQLITE_UTF8, (void*)p, recoverReadI32, 0, 0
+      );
+    }
 
     if( rc!=SQLITE_OK ){
       if( p->errCode==SQLITE_OK ) rc = recoverDbError(p, db);
@@ -852,6 +887,29 @@ static int recoverLostAndFound(sqlite3_recover *p){
     }
     recoverFinalize(p, pStmt);
 
+    /* Add all pages that appear to be part of the freelist to the bitmap. */
+    pStmt = recoverPrepare(p, p->dbOut,
+      "WITH trunk(pgno) AS ("
+      "  SELECT read_i32(getpage(1), 8) AS x WHERE x>0"
+      "    UNION"
+      "  SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0"
+      "),"
+      "trunkdata(pgno, data) AS ("
+      "  SELECT pgno, getpage(pgno) FROM trunk"
+      "),"
+      "freelist(data, n, freepgno) AS ("
+      "  SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata"
+      "    UNION ALL"
+      "  SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0"
+      ")"
+      "SELECT freepgno FROM freelist"
+    );
+    while( pStmt && SQLITE_ROW==sqlite3_step(pStmt) ){
+      i64 iPg = sqlite3_column_int64(pStmt, 0);
+      recoverBitmapSet(pMap, iPg);
+    }
+    recoverFinalize(p, pStmt);
+
     /* Add an entry for each page not already added to the bitmap to 
     ** the recovery.map table. This loop leaves the "parent" column
     ** of each recovery.map row set to NULL - to be filled in below.  */

manifest

@@ -1,5 +1,5 @@
-C Further\swork\son\smaking\sthe\srecover\sextension\scompatible\swith\sthe\s.recover\scommand.
-D 2022-09-03T20:07:39.011
+C Take\sthe\sfreelist\sinto\saccount\swhen\srecovering\sdata\sthat\sis\snot\slinked\sin\sto\sany\stree\sassociated\swith\sa\sschema\sentry.
+D 2022-09-03T20:31:36.832
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -389,8 +389,8 @@ F ext/rbu/sqlite3rbu.h 1dc88ab7bd32d0f15890ea08d23476c4198d3da3056985403991f8c9c
 F ext/rbu/test_rbu.c 03f6f177096a5f822d68d8e4069ad8907fe572c62ff2d19b141f59742821828a
 F ext/recover/recover1.test a848af8c82fe0731af835ff99475724f8654d2f24f772cc4e6f7ec4eb2ab71ea
 F ext/recover/recover_common.tcl 6679af7dffc858e345053a91c9b0a897595b4a13007aceffafca75304ccb137c
-F ext/recover/recoverold.test 33ccbe2393af0e82f292c135b725e3eca1e803960681cf6da41fc00d28bd8683
-F ext/recover/sqlite3recover.c 8d93b9aa056c3fae9a5e2736a4ffa71414bdb502863ef879e55bec7b37030266
+F ext/recover/recoverold.test 7578e9b938db15dc469a4af247e15866226f366bde0cbe09a40b0aef4a0506c8
+F ext/recover/sqlite3recover.c 6c9cbc993a970060f9fb881d78f6c7e182ec988a5e48acbf15bb4a5f05ce2902
 F ext/recover/sqlite3recover.h b82974790b528480163d87dcd84afffe7568393194c9ec8241cfbc3ee6bbdd1b
 F ext/recover/test_recover.c b8dddd96ccd4a62bc14cb3a8d5696407892e184fe7d45ecbedde954577857de2
 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15
@@ -2005,8 +2005,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 8df7c7d0dcd1b2fcdad00e765a9868407f0ced02ac4432ee2cdf25d83b130759
-R 93788fe8081a2011cde8fac3142d518f
+P f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2
+R 471b646b541e0fcab850e84cb036ac46
 U dan
-Z 21514150d144c868f0bbea27057fe308
+Z 9f579b130a06a2078244b88aebbd5365
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-f2ac315844d8db1bd1c6950a4fef7c459ddd37cc21a8f3daafa5639fad8118e2
+dbd1f1efb349a9c8886e42b3f07d3f4c576924136f111558c7294d0a272e415a