Fix a problem with the handling of delete markers by automerge on large databases.

FossilOrigin-Name: 2b09bd17eb85ea3c682e930d2aabc6441f2265e0

ext/fts3/fts3_write.c

@@ -2433,6 +2433,37 @@ static int fts3SegmentMaxLevel(
   return sqlite3_reset(pStmt);
 }
 
+/*
+** iAbsLevel is an absolute level that may be assumed to exist within
+** the database. This function checks if it is the largest level number
+** within its index. Assuming no error occurs, *pbMax is set to 1 if
+** iAbsLevel is indeed the largest level, or 0 otherwise, and SQLITE_OK
+** is returned. If an error occurs, an error code is returned and the
+** final value of *pbMax is undefined.
+*/
+static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){
+
+  /* Set pStmt to the compiled version of:
+  **
+  **   SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?
+  **
+  ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR).
+  */
+  sqlite3_stmt *pStmt;
+  int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
+  if( rc!=SQLITE_OK ) return rc;
+  sqlite3_bind_int64(pStmt, 1, iAbsLevel+1);
+  sqlite3_bind_int64(pStmt, 2, 
+      ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL
+  );
+
+  *pbMax = 0;
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    *pbMax = sqlite3_column_type(pStmt, 0)==SQLITE_NULL;
+  }
+  return sqlite3_reset(pStmt);
+}
+
 /*
 ** Delete all entries in the %_segments table associated with the segment
 ** opened with seg-reader pSeg. This function does not affect the contents
@@ -4836,8 +4867,15 @@ int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){
     if( rc==SQLITE_OK ){
       rc = fts3IncrmergeOutputIdx(p, iAbsLevel, &iIdx);
       assert( bUseHint==1 || bUseHint==0 );
-      if( (iIdx-bUseHint)==0 ) pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
+      if( iIdx==0 || (bUseHint && iIdx==1) ){
+        int bIgnore;
+        rc = fts3SegmentIsMaxLevel(p, iAbsLevel+1, &bIgnore);
+        if( bIgnore ){
+          pFilter->flags |= FTS3_SEGMENT_IGNORE_EMPTY;
+        }
+      }
     }
+
     if( rc==SQLITE_OK ){
       rc = fts3IncrmergeCsr(p, iAbsLevel, nSeg, pCsr);
     }

manifest

@@ -1,5 +1,5 @@
-C Do\snot\ssearch\sfor\spromotable\ssegments\sfollowing\san\sFTS\s'optimize'\soperation\sor\screation\sof\sa\snew\ssegment\son\sthe\soldest\sexisting\slevel.
-D 2014-05-14T19:49:17.392
+C Fix\sa\sproblem\swith\sthe\shandling\sof\sdelete\smarkers\sby\sautomerge\son\slarge\sdatabases.
+D 2014-05-15T18:36:39.233
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in dd2b1aba364ff9b05de41086f74407f285c57670
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9
 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d
-F ext/fts3/fts3_write.c e58c8df149436dcc85e2face5516b75a01c7a583
+F ext/fts3/fts3_write.c b90173c62460a215498b0da3046f5e429b3a66b0
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
@@ -1172,7 +1172,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 7268119f74602929b372b88eca7b3c3b4964f7d5
-R 38e0e000cd94731b650020d6321d9a1b
+P 34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab
+R e26158b3998e23a04d5e7968b1e6dfac
 U dan
-Z 73a15f3066e0e6dc1f73e5ba383c60f7
+Z 27219917e93f1cfe1a1e1c19124576cc

manifest.uuid

@@ -1 +1 @@
-34f6b4b8e4cb8ea45fd518672dc96335d1ce06ab
+2b09bd17eb85ea3c682e930d2aabc6441f2265e0