From d7889b26acd8fa664677fe923e091d356374fb53 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Tue, 21 Apr 2015 19:07:39 +0000
Subject: [PATCH] Fix an fts5 problem with large deletes.

FossilOrigin-Name: e50e8031d6f804ebe50e0eec9a8b6e7f0152ecc3
---
 ext/fts5/fts5_index.c        | 11 +++--
 ext/fts5/test/fts5dlidx.test | 80 ++++++++++++++++++++++++++++++++++++
 manifest                     | 13 +++---
 manifest.uuid                |  2 +-
 4 files changed, 96 insertions(+), 10 deletions(-)
 create mode 100644 ext/fts5/test/fts5dlidx.test

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 7d48d074da..87fd11a096 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -1426,7 +1426,7 @@ static int fts5DlidxIterNext(Fts5DlidxIter *pIter){
 }
 
 static int fts5DlidxIterEof(Fts5Index *p, Fts5DlidxIter *pIter){
-  return (p->rc!=SQLITE_OK || pIter->bEof);
+  return pIter->bEof;
 }
 
 static void fts5DlidxIterLast(Fts5DlidxIter *pIter){
@@ -1460,6 +1460,7 @@ static int fts5DlidxIterPrev(Fts5DlidxIter *pIter){
     pIter->iRowid -= iVal;
     pIter->iLeafPgno--;
 
+    /* Skip backwards passed any 0x00 bytes. */
     while( iOff>pIter->iFirstOff 
         && a[iOff-1]==0x00 && (a[iOff-2] & 0x80)==0 
     ){
@@ -3698,6 +3699,7 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
             fts5PutU16(&pBuf->p[0], pBuf->n);   /* first docid on page */
             pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iRowid);
             bFirstDocid = 0;
+            fts5WriteDlidxAppend(p, &writer, iRowid);
           }else{
             pBuf->n += sqlite3PutVarint(&pBuf->p[pBuf->n], iDelta);
           }
@@ -3720,13 +3722,16 @@ static void fts5FlushOneHash(Fts5Index *p, int iHash, int *pnLeaf){
                 n = nCopy - iPos;
               }else{
                 n = fts5PoslistPrefix(&pPoslist[iPos], nSpace);
+                assert( n>=nSpace );
               }
               assert( n>0 );
               fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n);
               iPos += n;
+              if( pBuf->n>=pgsz ){
+                fts5WriteFlushLeaf(p, &writer);
+                pBuf = &writer.aWriter[0].buf;
+              }
               if( iPos>=nCopy ) break;
-              fts5WriteFlushLeaf(p, &writer);
-              pBuf = &writer.aWriter[0].buf;
             }
             bFirstDocid = 1;
           }
diff --git a/ext/fts5/test/fts5dlidx.test b/ext/fts5/test/fts5dlidx.test
new file mode 100644
index 0000000000..0bfc3f331e
--- /dev/null
+++ b/ext/fts5/test/fts5dlidx.test
@@ -0,0 +1,80 @@
+# 2015 April 21
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This test is focused on uses of doclist-index records.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5dlidx
+
+if { $tcl_platform(wordSize)<8 } {
+  finish_test
+  return
+}
+
+proc do_fb_test {tn sql res} {
+  set res2 [lsort -integer -decr $res]
+  uplevel [list do_execsql_test $tn.1 $sql $res]
+  uplevel [list do_execsql_test $tn.2 "$sql ORDER BY rowid DESC" $res2]
+}
+
+do_execsql_test 1.0 { 
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
+}
+
+foreach {tn spc1 spc2 mul} {
+  1 10  100 1000
+  2 1   1   128
+} {
+  set xdoc [list]
+  set ydoc [list]
+  
+  execsql { DELETE FROM t1 }
+
+  do_test 1.$tn.1 {
+  
+    execsql BEGIN
+    for {set i 0} {$i < 10000} {incr i} {
+      set rowid [expr $i * $mul]
+      set doc "a b c a b c a b c a b c a b c"
+      if {($i % $spc1)==0} { 
+        lappend xdoc $rowid
+        append doc " x" 
+        if {($i % $spc2)==0} { 
+          lappend ydoc $rowid
+          append doc " y" 
+        }
+      }
+      execsql { INSERT INTO t1(rowid, x) VALUES($rowid, $doc) }
+    }
+    execsql COMMIT
+    execsql { INSERT INTO t1(t1) VALUES('integrity-check') }
+  } {}
+  
+  do_execsql_test 1.$tn.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  
+  do_fb_test 1.$tn.3.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND x' } $xdoc
+  do_fb_test 1.$tn.3.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'x AND a' } $xdoc
+  
+  do_fb_test 1.$tn.4.1 { SELECT rowid FROM t1 WHERE t1 MATCH 'a AND y' } $ydoc
+  do_fb_test 1.$tn.4.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'y AND a' } $ydoc
+  
+  do_fb_test 1.$tn.5.1 { 
+    SELECT rowid FROM t1 WHERE t1 MATCH 'a + b + c + x' } $xdoc
+  do_fb_test 1.$tn.5.2 { 
+    SELECT rowid FROM t1 WHERE t1 MATCH 'b + c + x + y' } $ydoc
+
+}
+
+
+finish_test
+
diff --git a/manifest b/manifest
index cb1f4c510b..5c12ac6c1c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\sfts5\sproblems\swith\svery\slarge\sposition\slists.
-D 2015-04-20T18:48:57.780
+C Fix\san\sfts5\sproblem\swith\slarge\sdeletes.
+D 2015-04-21T19:07:39.210
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 88a3e6261286db378fdffa1124cad11b3c05f5bb
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 3ba56cc6824c9f7b1e0695159e0a9c636f6b4a23
 F ext/fts5/fts5_config.c 0847facc8914f57ea4452c43ce109200dc65e894
 F ext/fts5/fts5_expr.c 5215137efab527577d36bdf9e44bfc2ec3e1be98
 F ext/fts5/fts5_hash.c 3cb5a3d04dd2030eb0ac8d544711dfd37c0e6529
-F ext/fts5/fts5_index.c f840e35cceafcd0597688467010a4d12feea9c76
+F ext/fts5/fts5_index.c abf74b5e0d96b954911b6db3e13dce008c091303
 F ext/fts5/fts5_storage.c ac0f0937059c8d4f38a1f13aa5f2c2cd7edf3e0d
 F ext/fts5/fts5_tcl.c 617b6bb96545be8d9045de6967c688cd9cd15541
 F ext/fts5/fts5_tokenize.c c07f2c2f749282c1dbbf46bde1f6d7095c740b8b
@@ -136,6 +136,7 @@ F ext/fts5/test/fts5auxdata.test c69b86092bf1a157172de5f9169731af3403179b
 F ext/fts5/test/fts5bigpl.test b1cfd00561350ab04994ba7dd9d48468e5e0ec3b
 F ext/fts5/test/fts5content.test 8dc302fccdff834d946497e9d862750ea87d4517
 F ext/fts5/test/fts5corrupt.test dbdcfe75749ed2f2eb3915cf68fd55d3dc3b058d
+F ext/fts5/test/fts5dlidx.test 710d1eaf44e6fbb09dfa73b7fd488227d8cc751a
 F ext/fts5/test/fts5ea.test 04695560a444fcc00c3c4f27783bdcfbf71f030c
 F ext/fts5/test/fts5eb.test 728a1f23f263548f5c29b29dfb851b5f2dbe723e
 F ext/fts5/test/fts5fault1.test fbd8612889234849ff041f5b36f8e390feeed46e
@@ -1293,7 +1294,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 50fae1f0006c0e946b5214e73eedf2687a0016f9
-R e3e266d7af429931ea61b2ad868bf28e
+P 2ea8f9cbe67dac60c1a0a661c95a03ecfa9a0b9a
+R f13e2690ef21877d2344bab07fedcde6
 U dan
-Z 88f214a9049d68201f885f825375d535
+Z 02eb8d79520bf4e8512cc80bc0c885cd
diff --git a/manifest.uuid b/manifest.uuid
index 6b9e7e2cd3..a63e0364d4 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-2ea8f9cbe67dac60c1a0a661c95a03ecfa9a0b9a
\ No newline at end of file
+e50e8031d6f804ebe50e0eec9a8b6e7f0152ecc3
\ No newline at end of file