From a5aa8e1db2e2eef9e00e67798fc67dfedef3cfa8 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 10 Sep 2015 05:40:17 +0000
Subject: [PATCH] Change the array of 16-bit offsets at the end of each page to
 an array of varints.

FossilOrigin-Name: fab245bea4f283714c17bca22428d5eb4db5935a
---
 ext/fts5/fts5_index.c         | 222 +++++++++++++++++-----------------
 ext/fts5/test/fts5simple.test |  58 ++++++++-
 ext/fts5/tool/loadfts5.tcl    |  17 ++-
 manifest                      |  16 +--
 manifest.uuid                 |   2 +-
 5 files changed, 191 insertions(+), 124 deletions(-)

diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index 9bc08e7443..a7e654d745 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -378,6 +378,7 @@ struct Fts5Structure {
 */
 struct Fts5PageWriter {
   int pgno;                       /* Page number for this page */
+  int iPrevPgidx;                 /* Previous value written into pgidx */
   Fts5Buffer buf;                 /* Buffer containing leaf data */
   Fts5Buffer pgidx;               /* Buffer containing page-index */
   Fts5Buffer term;                /* Buffer containing previous term on page */
@@ -492,7 +493,7 @@ struct Fts5SegIter {
   int iTermLeafPgno;
   int iTermLeafOffset;
 
-  int iTermIdx;
+  int iPgidxOff;                  /* Next offset in pgidx */
   int iEndofDoclist;
 
   /* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
@@ -532,8 +533,6 @@ struct Fts5SegIter {
 
 #define fts5LeafFirstRowidOff(x) (fts5GetU16((x)->p))
 
-#define fts5LeafFirstTermOff(x) fts5LeafTermOff(x, 0)
-
 /*
 ** poslist:
 **   Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered.
@@ -648,6 +647,11 @@ static int fts5BlobCompare(
 }
 #endif
 
+static int fts5LeafFirstTermOff(Fts5Data *pLeaf){
+  int ret;
+  fts5GetVarint32(&pLeaf->p[pLeaf->szLeaf], ret);
+  return ret;
+}
 
 /*
 ** Close the read-only blob handle, if it is open.
@@ -1523,20 +1527,6 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){
   }
 }
 
-static void fts5SegIterLoadEod(Fts5Index *p, Fts5SegIter *pIter){
-  Fts5Data *pLeaf = pIter->pLeaf;
-  int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2;
-
-  assert( pIter->iLeafPgno==pIter->iTermLeafPgno );
-  if( (pIter->iTermIdx+1)<nPg ){
-    int iRead = pLeaf->szLeaf + (pIter->iTermIdx + 1) * 2;
-    pIter->iEndofDoclist = fts5GetU16(&pLeaf->p[iRead]);
-  }else{
-    pIter->iEndofDoclist = pLeaf->nn+1;
-  }
-
-}
-
 static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
   u8 *a = pIter->pLeaf->p;        /* Buffer to read data from */
   int iOff = pIter->iLeafOffset;
@@ -1583,7 +1573,14 @@ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){
   pIter->iTermLeafPgno = pIter->iLeafPgno;
   pIter->iLeafOffset = iOff;
 
-  fts5SegIterLoadEod(p, pIter);
+  if( pIter->iPgidxOff>=pIter->pLeaf->nn ){
+    pIter->iEndofDoclist = pIter->pLeaf->nn+1;
+  }else{
+    int nExtra;
+    pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], nExtra);
+    pIter->iEndofDoclist += nExtra;
+  }
+
   fts5SegIterLoadRowid(p, pIter);
 }
 
@@ -1621,7 +1618,7 @@ static void fts5SegIterInit(
     pIter->iLeafOffset = 4;
     assert_nc( pIter->pLeaf->nn>4 );
     assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 );
-    pIter->iTermIdx = 0;
+    pIter->iPgidxOff = pIter->pLeaf->szLeaf+1;
     fts5SegIterLoadTerm(p, pIter, 0);
     fts5SegIterLoadNPos(p, pIter);
   }
@@ -1789,11 +1786,6 @@ static void fts5SegIterNext(
         assert_nc( iOff<=pIter->iEndofDoclist );
         if( iOff>=pIter->iEndofDoclist ){
           bNewTerm = 1;
-          if( pIter->iTermLeafPgno==pIter->iLeafPgno ){
-            pIter->iTermIdx++;
-          }else{
-            pIter->iTermIdx = 0;
-          }
           if( iOff!=fts5LeafFirstTermOff(pLeaf) ){
             iOff += fts5GetVarint32(&a[iOff], nKeep);
           }
@@ -1835,12 +1827,21 @@ static void fts5SegIterNext(
           if( (iOff = fts5LeafFirstRowidOff(pLeaf)) && iOff<pLeaf->szLeaf ){
             iOff += sqlite3Fts5GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
             pIter->iLeafOffset = iOff;
+
+            if( pLeaf->nn>pLeaf->szLeaf ){
+              pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
+                  &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist
+              );
+            }
+
           }
           else if( pLeaf->nn>pLeaf->szLeaf ){
-            iOff = fts5LeafFirstTermOff(pLeaf);
+            pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32(
+                &pLeaf->p[pLeaf->szLeaf], iOff
+            );
             pIter->iLeafOffset = iOff;
+            pIter->iEndofDoclist = iOff;
             bNewTerm = 1;
-            pIter->iTermIdx = 0;
           }
           if( iOff>=pLeaf->szLeaf ){
             p->rc = FTS5_CORRUPT;
@@ -1856,6 +1857,7 @@ static void fts5SegIterNext(
             fts5DataRelease(pIter->pLeaf);
             pIter->pLeaf = 0;
           }else{
+            int nExtra;
             fts5SegIterLoadTerm(p, pIter, nKeep);
             fts5SegIterLoadNPos(p, pIter);
             if( pbNewTerm ) *pbNewTerm = 1;
@@ -1941,7 +1943,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
     if( fts5LeafIsTermless(pLast) ){
       pIter->iEndofDoclist = pLast->nn+1;
     }else{
-      pIter->iEndofDoclist = fts5LeafTermOff(pLast, 0);
+      pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast);
     }
 
   }
@@ -2010,36 +2012,35 @@ static void fts5LeafSeek(
 ){
   int iOff;
   const u8 *a = pIter->pLeaf->p;
-  int n = pIter->pLeaf->szLeaf;
+  int szLeaf = pIter->pLeaf->szLeaf;
+  int n = pIter->pLeaf->nn;
 
   int nMatch = 0;
   int nKeep = 0;
   int nNew = 0;
   int iTerm = 0;
-  int nPgTerm = (pIter->pLeaf->nn - pIter->pLeaf->szLeaf) >> 1;
+  int iTermOff;
+  int iPgidx;                     /* Current offset in pgidx */
+  int bEndOfPage = 0;
 
   assert( p->rc==SQLITE_OK );
-  assert( pIter->pLeaf );
 
-  iOff = fts5LeafFirstTermOff(pIter->pLeaf);
-  if( iOff<4 || iOff>=n ){
-    p->rc = FTS5_CORRUPT;
-    return;
-  }
+  iPgidx = szLeaf;
+  iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff);
+  iOff = iTermOff;
 
   while( 1 ){
-    int i;
-    int nCmp;
 
     /* Figure out how many new bytes are in this term */
     fts5IndexGetVarint32(a, iOff, nNew);
-
     if( nKeep<nMatch ){
       goto search_failed;
     }
 
     assert( nKeep>=nMatch );
     if( nKeep==nMatch ){
+      int nCmp;
+      int i;
       nCmp = MIN(nNew, nTerm-nMatch);
       for(i=0; i<nCmp; i++){
         if( a[iOff+i]!=pTerm[nMatch+i] ) break;
@@ -2056,14 +2057,15 @@ static void fts5LeafSeek(
         goto search_failed;
       }
     }
-    iOff += nNew;
 
-    iTerm++;
-    if( iTerm>=nPgTerm ){
-      iOff = n;
+    if( iPgidx>=n ){
+      bEndOfPage = 1;
       break;
     }
-    iOff = fts5GetU16(&a[n + iTerm*2]);
+
+    iPgidx += fts5GetVarint32(&a[iPgidx], nKeep);
+    iTermOff += nKeep;
+    iOff = iTermOff;
 
     /* Read the nKeep field of the next term. */
     fts5IndexGetVarint32(a, iOff, nKeep);
@@ -2074,14 +2076,14 @@ static void fts5LeafSeek(
     fts5DataRelease(pIter->pLeaf);
     pIter->pLeaf = 0;
     return;
-  }else if( iOff>=n ){
+  }else if( bEndOfPage ){
     do {
       iTerm = 0;
       fts5SegIterNextPage(p, pIter);
       if( pIter->pLeaf==0 ) return;
       a = pIter->pLeaf->p;
       if( fts5LeafIsTermless(pIter->pLeaf)==0 ){
-        iOff = fts5LeafFirstTermOff(pIter->pLeaf);
+        fts5GetVarint32(&pIter->pLeaf->p[pIter->pLeaf->szLeaf], iOff);
         if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){
           p->rc = FTS5_CORRUPT;
         }else{
@@ -2094,7 +2096,6 @@ static void fts5LeafSeek(
   }
 
  search_success:
-  pIter->iTermIdx = iTerm;
 
   pIter->iLeafOffset = iOff + nNew;
   pIter->iTermLeafOffset = pIter->iLeafOffset;
@@ -2103,7 +2104,15 @@ static void fts5LeafSeek(
   fts5BufferSet(&p->rc, &pIter->term, nKeep, pTerm);
   fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]);
 
-  fts5SegIterLoadEod(p, pIter);
+  if( iPgidx>=n ){
+    pIter->iEndofDoclist = pIter->pLeaf->nn+1;
+  }else{
+    int nExtra;
+    iPgidx += fts5GetVarint32(&a[iPgidx], nExtra);
+    pIter->iEndofDoclist = iTermOff + nExtra;
+  }
+  pIter->iPgidxOff = iPgidx;
+
   fts5SegIterLoadRowid(p, pIter);
   fts5SegIterLoadNPos(p, pIter);
 }
@@ -3173,6 +3182,7 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){
   fts5BufferZero(&pPage->buf);
   fts5BufferZero(&pPage->pgidx);
   fts5BufferAppendBlob(&p->rc, &pPage->buf, 4, zero);
+  pPage->iPrevPgidx = 0;
   pPage->pgno++;
 
   /* Increase the leaves written counter */
@@ -3204,7 +3214,7 @@ static void fts5WriteAppendTerm(
   assert( pPage->buf.n>4 || pWriter->bFirstTermInPage );
 
   /* If the current leaf page is full, flush it to disk. */
-  if( (pPage->buf.n + pPage->pgidx.n + nTerm + 2)>=p->pConfig->pgsz ){
+  if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){
     if( pPage->buf.n>4 ){
       fts5WriteFlushLeaf(p, pWriter);
     }
@@ -3212,8 +3222,14 @@ static void fts5WriteAppendTerm(
   }
   
   /* TODO1: Updating pgidx here. */
+  pPgidx->n += sqlite3Fts5PutVarint(
+      &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx
+  );
+  pPage->iPrevPgidx = pPage->buf.n;
+#if 0
   fts5PutU16(&pPgidx->p[pPgidx->n], pPage->buf.n);
   pPgidx->n += 2;
+#endif
 
   if( pWriter->bFirstTermInPage ){
     nPrefix = 0;
@@ -3408,52 +3424,6 @@ static void fts5WriteInit(
   }
 }
 
-/*
-** The buffer passed as the second argument contains a leaf page that is
-** missing its page-idx array. The first term is guaranteed to start at
-** byte offset 4 of the buffer. The szLeaf field of the leaf page header
-** is already populated.
-**
-** This function appends a page-index to the buffer. The buffer is 
-** guaranteed to be large enough to fit the page-index.
-*/
-static void fts5MakePageidx(
-  Fts5Index *p,                   /* Fts index object to store any error in */
-  Fts5Data *pOld,                 /* Original page data */
-  int iOldidx,                    /* Index of term in pOld pgidx */
-  Fts5Buffer *pBuf                /* Buffer containing new page (no pgidx) */
-){
-  if( p->rc==SQLITE_OK ){
-    int iOff;
-    int iEnd;
-    int nByte;
-    int iEndOld;                  /* Byte after term iOldIdx on old page */
-    int iEndNew;                  /* Byte after term iOldIdx on new page */
-    int ii;
-    int nPgIdx = (pOld->nn - pOld->szLeaf) / 2;
-
-    /* Determine end of term on old page */
-    iEndOld = fts5LeafTermOff(pOld, iOldidx);
-    if( iOldidx>0 ){
-      iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte);
-    }
-    iEndOld += fts5GetVarint32(&pOld->p[iEndOld], nByte);
-    iEndOld += nByte;
-
-    /* Determine end of term on new page */
-    iEndNew = 4 + fts5GetVarint32(&pBuf->p[4], nByte);
-    iEndNew += nByte;
-
-    fts5PutU16(&pBuf->p[pBuf->n], 4);
-    pBuf->n += 2;
-    for(ii=iOldidx+1; ii<nPgIdx; ii++){
-      int iVal = fts5LeafTermOff(pOld, ii);
-      fts5PutU16(&pBuf->p[pBuf->n], iVal + (iEndNew - iEndOld));
-      pBuf->n += 2;
-    }
-  }
-}
-
 /*
 ** Iterator pIter was used to iterate through the input segments of on an
 ** incremental merge operation. This function is called if the incremental
@@ -3495,7 +3465,16 @@ static void fts5TrimSegments(Fts5Index *p, Fts5IndexIter *pIter){
         }
 
         /* Set up the new page-index array */
-        fts5MakePageidx(p, pData, pSeg->iTermIdx, &buf);
+        fts5BufferAppendVarint(&p->rc, &buf, 4);
+        if( pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+         && pSeg->iEndofDoclist<pData->szLeaf 
+        ){
+          int nDiff = pData->szLeaf - pSeg->iEndofDoclist;
+          fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4);
+          fts5BufferAppendBlob(&p->rc, &buf, 
+              pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff]
+          );
+        }
 
         fts5DataRelease(pData);
         pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno;
@@ -4942,16 +4921,25 @@ static void fts5IndexIntegrityCheckEmpty(
 
 static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){
   int nPg = (pLeaf->nn - pLeaf->szLeaf) / 2;
+  int iTermOff = 0;
   int ii;
+
   Fts5Buffer buf1 = {0,0,0};
   Fts5Buffer buf2 = {0,0,0};
 
-  for(ii=0; p->rc==SQLITE_OK && ii<nPg; ii++){
+  ii = pLeaf->szLeaf;
+  while( ii<pLeaf->nn && p->rc==SQLITE_OK ){
     int res;
-    int iOff = fts5LeafTermOff(pLeaf, ii);
+    int iOff;
+    int nIncr;
+
+    ii += fts5GetVarint32(&pLeaf->p[ii], nIncr);
+    iTermOff += nIncr;
+    iOff = iTermOff;
+
     if( iOff>=pLeaf->szLeaf ){
       p->rc = FTS5_CORRUPT;
-    }else if( ii==0 ){
+    }else if( iTermOff==nIncr ){
       int nByte;
       iOff += fts5GetVarint32(&pLeaf->p[iOff], nByte);
       if( (iOff+nByte)>pLeaf->szLeaf ){
@@ -5426,14 +5414,14 @@ static void fts5DecodeFunction(
       fts5DecodeStructure(&rc, &s, a, n);
     }
   }else{
-    Fts5Buffer term;
+    Fts5Buffer term;              /* Current term read from page */
+    int szLeaf;                   /* Offset of pgidx in a[] */
+    int iPgidxOff;
+    int iPgidxPrev = 0;           /* Previous value read from pgidx */
     int iTermOff = 0;
-    int szLeaf = 0;
     int iRowidOff = 0;
     int iOff;
-    int nPgTerm = 0;
     int nDoclist;
-    int i;
 
     memset(&term, 0, sizeof(Fts5Buffer));
 
@@ -5442,10 +5430,9 @@ static void fts5DecodeFunction(
       goto decode_out;
     }else{
       iRowidOff = fts5GetU16(&a[0]);
-      szLeaf = fts5GetU16(&a[2]);
-      nPgTerm = (n - szLeaf) / 2;
-      if( nPgTerm ){
-        iTermOff = fts5GetU16(&a[szLeaf]);
+      iPgidxOff = szLeaf = fts5GetU16(&a[2]);
+      if( iPgidxOff<n ){
+        fts5GetVarint32(&a[iPgidxOff], iTermOff);
       }
     }
 
@@ -5464,17 +5451,30 @@ static void fts5DecodeFunction(
     nDoclist = (iTermOff ? iTermOff : szLeaf) - iOff;
     fts5DecodeDoclist(&rc, &s, &a[iOff], nDoclist);
 
-    for(i=0; i<nPgTerm; i++){
-      int nByte;
-      int iEnd = (i+1)<nPgTerm ? fts5GetU16(&a[szLeaf+i*2+2]) : szLeaf;
-      iOff = fts5GetU16(&a[szLeaf + i*2]);
-      if( i>0 ){
+    while( iPgidxOff<n ){
+      int bFirst = (iPgidxOff==szLeaf);     /* True for first term on page */
+      int nByte;                            /* Bytes of data */
+      int iEnd;
+      
+      iPgidxOff += fts5GetVarint32(&a[iPgidxOff], nByte);
+      iPgidxPrev += nByte;
+      iOff = iPgidxPrev;
+
+      if( iPgidxOff<n ){
+        fts5GetVarint32(&a[iPgidxOff], nByte);
+        iEnd = iPgidxPrev + nByte;
+      }else{
+        iEnd = szLeaf;
+      }
+
+      if( bFirst==0 ){
         iOff += fts5GetVarint32(&a[iOff], nByte);
         term.n = nByte;
       }
       iOff += fts5GetVarint32(&a[iOff], nByte);
       fts5BufferAppendBlob(&rc, &term, nByte, &a[iOff]);
       iOff += nByte;
+
       sqlite3Fts5BufferAppendPrintf(
           &rc, &s, " term=%.*s", term.n, (const char*)term.p
       );
diff --git a/ext/fts5/test/fts5simple.test b/ext/fts5/test/fts5simple.test
index eeac11b5ca..c93519e6b9 100644
--- a/ext/fts5/test/fts5simple.test
+++ b/ext/fts5/test/fts5simple.test
@@ -66,8 +66,6 @@ do_execsql_test 3.1 {
   INSERT INTO t1(t1) VALUES('integrity-check');
 } {}
 
-}
-
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 4.1 {
@@ -115,5 +113,61 @@ do_execsql_test 5.3 {
 
 #db eval { SELECT fts5_decode(rowid, block) as x FROM yy_data } { puts $x }
 
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE tt USING fts5(content);
+  INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+  INSERT INTO tt VALUES('aa');
+}
+
+do_execsql_test 5.2 {
+  SELECT rowid FROM tt WHERE tt MATCH 'a*';
+} {1}
+
+do_execsql_test 5.3 {
+  DELETE FROM tt;
+  BEGIN;
+    INSERT INTO tt VALUES('aa');
+    INSERT INTO tt VALUES('ab');
+  COMMIT;
+} {}
+
+do_execsql_test 5.4 {
+  SELECT rowid FROM tt WHERE tt MATCH 'a*';
+} {1 2}
+
+}
+
+do_execsql_test 5.5 {
+  DELETE FROM tt;
+  BEGIN;
+    INSERT INTO tt VALUES('aa');
+    INSERT INTO tt VALUES('ab');
+    INSERT INTO tt VALUES('aa');
+    INSERT INTO tt VALUES('ab');
+    INSERT INTO tt VALUES('aa');
+    INSERT INTO tt VALUES('ab');
+    INSERT INTO tt VALUES('aa');
+    INSERT INTO tt VALUES('ab');
+  COMMIT;
+  SELECT rowid FROM tt WHERE tt MATCH 'a*';
+} {1 2 3 4 5 6 7 8}
+
+do_execsql_test 5.6 {
+  INSERT INTO tt(tt) VALUES('integrity-check');
+}
+
+reset_db
+do_execsql_test 5.7 {
+  CREATE VIRTUAL TABLE tt USING fts5(content);
+  INSERT INTO tt(tt, rank) VALUES('pgsz', 32);
+  INSERT INTO tt VALUES('aa ab ac ad ae af');
+}
+
+do_execsql_test 5.8 {
+  SELECT rowid FROM tt WHERE tt MATCH 'a*';
+} {1}
+
 finish_test
 
diff --git a/ext/fts5/tool/loadfts5.tcl b/ext/fts5/tool/loadfts5.tcl
index 7409cdb1a5..4bf89d7817 100644
--- a/ext/fts5/tool/loadfts5.tcl
+++ b/ext/fts5/tool/loadfts5.tcl
@@ -19,6 +19,12 @@ proc load_hierachy {dir} {
       db eval { INSERT INTO t1 VALUES($f, loadfile($f)) }
       incr ::nRow
 
+      if {$::O(trans) && ($::nRow % $::O(trans))==0} {
+        db eval { COMMIT }
+        db eval { INSERT INTO t1(t1) VALUES('integrity-check') }
+        db eval { BEGIN }
+      }
+
       if {($::nRow % $::nRowPerDot)==0} {
         puts -nonewline .
         if {($::nRow % (65*$::nRowPerDot))==0} { puts "" }
@@ -41,6 +47,7 @@ proc usage {} {
   puts stderr "  -automerge N (set the automerge parameter to N)"
   puts stderr "  -crisismerge N (set the crisismerge parameter to N)"
   puts stderr "  -prefix PREFIX (comma separated prefix= argument)"
+  puts stderr "  -trans N     (commit after N inserts - 0 == never)"
   exit 1
 }
 
@@ -51,6 +58,7 @@ set O(delete)     0
 set O(automerge)  -1
 set O(crisismerge)  -1
 set O(prefix)     ""
+set O(trans)      0
 
 if {[llength $argv]<2} usage
 set nOpt [expr {[llength $argv]-2}]
@@ -77,6 +85,11 @@ for {set i 0} {$i < $nOpt} {incr i} {
       if { [incr i]>=$nOpt } usage
       set O(limit) [lindex $argv $i]
     }
+
+    -trans {
+      if { [incr i]>=$nOpt } usage
+      set O(trans) [lindex $argv $i]
+    }
     
     -automerge {
       if { [incr i]>=$nOpt } usage
@@ -106,7 +119,7 @@ catch { load_static_extension db fts5 }
 db func loadfile loadfile
 db eval "PRAGMA page_size=4096"
 
-db transaction {
+db eval BEGIN
   set pref ""
   if {$O(prefix)!=""} { set pref ", prefix='$O(prefix)'" }
   catch {
@@ -127,7 +140,7 @@ db transaction {
     }
   }
   load_hierachy [lindex $argv end]
-}
+db eval COMMIT
 
 
 
diff --git a/manifest b/manifest
index 3a493d4cfa..b78f6f361c 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sbug\sin\spreprocessor\smacros\swithin\sfts5_main.c.
-D 2015-09-09T08:15:06.914
+C Change\sthe\sarray\sof\s16-bit\soffsets\sat\sthe\send\sof\seach\spage\sto\san\sarray\sof\svarints.
+D 2015-09-10T05:40:17.756
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -112,7 +112,7 @@ F ext/fts5/fts5_buffer.c 64dcaf36a3ebda9e84b7c3b8788887ec325e12a4
 F ext/fts5/fts5_config.c 57ee5fe71578cb494574fc0e6e51acb9a22a8695
 F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4
 F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 677c859b2064e00903a9ad847a1cfca8d36ce595
+F ext/fts5/fts5_index.c a13a652f042d2c88f74ad5edea893e68aadc1d99
 F ext/fts5/fts5_main.c 4b04c934084ea24a858438a04b5be8af3a9e0311
 F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
 F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
@@ -173,7 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
 F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
 F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
 F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
-F ext/fts5/test/fts5simple.test 40b76f77d56524a882473664c792bd4e8f2f2a0a
+F ext/fts5/test/fts5simple.test f629e24a35a9f31cfb16c9920e8c2316e3d93e94
 F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
@@ -182,7 +182,7 @@ F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5version.test 205beb2a67d9496af64df959e6a19238f69b83e8
 F ext/fts5/test/fts5vocab.test cdf97b9678484e9bad5062edf9c9106e5c3b0c5c
-F ext/fts5/tool/loadfts5.tcl 78253d64562774e7fd62b72ce866eeb3fcac4d0e
+F ext/fts5/tool/loadfts5.tcl 58e90407cc5c2b1770460119488fd7c0090d4dd3
 F ext/fts5/tool/mkfts5c.tcl 5745072c7de346e18c7f491e4c3281fe8a1cfe51
 F ext/fts5/tool/showfts5.tcl 9eaf6c3df352f98a2ab5ce1921dd94128ab1381d
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
@@ -1384,7 +1384,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 00d990061dec3661b0376bd167082942d5563bfe
-R 9536c4653d5903bd9893d3d3b5a1950d
+P 0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c
+R 25079dc3a625e241875e08ccc405324e
 U dan
-Z 248c3a89f7d69902ace7cabd0e5d1724
+Z c053aed547ab3325e91f842560bd3190
diff --git a/manifest.uuid b/manifest.uuid
index 355df4f8f3..2645b94c08 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-0eb2b9521fad6fa36e6fa374c2bc1f70b5180f7c
\ No newline at end of file
+fab245bea4f283714c17bca22428d5eb4db5935a
\ No newline at end of file