From e7036202bcc2bae5420332388dee8a74fce969a1 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 17 Aug 2016 19:05:12 +0000
Subject: [PATCH 01/14] Bias the fts5 snippet() function to return snippets
 that look like they start at the start of sentences.

FossilOrigin-Name: 60de159476edbd48dc363f7f77f09c32ea68422f
---
 ext/fts5/fts5_aux.c             | 209 +++++++++++++++++++++++++-------
 ext/fts5/test/fts5af.test       |   5 +-
 ext/fts5/test/fts5unicode2.test |   4 +-
 manifest                        |  16 +--
 manifest.uuid                   |   2 +-
 5 files changed, 181 insertions(+), 55 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index d8147f8111..ca55fd6b01 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -246,6 +246,115 @@ static void fts5HighlightFunction(
 ** End of highlight() implementation.
 **************************************************************************/
 
+/*
+** Context object passed to the fts5SentenceFinderCb() function.
+*/
+typedef struct Fts5SFinder Fts5SFinder;
+struct Fts5SFinder {
+  int iPos;                       /* Current token position */
+  int nFirstAlloc;                /* Allocated size of aFirst[] */
+  int nFirst;                     /* Number of entries in aFirst[] */
+  int *aFirst;                    /* Array of first token in each sentence */
+  const char *zDoc;               /* Document being tokenized */
+};
+
+/*
+** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if
+** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an
+** error occurs.
+*/
+static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){
+  if( p->nFirstAlloc==p->nFirst ){
+    int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64;
+    int *aNew;
+
+    aNew = (int*)sqlite3_realloc(p->aFirst, nNew*sizeof(int));
+    if( aNew==0 ) return SQLITE_NOMEM;
+    p->aFirst = aNew;
+    p->nFirstAlloc = nNew;
+  }
+  p->aFirst[p->nFirst++] = iAdd;
+  return SQLITE_OK;
+}
+
+/*
+** This function is an xTokenize() callback used by the auxiliary snippet()
+** function. Its job is to identify tokens that are the first in a sentence.
+** For each such token, an entry is added to the SFinder.aFirst[] array.
+*/
+static int fts5SentenceFinderCb(
+  void *pContext,                 /* Pointer to HighlightContext object */
+  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
+  const char *pToken,             /* Buffer containing token */
+  int nToken,                     /* Size of token in bytes */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
+){
+  int rc = SQLITE_OK;
+
+  if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+    Fts5SFinder *p = (Fts5SFinder*)pContext;
+    if( p->iPos>0 ){
+      int i;
+      char c = 0;
+      for(i=iStartOff-1; i>=0; i--){
+        c = p->zDoc[i];
+        if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break;
+      }
+      if( i!=iStartOff-1 && (c=='.' || c==':') ){
+        rc = fts5SentenceFinderAdd(p, p->iPos);
+      }
+    }else{
+      rc = fts5SentenceFinderAdd(p, 0);
+    }
+    p->iPos++;
+  }
+  return rc;
+}
+
+static int fts5SnippetScore(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  int nDocsize,                   /* Size of column in tokens */
+  unsigned char *aSeen,           /* Array with one element per query phrase */
+  int iCol,                       /* Column to score */
+  int iPos,                       /* Starting offset to score */
+  int nToken,                     /* Max tokens per snippet */
+  int *pnScore,                   /* OUT: Score */
+  int *piPos                      /* OUT: Adjusted offset */
+){
+  int rc;
+  int i;
+  int ip = 0;
+  int ic = 0;
+  int iOff = 0;
+  int iFirst = -1;
+  int nInst;
+  int nScore = 0;
+
+  rc = pApi->xInstCount(pFts, &nInst);
+  for(i=0; i<nInst && rc==SQLITE_OK; i++){
+    rc = pApi->xInst(pFts, i, &ip, &ic, &iOff);
+    if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOff<(iPos+nToken) ){
+      nScore += (aSeen[ip] ? 1 : 1000);
+      aSeen[ip] = 1;
+      if( iFirst<0 ) iFirst = iOff;
+    }
+  }
+
+  *pnScore = nScore;
+  if( piPos ){
+    int iLast = iOff + pApi->xPhraseSize(pFts, ip);
+    int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
+
+    if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
+    if( iAdj<0 ) iAdj = 0;
+    *piPos = iAdj;
+  }
+
+  return rc;
+}
+
 /*
 ** Implementation of snippet() function.
 */
@@ -267,9 +376,10 @@ static void fts5SnippetFunction(
   unsigned char *aSeen;           /* Array of "seen instance" flags */
   int iBestCol;                   /* Column containing best snippet */
   int iBestStart = 0;             /* First token of best snippet */
-  int iBestLast;                  /* Last token of best snippet */
   int nBestScore = 0;             /* Score of best snippet */
   int nColSize = 0;               /* Total size of iBestCol in tokens */
+  Fts5SFinder sFinder;            /* Used to find the beginnings of sentences */
+  int nCol;
 
   if( nVal!=5 ){
     const char *zErr = "wrong number of arguments to function snippet()";
@@ -277,13 +387,13 @@ static void fts5SnippetFunction(
     return;
   }
 
+  nCol = pApi->xColumnCount(pFts);
   memset(&ctx, 0, sizeof(HighlightContext));
   iCol = sqlite3_value_int(apVal[0]);
   ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
   zEllips = (const char*)sqlite3_value_text(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
-  iBestLast = nToken-1;
 
   iBestCol = (iCol>=0 ? iCol : 0);
   nPhrase = pApi->xPhraseCount(pFts);
@@ -291,43 +401,64 @@ static void fts5SnippetFunction(
   if( aSeen==0 ){
     rc = SQLITE_NOMEM;
   }
-
   if( rc==SQLITE_OK ){
     rc = pApi->xInstCount(pFts, &nInst);
   }
-  for(i=0; rc==SQLITE_OK && i<nInst; i++){
-    int ip, iSnippetCol, iStart;
-    memset(aSeen, 0, nPhrase);
-    rc = pApi->xInst(pFts, i, &ip, &iSnippetCol, &iStart);
-    if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){
-      int nScore = 1000;
-      int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip);
-      int j;
-      aSeen[ip] = 1;
 
-      for(j=i+1; rc==SQLITE_OK && j<nInst; j++){
-        int ic; int io; int iFinal;
-        rc = pApi->xInst(pFts, j, &ip, &ic, &io);
-        iFinal = io + pApi->xPhraseSize(pFts, ip) - 1;
-        if( rc==SQLITE_OK && ic==iSnippetCol && iLast<iStart+nToken ){
-          nScore += aSeen[ip] ? 1000 : 1;
-          aSeen[ip] = 1;
-          if( iFinal>iLast ) iLast = iFinal;
+  memset(&sFinder, 0, sizeof(Fts5SFinder));
+  for(i=0; i<nCol; i++){
+    if( iCol<0 || iCol==i ){
+      int nDoc;
+      int nDocsize;
+      int ii;
+      sFinder.iPos = 0;
+      sFinder.nFirst = 0;
+      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      );
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xColumnSize(pFts, i, &nDocsize);
+      if( rc!=SQLITE_OK ) break;
+
+      for(ii=0; rc==SQLITE_OK && ii<sFinder.nFirst; ii++){
+        int nScore;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+            sFinder.aFirst[ii], nToken, &nScore, 0
+        );
+
+        /* Bonus of 100 points for starting at the start of a sentence */
+        nScore += 100;            
+
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = sFinder.aFirst[ii];
+          nColSize = nDocsize;
         }
       }
 
-      if( rc==SQLITE_OK && nScore>nBestScore ){
-        iBestCol = iSnippetCol;
-        iBestStart = iStart;
-        iBestLast = iLast;
-        nBestScore = nScore;
+      for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
+        int ip, ic, io;
+        int nScore;
+        rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
+        if( ic!=i || rc!=SQLITE_OK ) continue;
+        memset(aSeen, 0, nPhrase);
+        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
+            io, nToken, &nScore, &io
+        );
+        if( rc==SQLITE_OK && nScore>nBestScore ){
+          nBestScore = nScore;
+          iBestCol = i;
+          iBestStart = io;
+          nColSize = nDocsize;
+        }
       }
     }
   }
 
-  if( rc==SQLITE_OK ){
-    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
-  }
   if( rc==SQLITE_OK ){
     rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   }
@@ -336,14 +467,6 @@ static void fts5SnippetFunction(
       rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
     }
 
-    if( (iBestStart+nToken-1)>iBestLast ){
-      iBestStart -= (iBestStart+nToken-1-iBestLast) / 2;
-    }
-    if( iBestStart+nToken>nColSize ){
-      iBestStart = nColSize - nToken;
-    }
-    if( iBestStart<0 ) iBestStart = 0;
-
     ctx.iRangeStart = iBestStart;
     ctx.iRangeEnd = iBestStart + nToken - 1;
 
@@ -365,15 +488,15 @@ static void fts5SnippetFunction(
     }else{
       fts5HighlightAppend(&rc, &ctx, zEllips, -1);
     }
-
-    if( rc==SQLITE_OK ){
-      sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
-    }else{
-      sqlite3_result_error_code(pCtx, rc);
-    }
-    sqlite3_free(ctx.zOut);
   }
+  if( rc==SQLITE_OK ){
+    sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT);
+  }else{
+    sqlite3_result_error_code(pCtx, rc);
+  }
+  sqlite3_free(ctx.zOut);
   sqlite3_free(aSeen);
+  sqlite3_free(sFinder.aFirst);
 }
 
 /************************************************************************/
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 21854662d3..903dd2910f 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -59,7 +59,6 @@ proc do_snippet_test {tn doc match res} {
 
 
 foreach {tn doc res} {
-
   1.1 {X o o o o o o} {[X] o o o o o o}
   1.2 {o X o o o o o} {o [X] o o o o o}
   1.3 {o o X o o o o} {o o [X] o o o o}
@@ -111,6 +110,10 @@ foreach {tn doc res} {
 
   8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
       {...o o [X] [X] [X] o o...}
+  8.2 {o o o o. X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {...[X] o o o o o o...}
+
+
 } {
   do_snippet_test 1.$tn $doc X $res
 }
diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test
index d3ff5128da..9fead57c07 100644
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@@ -160,11 +160,11 @@ foreach {tn query snippet} {
      the maximum x value.
   }
   4 "rollback" {
-     ...[ROLLBACK]. Instead, the pending statement
+     ...Instead, the pending statement
      will return SQLITE_ABORT upon next access after the [ROLLBACK].
   }
   5 "rOllback" {
-     ...[ROLLBACK]. Instead, the pending statement
+     ...Instead, the pending statement
      will return SQLITE_ABORT upon next access after the [ROLLBACK].
   }
   6 "lang*" {
diff --git a/manifest b/manifest
index 2dccb1db29..4fbf53dc5a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sa\sproblem\sin\sthe\sfts5\ssnippet()\sauxiliary\sfunction.
-D 2016-08-17T11:14:39.949
+C Bias\sthe\sfts5\ssnippet()\sfunction\sto\sreturn\ssnippets\sthat\slook\slike\sthey\sstart\sat\sthe\sstart\sof\ssentences.
+D 2016-08-17T19:05:12.783
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c e4bec077c5190946dbaac72c6555defd823724ca
+F ext/fts5/fts5_aux.c 5921bbef4c78b86159111335135837a867f1ff8a
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b6afd7c28ad62d546c30f387fb971f3aaebaac0d
+F ext/fts5/test/fts5af.test 38bfc8c3f2fd4687a37ae3a0d4d64e029273df36
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -190,7 +190,7 @@ F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
+F ext/fts5/test/fts5unicode2.test cb4918278514eeb6f398a28ce71ba52934f50ce5
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f9035b8e2ea331801402bcb62b203ab092949770
-R c6d38a9772924d9b143e2aa32916b131
+P a861713cc6a3868a1c89240e8340bc7b2b9559da
+R ddec9217c48a760254ac22da18301cf5
 U dan
-Z c5e29b567d02bec227e0b49d5b8a188c
+Z 28e0d85dd23359fe591994acb5271e6f
diff --git a/manifest.uuid b/manifest.uuid
index 77bfe841ee..cfefaabd19 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a861713cc6a3868a1c89240e8340bc7b2b9559da
\ No newline at end of file
+60de159476edbd48dc363f7f77f09c32ea68422f
\ No newline at end of file

From b39a5ac22946a1de8b70e48260d02d6bd4598d0c Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 18 Aug 2016 14:47:48 +0000
Subject: [PATCH 02/14] Adjust some tests to account for recent changes to the
 fts5 snippet function.

FossilOrigin-Name: 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
---
 ext/fts5/test/fts5af.test       | 76 ++++++++++++++++++---------------
 ext/fts5/test/fts5unicode2.test |  8 ++--
 manifest                        | 14 +++---
 manifest.uuid                   |  2 +-
 4 files changed, 54 insertions(+), 46 deletions(-)

diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 903dd2910f..3b71a9ff5f 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -59,6 +59,7 @@ proc do_snippet_test {tn doc match res} {
 
 
 foreach {tn doc res} {
+
   1.1 {X o o o o o o} {[X] o o o o o o}
   1.2 {o X o o o o o} {o [X] o o o o o}
   1.3 {o o X o o o o} {o o [X] o o o o}
@@ -71,49 +72,56 @@ foreach {tn doc res} {
   2.2 {o X o o o o o o} {o [X] o o o o o...}
   2.3 {o o X o o o o o} {o o [X] o o o o...}
   2.4 {o o o X o o o o} {o o o [X] o o o...}
-  2.5 {o o o o X o o o} {...o o o [X] o o o}
-  2.6 {o o o o o X o o} {...o o o o [X] o o}
-  2.7 {o o o o o o X o} {...o o o o o [X] o}
+  2.5 {o o o o X o o o} {o o o o [X] o o...}
+  2.6 {o o o o o X o o} {o o o o o [X] o...}
+  2.7 {o o o o o o X o} {o o o o o o [X]...}
   2.8 {o o o o o o o X} {...o o o o o o [X]}
 
+  2.9  {o o o o o o o X o}       {...o o o o o [X] o}
+  2.10 {o o o o o o o X o o}     {...o o o o [X] o o}
+  2.11 {o o o o o o o X o o o}   {...o o o [X] o o o}
+  2.12 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+
+
   3.1 {X o o o o o o o o} {[X] o o o o o o...}
   3.2 {o X o o o o o o o} {o [X] o o o o o...}
   3.3 {o o X o o o o o o} {o o [X] o o o o...}
   3.4 {o o o X o o o o o} {o o o [X] o o o...}
-  3.5 {o o o o X o o o o} {...o o o [X] o o o...}
-  3.6 {o o o o o X o o o} {...o o o [X] o o o}
-  3.7 {o o o o o o X o o} {...o o o o [X] o o}
-  3.8 {o o o o o o o X o} {...o o o o o [X] o}
-  3.9 {o o o o o o o o X} {...o o o o o o [X]}
+
+  3.5 {o o o o o o o X o o o o} {...o o o [X] o o o...}
+  3.6 {o o o o o o o o X o o o} {...o o o [X] o o o}
+  3.7 {o o o o o o o o o X o o} {...o o o o [X] o o}
+  3.8 {o o o o o o o o o o X o} {...o o o o o [X] o}
+  3.9 {o o o o o o o o o o o X} {...o o o o o o [X]}
 
   4.1 {X o o o o o X o o} {[X] o o o o o [X]...}
-  4.2 {o X o o o o o X o} {...[X] o o o o o [X]...}
-  4.3 {o o X o o o o o X} {...[X] o o o o o [X]}
+  4.2 {o o o o o o o X o o o o o X o} {...[X] o o o o o [X]...}
+  4.3 {o o o o o o o o X o o o o o X} {...[X] o o o o o [X]}
 
   5.1 {X o o o o X o o o} {[X] o o o o [X] o...}
-  5.2 {o X o o o o X o o} {...[X] o o o o [X] o...}
-  5.3 {o o X o o o o X o} {...[X] o o o o [X] o}
-  5.4 {o o o X o o o o X} {...o [X] o o o o [X]}
+  5.2 {o o o o o o o X o o o o X o o} {...[X] o o o o [X] o...}
+  5.3 {o o o o o o o o X o o o o X o} {...[X] o o o o [X] o}
+  5.4 {o o o o o o o o o X o o o o X} {...o [X] o o o o [X]}
 
   6.1 {X o o o X o o o} {[X] o o o [X] o o...}
   6.2 {o X o o o X o o o} {o [X] o o o [X] o...}
-  6.3 {o o X o o o X o o} {...o [X] o o o [X] o...}
-  6.4 {o o o X o o o X o} {...o [X] o o o [X] o}
-  6.5 {o o o o X o o o X} {...o o [X] o o o [X]}
+  6.3 {o o o o o o o X o o o X o o} {...o [X] o o o [X] o...}
+  6.4 {o o o o o o o o X o o o X o} {...o [X] o o o [X] o}
+  6.5 {o o o o o o o o o X o o o X} {...o o [X] o o o [X]}
 
   7.1 {X o o X o o o o o} {[X] o o [X] o o o...}
   7.2 {o X o o X o o o o} {o [X] o o [X] o o...}
-  7.3 {o o X o o X o o o} {...o [X] o o [X] o o...}
-  7.4 {o o o X o o X o o} {...o [X] o o [X] o o}
-  7.5 {o o o o X o o X o} {...o o [X] o o [X] o}
-  7.6 {o o o o o X o o X} {...o o o [X] o o [X]}
+  7.3 {o o o o o o o X o o X o o o} {...o [X] o o [X] o o...}
+  7.4 {o o o o o o o o X o o X o o} {...o [X] o o [X] o o}
+  7.5 {o o o o o o o o o X o o X o} {...o o [X] o o [X] o}
+  7.6 {o o o o o o o o o o X o o X} {...o o o [X] o o [X]}
 
-  8.1 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o}
+  8.1 {o o o o o o o o o X o o o o o o o o o o o o o o o o X X X o o o}
       {...o o [X] [X] [X] o o...}
-  8.2 {o o o o. X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
-      {...[X] o o o o o o...}
-
-
+  8.2 {o o o o o o o. o o X o o o o o o o o o o o o o o o o X X X o o o} 
+      {...o o [X] o o o o...}
+  8.3 {o o o o X o o o o o o o o o o o o o o o o o o o o o X X X o o o} 
+      {o o o o [X] o o...}
 } {
   do_snippet_test 1.$tn $doc X $res
 }
@@ -130,19 +138,19 @@ if {[detail_is_full]} {
     2.1 {X Y o o o o o o} {[X Y] o o o o o...}
     2.2 {o X Y o o o o o} {o [X Y] o o o o...}
     2.3 {o o X Y o o o o} {o o [X Y] o o o...}
-    2.4 {o o o X Y o o o} {...o o [X Y] o o o}
-    2.5 {o o o o X Y o o} {...o o o [X Y] o o}
-    2.6 {o o o o o X Y o} {...o o o o [X Y] o}
-    2.7 {o o o o o o X Y} {...o o o o o [X Y]}
+    2.4 {o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    2.5 {o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    2.6 {o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    2.7 {o o o o o o o o o o X Y} {...o o o o o [X Y]}
 
     3.1 {X Y o o o o o o o} {[X Y] o o o o o...}
     3.2 {o X Y o o o o o o} {o [X Y] o o o o...}
     3.3 {o o X Y o o o o o} {o o [X Y] o o o...}
-    3.4 {o o o X Y o o o o} {...o o [X Y] o o o...}
-    3.5 {o o o o X Y o o o} {...o o [X Y] o o o}
-    3.6 {o o o o o X Y o o} {...o o o [X Y] o o}
-    3.7 {o o o o o o X Y o} {...o o o o [X Y] o}
-    3.8 {o o o o o o o X Y} {...o o o o o [X Y]}
+    3.4 {o o o o o o o X Y o o o o} {...o o [X Y] o o o...}
+    3.5 {o o o o o o o o X Y o o o} {...o o [X Y] o o o}
+    3.6 {o o o o o o o o o X Y o o} {...o o o [X Y] o o}
+    3.7 {o o o o o o o o o o X Y o} {...o o o o [X Y] o}
+    3.8 {o o o o o o o o o o o X Y} {...o o o o o [X Y]}
   } {
     do_snippet_test 2.$tn $doc "X + Y" $res
   }
diff --git a/ext/fts5/test/fts5unicode2.test b/ext/fts5/test/fts5unicode2.test
index 9fead57c07..8e5bb8e22b 100644
--- a/ext/fts5/test/fts5unicode2.test
+++ b/ext/fts5/test/fts5unicode2.test
@@ -160,12 +160,12 @@ foreach {tn query snippet} {
      the maximum x value.
   }
   4 "rollback" {
-     ...Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
   }
   5 "rOllback" {
-     ...Instead, the pending statement
-     will return SQLITE_ABORT upon next access after the [ROLLBACK].
+     Pending statements no longer block [ROLLBACK]. Instead, the pending
+     statement will return SQLITE_ABORT upon...
   }
   6 "lang*" {
      Added support for the FTS4 [languageid] option.
diff --git a/manifest b/manifest
index 4fbf53dc5a..9749b2bf53 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Bias\sthe\sfts5\ssnippet()\sfunction\sto\sreturn\ssnippets\sthat\slook\slike\sthey\sstart\sat\sthe\sstart\sof\ssentences.
-D 2016-08-17T19:05:12.783
+C Adjust\ssome\stests\sto\saccount\sfor\srecent\schanges\sto\sthe\sfts5\ssnippet\sfunction.
+D 2016-08-18T14:47:48.448
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test 38bfc8c3f2fd4687a37ae3a0d4d64e029273df36
+F ext/fts5/test/fts5af.test b601bbab5a35ca15dd3d1069ee7075a859567073
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -190,7 +190,7 @@ F ext/fts5/test/fts5tok1.test beb894c6f3468f10a574302f69ebe4436b0287c7
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
 F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
-F ext/fts5/test/fts5unicode2.test cb4918278514eeb6f398a28ce71ba52934f50ce5
+F ext/fts5/test/fts5unicode2.test 529ac7e8648c943bc87bfed1e427128a2f3f9e33
 F ext/fts5/test/fts5unicode3.test 35c3d02aa7acf7d43d8de3bfe32c15ba96e8928e
 F ext/fts5/test/fts5unindexed.test e9539d5b78c677315e7ed8ea911d4fd25437c680
 F ext/fts5/test/fts5update.test 57c7012a7919889048947addae10e0613df45529
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a861713cc6a3868a1c89240e8340bc7b2b9559da
-R ddec9217c48a760254ac22da18301cf5
+P 60de159476edbd48dc363f7f77f09c32ea68422f
+R 8546174dc7a44b2b80755efe3cb03b8a
 U dan
-Z 28e0d85dd23359fe591994acb5271e6f
+Z b3f4b9f04ef4ae0622fcf2d28707aee5
diff --git a/manifest.uuid b/manifest.uuid
index cfefaabd19..eb9043217f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-60de159476edbd48dc363f7f77f09c32ea68422f
\ No newline at end of file
+184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
\ No newline at end of file

From 7ca1ea16ff99968db8994405c2d1c542e9a9d22f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Aug 2016 14:25:38 +0000
Subject: [PATCH 03/14] Have the fts5 snippet() function avoid favouring
 snippets that begin with sentences that do not contain search terms. Add an
 extra bias in favour of the first sentence in the document.

FossilOrigin-Name: d8180af2adfc101dfce95a0498b5cd7b30643b30
---
 ext/fts5/fts5_aux.c | 47 ++++++++++++++++++++++++++-------------------
 manifest            | 12 ++++++------
 manifest.uuid       |  2 +-
 3 files changed, 34 insertions(+), 27 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index ca55fd6b01..8efe05968e 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -422,39 +422,46 @@ static void fts5SnippetFunction(
       rc = pApi->xColumnSize(pFts, i, &nDocsize);
       if( rc!=SQLITE_OK ) break;
 
-      for(ii=0; rc==SQLITE_OK && ii<sFinder.nFirst; ii++){
-        int nScore;
-        memset(aSeen, 0, nPhrase);
-        rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
-            sFinder.aFirst[ii], nToken, &nScore, 0
-        );
-
-        /* Bonus of 100 points for starting at the start of a sentence */
-        nScore += 100;            
-
-        if( rc==SQLITE_OK && nScore>nBestScore ){
-          nBestScore = nScore;
-          iBestCol = i;
-          iBestStart = sFinder.aFirst[ii];
-          nColSize = nDocsize;
-        }
-      }
-
       for(ii=0; rc==SQLITE_OK && ii<nInst; ii++){
         int ip, ic, io;
+        int iAdj;
         int nScore;
+        int jj;
+
         rc = pApi->xInst(pFts, ii, &ip, &ic, &io);
         if( ic!=i || rc!=SQLITE_OK ) continue;
         memset(aSeen, 0, nPhrase);
         rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i,
-            io, nToken, &nScore, &io
+            io, nToken, &nScore, &iAdj
         );
         if( rc==SQLITE_OK && nScore>nBestScore ){
           nBestScore = nScore;
           iBestCol = i;
-          iBestStart = io;
+          iBestStart = iAdj;
           nColSize = nDocsize;
         }
+
+        if( rc==SQLITE_OK && sFinder.nFirst ){
+          for(jj=0; jj<(sFinder.nFirst-1); jj++){
+            if( sFinder.aFirst[jj+1]>io ) break;
+          }
+
+          if( sFinder.aFirst[jj]<io ){
+            int nScore;
+            memset(aSeen, 0, nPhrase);
+            rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, 
+              sFinder.aFirst[jj], nToken, &nScore, 0
+            );
+
+            nScore += (sFinder.aFirst[jj]==0 ? 120 : 100);
+            if( rc==SQLITE_OK && nScore>nBestScore ){
+              nBestScore = nScore;
+              iBestCol = i;
+              iBestStart = sFinder.aFirst[jj];
+              nColSize = nDocsize;
+            }
+          }
+        }
       }
     }
   }
diff --git a/manifest b/manifest
index 9749b2bf53..f261342285 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Adjust\ssome\stests\sto\saccount\sfor\srecent\schanges\sto\sthe\sfts5\ssnippet\sfunction.
-D 2016-08-18T14:47:48.448
+C Have\sthe\sfts5\ssnippet()\sfunction\savoid\sfavouring\ssnippets\sthat\sbegin\swith\ssentences\sthat\sdo\snot\scontain\ssearch\sterms.\sAdd\san\sextra\sbias\sin\sfavour\sof\sthe\sfirst\ssentence\sin\sthe\sdocument.
+D 2016-08-19T14:25:38.928
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 5921bbef4c78b86159111335135837a867f1ff8a
+F ext/fts5/fts5_aux.c 26a2b7e5f02cdc6b7322b85f473e488de452a5be
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 60de159476edbd48dc363f7f77f09c32ea68422f
-R 8546174dc7a44b2b80755efe3cb03b8a
+P 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
+R 1ef5a5b8d26eab21fbb58bf0464e61ed
 U dan
-Z b3f4b9f04ef4ae0622fcf2d28707aee5
+Z ed743fa5b6cead3848c470a1b5a47844
diff --git a/manifest.uuid b/manifest.uuid
index eb9043217f..c3129c10b9 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
\ No newline at end of file
+d8180af2adfc101dfce95a0498b5cd7b30643b30
\ No newline at end of file

From c9e75fb23abf0a7ba620bbc886efa63d9fefd8b0 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 19 Aug 2016 18:37:35 +0000
Subject: [PATCH 04/14] Register any built-in fts5 module before loading
 automatic extensions. This allows automatic extensions to register fts5
 tokenizers and auxiliary functions.

FossilOrigin-Name: b10e31dce8e590e1ba30f512374a8c16656bcc72
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/main.c    | 17 ++++++++++-------
 3 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/manifest b/manifest
index f261342285..a68dc1c021 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\sthe\sfts5\ssnippet()\sfunction\savoid\sfavouring\ssnippets\sthat\sbegin\swith\ssentences\sthat\sdo\snot\scontain\ssearch\sterms.\sAdd\san\sextra\sbias\sin\sfavour\sof\sthe\sfirst\ssentence\sin\sthe\sdocument.
-D 2016-08-19T14:25:38.928
+C Register\sany\sbuilt-in\sfts5\smodule\sbefore\sloading\sautomatic\sextensions.\sThis\sallows\sautomatic\sextensions\sto\sregister\sfts5\stokenizers\sand\sauxiliary\sfunctions.
+D 2016-08-19T18:37:35.386
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -349,7 +349,7 @@ F src/hwtime.h 747c1bbe9df21a92e9c50f3bbec1de841dc5e5da
 F src/insert.c 8f4e9fcbd8e95e85f15647ba8b413b18d556ec2b
 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e
 F src/loadext.c dd7a2b77902cc66c22555aef02e1a682554b7aec
-F src/main.c 16c1b2114eae8804caf3a8de8cb47bf2c6d83ad3
+F src/main.c 4f29b8d1b705fc3b73b26f892e85df2bfca31cff
 F src/malloc.c 1443d1ad95d67c21d77af7ae3f44678252f0efec
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 184ecbe9c0c3280a22cdeda5c7836a7b280c3e1f
-R 1ef5a5b8d26eab21fbb58bf0464e61ed
+P d8180af2adfc101dfce95a0498b5cd7b30643b30
+R 60073adaf70faac8deb27fa21c3071ec
 U dan
-Z ed743fa5b6cead3848c470a1b5a47844
+Z db6d5df26cfba25421660b26226c4ab6
diff --git a/manifest.uuid b/manifest.uuid
index c3129c10b9..674031237c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d8180af2adfc101dfce95a0498b5cd7b30643b30
\ No newline at end of file
+b10e31dce8e590e1ba30f512374a8c16656bcc72
\ No newline at end of file
diff --git a/src/main.c b/src/main.c
index f31f865818..4e4836238b 100644
--- a/src/main.c
+++ b/src/main.c
@@ -2948,11 +2948,20 @@ static int openDatabase(
   */
   sqlite3Error(db, SQLITE_OK);
   sqlite3RegisterPerConnectionBuiltinFunctions(db);
+  rc = sqlite3_errcode(db);
+
+#ifdef SQLITE_ENABLE_FTS5
+  /* Register any built-in FTS5 module before loading the automatic
+  ** extensions. This allows automatic extensions to register FTS5 
+  ** tokenizers and auxiliary functions.  */
+  if( !db->mallocFailed && rc==SQLITE_OK ){
+    rc = sqlite3Fts5Init(db);
+  }
+#endif
 
   /* Load automatic extensions - extensions that have been registered
   ** using the sqlite3_automatic_extension() API.
   */
-  rc = sqlite3_errcode(db);
   if( rc==SQLITE_OK ){
     sqlite3AutoLoadExtensions(db);
     rc = sqlite3_errcode(db);
@@ -2981,12 +2990,6 @@ static int openDatabase(
   }
 #endif
 
-#ifdef SQLITE_ENABLE_FTS5
-  if( !db->mallocFailed && rc==SQLITE_OK ){
-    rc = sqlite3Fts5Init(db);
-  }
-#endif
-
 #ifdef SQLITE_ENABLE_ICU
   if( !db->mallocFailed && rc==SQLITE_OK ){
     rc = sqlite3IcuInit(db);

From 783e778f3865bb86a5dfcd1bc830f746ed2705bb Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 20 Aug 2016 17:23:45 +0000
Subject: [PATCH 05/14] Fix other minor problems with the fts5 snippet()
 function.

FossilOrigin-Name: b56b2a85cf377686727f3929208ede43f797cf76
---
 ext/fts5/fts5_aux.c       |  5 ++++-
 ext/fts5/test/fts5af.test |  8 ++++++++
 manifest                  | 14 +++++++-------
 manifest.uuid             |  2 +-
 4 files changed, 20 insertions(+), 9 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 8efe05968e..f2aacbff20 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -441,7 +441,7 @@ static void fts5SnippetFunction(
           nColSize = nDocsize;
         }
 
-        if( rc==SQLITE_OK && sFinder.nFirst ){
+        if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){
           for(jj=0; jj<(sFinder.nFirst-1); jj++){
             if( sFinder.aFirst[jj+1]>io ) break;
           }
@@ -469,6 +469,9 @@ static void fts5SnippetFunction(
   if( rc==SQLITE_OK ){
     rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn);
   }
+  if( rc==SQLITE_OK && nColSize==0 ){
+    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
+  }
   if( ctx.zIn ){
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index 3b71a9ff5f..c8f815dfda 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -156,6 +156,14 @@ if {[detail_is_full]} {
   }
 }
 
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('xyz', '1 2 3 4 5 6 7 8 9 10 11 12 13');
+  SELECT snippet(x1, 1, '[', ']', '...', 5) FROM x1('xyz');
+} {
+  {1 2 3 4 5...}
+}
+
 } ;# foreach_detail_mode 
 
 finish_test
diff --git a/manifest b/manifest
index a68dc1c021..9d9fddff14 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Register\sany\sbuilt-in\sfts5\smodule\sbefore\sloading\sautomatic\sextensions.\sThis\sallows\sautomatic\sextensions\sto\sregister\sfts5\stokenizers\sand\sauxiliary\sfunctions.
-D 2016-08-19T18:37:35.386
+C Fix\sother\sminor\sproblems\swith\sthe\sfts5\ssnippet()\sfunction.
+D 2016-08-20T17:23:45.034
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 26a2b7e5f02cdc6b7322b85f473e488de452a5be
+F ext/fts5/fts5_aux.c 13246759ce03614a3b214894fa4b0f76891e97c0
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b601bbab5a35ca15dd3d1069ee7075a859567073
+F ext/fts5/test/fts5af.test b21abd9455c7351a9ef872b650d16f9dd8db2c49
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d8180af2adfc101dfce95a0498b5cd7b30643b30
-R 60073adaf70faac8deb27fa21c3071ec
+P b10e31dce8e590e1ba30f512374a8c16656bcc72
+R adacbf86d42eaec5a7639c5010230497
 U dan
-Z db6d5df26cfba25421660b26226c4ab6
+Z 520e1a56589636baad586098a109d286
diff --git a/manifest.uuid b/manifest.uuid
index 674031237c..eda24ca294 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b10e31dce8e590e1ba30f512374a8c16656bcc72
\ No newline at end of file
+b56b2a85cf377686727f3929208ede43f797cf76
\ No newline at end of file

From 37ff4d4c77662a01c7b1902486827426317142dd Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Wed, 24 Aug 2016 18:50:47 +0000
Subject: [PATCH 06/14] Fix a bug in the fts5 snippet function causing it to
 return text containing zero phrase instances.

FossilOrigin-Name: b174ed2bde43a1872374ce18b85ac9f067e86fbb
---
 ext/fts5/fts5_aux.c       |  4 ++--
 ext/fts5/test/fts5af.test | 11 +++++++++++
 manifest                  | 14 +++++++-------
 manifest.uuid             |  2 +-
 4 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index f2aacbff20..e171233050 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -331,6 +331,7 @@ static int fts5SnippetScore(
   int iFirst = -1;
   int nInst;
   int nScore = 0;
+  int iLast = 0;
 
   rc = pApi->xInstCount(pFts, &nInst);
   for(i=0; i<nInst && rc==SQLITE_OK; i++){
@@ -339,14 +340,13 @@ static int fts5SnippetScore(
       nScore += (aSeen[ip] ? 1 : 1000);
       aSeen[ip] = 1;
       if( iFirst<0 ) iFirst = iOff;
+      iLast = iOff + pApi->xPhraseSize(pFts, ip);
     }
   }
 
   *pnScore = nScore;
   if( piPos ){
-    int iLast = iOff + pApi->xPhraseSize(pFts, ip);
     int iAdj = iFirst - (nToken - (iLast-iFirst)) / 2;
-
     if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken;
     if( iAdj<0 ) iAdj = 0;
     *piPos = iAdj;
diff --git a/ext/fts5/test/fts5af.test b/ext/fts5/test/fts5af.test
index c8f815dfda..6aab55a557 100644
--- a/ext/fts5/test/fts5af.test
+++ b/ext/fts5/test/fts5af.test
@@ -164,6 +164,17 @@ do_execsql_test 4.0 {
   {1 2 3 4 5...}
 }
 
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE p1 USING fts5(a, b);
+  INSERT INTO p1 VALUES(
+    'x a a a a a a a a a a',
+    'a a a a a a a a a a a a a a a a a a a x'
+  );
+}
+do_execsql_test 5.1 {
+  SELECT snippet(p1, 0, '[', ']', '...', 6) FROM p1('x');
+} {{[x] a a a a a...}}
+
 } ;# foreach_detail_mode 
 
 finish_test
diff --git a/manifest b/manifest
index 9d9fddff14..6ce61f548e 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sother\sminor\sproblems\swith\sthe\sfts5\ssnippet()\sfunction.
-D 2016-08-20T17:23:45.034
+C Fix\sa\sbug\sin\sthe\sfts5\ssnippet\sfunction\scausing\sit\sto\sreturn\stext\scontaining\szero\sphrase\sinstances.
+D 2016-08-24T18:50:47.193
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc d66d0395c38571aab3804f8db0fa20707ae4609a
@@ -99,7 +99,7 @@ F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
 F ext/fts5/fts5.h 62f3e33ceeb9a428db139f9c012186b371da1cc7
 F ext/fts5/fts5Int.h b2eda36e0f224365c8e23dc8f559311834f1c13f
-F ext/fts5/fts5_aux.c 13246759ce03614a3b214894fa4b0f76891e97c0
+F ext/fts5/fts5_aux.c 2f20784a344701d4c72986e2e692062dd47d568c
 F ext/fts5/fts5_buffer.c 4c1502d4c956cd092c89ce4480867f9d8bf325cd
 F ext/fts5/fts5_config.c 5af9c360e99669d29f06492c370892394aba0857
 F ext/fts5/fts5_expr.c 1ee97156421919e497595bfa962bb88ad1665401
@@ -122,7 +122,7 @@ F ext/fts5/test/fts5ab.test 30325a89453280160106be411bba3acf138e6d1b
 F ext/fts5/test/fts5ac.test 55cad4275a1f5acabfe14d8442a8046b47e49e5f
 F ext/fts5/test/fts5ad.test 36995f0586f30f5602074e012b9224c71ec5171c
 F ext/fts5/test/fts5ae.test 612dcb51f4069226791ff14c17dbfb3138c56f20
-F ext/fts5/test/fts5af.test b21abd9455c7351a9ef872b650d16f9dd8db2c49
+F ext/fts5/test/fts5af.test c92825778ed2adc80014832762c056bbe968ef88
 F ext/fts5/test/fts5ag.test 27180de76c03036be75ee80b93d8c5f540014071
 F ext/fts5/test/fts5ah.test dfb7897711dbcda1dacb038aec310daca139fcf5
 F ext/fts5/test/fts5ai.test 3909d0b949b2afcaae4d5795cd79153da75381df
@@ -1510,7 +1510,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P b10e31dce8e590e1ba30f512374a8c16656bcc72
-R adacbf86d42eaec5a7639c5010230497
+P b56b2a85cf377686727f3929208ede43f797cf76
+R d95861b85eb0fcd438ce646b06ec5daf
 U dan
-Z 520e1a56589636baad586098a109d286
+Z 1e5d85bd62c3d7d96772d546309a0d7d
diff --git a/manifest.uuid b/manifest.uuid
index eda24ca294..3fabbae740 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-b56b2a85cf377686727f3929208ede43f797cf76
\ No newline at end of file
+b174ed2bde43a1872374ce18b85ac9f067e86fbb
\ No newline at end of file

From 6650190891d252ad692ded7f29cef33bee621d9b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Sat, 27 Aug 2016 18:35:55 +0000
Subject: [PATCH 07/14] Fix some comments in sqlite3session.h. No changes to
 code.

FossilOrigin-Name: 78cd64e202fcbe9ce69070b0f48ccd0c6b48538d
---
 ext/session/sqlite3session.h | 10 +++++++---
 manifest                     | 12 ++++++------
 manifest.uuid                |  2 +-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/ext/session/sqlite3session.h b/ext/session/sqlite3session.h
index 23d9a33aef..140138f36d 100644
--- a/ext/session/sqlite3session.h
+++ b/ext/session/sqlite3session.h
@@ -727,12 +727,12 @@ int sqlite3changeset_concat(
 
 
 /*
-** Changegroup handle.
+** CAPI3REF: Changegroup Handle
 */
 typedef struct sqlite3_changegroup sqlite3_changegroup;
 
 /*
-** CAPI3REF: Combine two or more changesets into a single changeset.
+** CAPI3REF: Create A New Changegroup Object
 **
 ** An sqlite3_changegroup object is used to combine two or more changesets
 ** (or patchsets) into a single changeset (or patchset). A single changegroup
@@ -769,6 +769,8 @@ typedef struct sqlite3_changegroup sqlite3_changegroup;
 int sqlite3changegroup_new(sqlite3_changegroup **pp);
 
 /*
+** CAPI3REF: Add A Changeset To A Changegroup
+**
 ** Add all changes within the changeset (or patchset) in buffer pData (size
 ** nData bytes) to the changegroup. 
 **
@@ -844,6 +846,8 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp);
 int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData);
 
 /*
+** CAPI3REF: Obtain A Composite Changeset From A Changegroup
+**
 ** Obtain a buffer containing a changeset (or patchset) representing the
 ** current contents of the changegroup. If the inputs to the changegroup
 ** were themselves changesets, the output is a changeset. Or, if the
@@ -872,7 +876,7 @@ int sqlite3changegroup_output(
 );
 
 /*
-** Delete a changegroup object.
+** CAPI3REF: Delete A Changegroup Object
 */
 void sqlite3changegroup_delete(sqlite3_changegroup*);
 
diff --git a/manifest b/manifest
index 6ec9fe1671..365264a60f 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fixes\sto\sfts5\ssnippet()\sfunction.
-D 2016-08-27T18:34:06.238
+C Fix\ssome\scomments\sin\ssqlite3session.h.\sNo\schanges\sto\scode.
+D 2016-08-27T18:35:55.201
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -301,7 +301,7 @@ F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sqlite3session.c 37485891b4add26cf61495df193c419f36556a32
-F ext/session/sqlite3session.h 69bf73cfd71e58f2ae5d2aa935b2c1a541aee555
+F ext/session/sqlite3session.h 7b9037818ee61f7429ca83e9866885ca6de5f764
 F ext/session/test_session.c 2caed9a659586428c63ca46e4900347b374487d4
 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 4d43c4698eef4e3db7556813f0274b4018c7c2b9 b174ed2bde43a1872374ce18b85ac9f067e86fbb
-R 1806c129c8fa52174952b67042480d44
+P d464a7b18d212720a4f4e2fa4e204f1aca99c837
+R b7e47d608375b4083f5b5619e523e8d7
 U dan
-Z 3f2d594eefc4bc080c151857cd7756e9
+Z 37c50abb5bb55673ed9c9ae92f5f1ddb
diff --git a/manifest.uuid b/manifest.uuid
index aa38fb6b68..af58d2d485 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d464a7b18d212720a4f4e2fa4e204f1aca99c837
\ No newline at end of file
+78cd64e202fcbe9ce69070b0f48ccd0c6b48538d
\ No newline at end of file

From 2e5c5052fd745dce7a09c7bce2c0fc97fcf04316 Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Sat, 27 Aug 2016 20:21:51 +0000
Subject: [PATCH 08/14] Fix typos in comments.  No changes to running code.

FossilOrigin-Name: a07269f2a0f87e0b736127f528f6caf3b63f9052
---
 ext/rbu/sqlite3rbu.h |  2 +-
 ext/rtree/rtree.c    |  2 +-
 manifest             | 20 ++++++++++----------
 manifest.uuid        |  2 +-
 src/treeview.c       |  2 +-
 src/vtab.c           |  2 +-
 6 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h
index 3f5f29a8e6..12112c14e1 100644
--- a/ext/rbu/sqlite3rbu.h
+++ b/ext/rbu/sqlite3rbu.h
@@ -104,7 +104,7 @@
 ** may also be named data<integer>_<target>, where <integer> is any sequence
 ** of zero or more numeric characters (0-9). This can be significant because
 ** tables within the RBU database are always processed in order sorted by 
-** name. By judicious selection of the the <integer> portion of the names
+** name. By judicious selection of the <integer> portion of the names
 ** of the RBU tables the user can therefore control the order in which they
 ** are processed. This can be useful, for example, to ensure that "external
 ** content" FTS4 tables are updated before their underlying content tables.
diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c
index fc69acd6fa..2301e6816f 100644
--- a/ext/rtree/rtree.c
+++ b/ext/rtree/rtree.c
@@ -1542,7 +1542,7 @@ static int rtreeFilter(
   if( idxNum==1 ){
     /* Special case - lookup by rowid. */
     RtreeNode *pLeaf;        /* Leaf on which the required cell resides */
-    RtreeSearchPoint *p;     /* Search point for the the leaf */
+    RtreeSearchPoint *p;     /* Search point for the leaf */
     i64 iRowid = sqlite3_value_int64(argv[0]);
     i64 iNode = 0;
     rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode);
diff --git a/manifest b/manifest
index 365264a60f..669a25cf06 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\ssome\scomments\sin\ssqlite3session.h.\sNo\schanges\sto\scode.
-D 2016-08-27T18:35:55.201
+C Fix\stypos\sin\scomments.\s\sNo\schanges\sto\srunning\scode.
+D 2016-08-27T20:21:51.309
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -255,10 +255,10 @@ F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48
 F ext/rbu/rbuvacuum.test 4a977447c15c2581ab668781d9ef4294382530e0
 F ext/rbu/rbuvacuum2.test 2569205b74ff40fbf3bda2fce33a58eb40eebdcc
 F ext/rbu/sqlite3rbu.c e074c38798b90591f7f0cf0032d62f152ce5a95e
-F ext/rbu/sqlite3rbu.h 1d91c5b7d066645bd1ff8e4b85c2b9b5dd29fb05
+F ext/rbu/sqlite3rbu.h 6fb6294c34a9ca93b5894a33bca530c6f08decba
 F ext/rbu/test_rbu.c 5aa22616afac6f71ebd3d9bc9bf1006cfabcca88
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
-F ext/rtree/rtree.c d26a815b0df1c412a6881dae8d7fd3c9c08cce68
+F ext/rtree/rtree.c 0b450226001c8ae4622e382b2aff79127581a763
 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e
 F ext/rtree/rtree1.test 42dadfc7b44a436cd74a1bebc0b9b689e4eaf7ec
 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba
@@ -444,7 +444,7 @@ F src/test_windirent.h 7edc57e2faa727026dbd5d010dd0e2e665d5aa01
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
 F src/tokenize.c 78c8085bc7af1922aa687f0f4bbd716821330de5
-F src/treeview.c c56d6ddbed564efda746236b35bcbb8238daac4b
+F src/treeview.c 70329ef46b86f94b16a98aacced793e0692217b5
 F src/trigger.c 11e20b3b12c847b3b9055594c0f1631266bb53fc
 F src/update.c 8179e699dbd45b92934fd02d3d8e3732e8da8802
 F src/utf.c 699001c79f28e48e9bcdf8a463da029ea660540c
@@ -459,7 +459,7 @@ F src/vdbeblob.c 3e82a797b60c3b9fed7b8de8c539ca7607874937
 F src/vdbemem.c 1ecaa5ee0caff07255f25d04e8dc88befb6f88d1
 F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c
 F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834
-F src/vtab.c 5ca4fa8b028f9e2ed4793ee1670911a83cfcefd8
+F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8
 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 02eeecc265f6ffd0597378f5d8ae9070b62a406a
 F src/wal.h 6dd221ed384afdc204bc61e25c23ef7fd5a511f2
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P d464a7b18d212720a4f4e2fa4e204f1aca99c837
-R b7e47d608375b4083f5b5619e523e8d7
-U dan
-Z 37c50abb5bb55673ed9c9ae92f5f1ddb
+P 78cd64e202fcbe9ce69070b0f48ccd0c6b48538d
+R 3fe3d686aa6a61df97c243454c02cd64
+U drh
+Z 90feb3c09ae2ca31e13afbfbff662667
diff --git a/manifest.uuid b/manifest.uuid
index af58d2d485..640141a0eb 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-78cd64e202fcbe9ce69070b0f48ccd0c6b48538d
\ No newline at end of file
+a07269f2a0f87e0b736127f528f6caf3b63f9052
\ No newline at end of file
diff --git a/src/treeview.c b/src/treeview.c
index 27996d46d7..4382c3ca25 100644
--- a/src/treeview.c
+++ b/src/treeview.c
@@ -120,7 +120,7 @@ void sqlite3TreeViewWith(TreeView *pView, const With *pWith, u8 moreToFollow){
 
 
 /*
-** Generate a human-readable description of a the Select object.
+** Generate a human-readable description of a Select object.
 */
 void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
   int n = 0;
diff --git a/src/vtab.c b/src/vtab.c
index 53800dd40e..c54dc65742 100644
--- a/src/vtab.c
+++ b/src/vtab.c
@@ -672,7 +672,7 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){
 ** This function is invoked by the vdbe to call the xCreate method
 ** of the virtual table named zTab in database iDb. 
 **
-** If an error occurs, *pzErr is set to point an an English language
+** If an error occurs, *pzErr is set to point to an English language
 ** description of the error and an SQLITE_XXX error code is returned.
 ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr.
 */

From a87070a2716fde5569db592c0cebfd7d3578f968 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Mon, 29 Aug 2016 14:18:18 +0000
Subject: [PATCH 09/14] Use some of the example code from the sessions
 documenatation in the sessions test cases.

FossilOrigin-Name: 6602974d17536bcb904a794bddedffd58926794b
---
 ext/session/session_common.tcl |   3 +
 ext/session/test_session.c     | 138 +++++++++++++++++++++++++++++----
 manifest                       |  16 ++--
 manifest.uuid                  |   2 +-
 4 files changed, 133 insertions(+), 26 deletions(-)

diff --git a/ext/session/session_common.tcl b/ext/session/session_common.tcl
index 38eebec0a0..06b05509b1 100644
--- a/ext/session/session_common.tcl
+++ b/ext/session/session_common.tcl
@@ -75,6 +75,9 @@ proc do_common_sql {sql} {
 }
 
 proc changeset_from_sql {sql {dbname main}} {
+  if {$dbname == "main"} {
+    return [sql_exec_changeset db $sql]
+  }
   set rc [catch {
     sqlite3session S db $dbname
     db eval "SELECT name FROM $dbname.sqlite_master WHERE type = 'table'" {
diff --git a/ext/session/test_session.c b/ext/session/test_session.c
index 103a1c2d7a..411354cc02 100644
--- a/ext/session/test_session.c
+++ b/ext/session/test_session.c
@@ -29,6 +29,107 @@ struct TestStreamInput {
   int iData;                      /* Bytes of data already read by sessions */
 };
 
+/*
+** Extract an sqlite3* db handle from the object passed as the second
+** argument. If successful, set *pDb to point to the db handle and return
+** TCL_OK. Otherwise, return TCL_ERROR.
+*/
+static int dbHandleFromObj(Tcl_Interp *interp, Tcl_Obj *pObj, sqlite3 **pDb){
+  Tcl_CmdInfo info;
+  if( 0==Tcl_GetCommandInfo(interp, Tcl_GetString(pObj), &info) ){
+    Tcl_AppendResult(interp, "no such handle: ", Tcl_GetString(pObj), 0);
+    return TCL_ERROR;
+  }
+
+  *pDb = *(sqlite3 **)info.objClientData;
+  return TCL_OK;
+}
+
+/*************************************************************************
+** The following code is copied byte-for-byte from the sessions module
+** documentation.  It is used by some of the sessions modules tests to
+** ensure that the example in the documentation does actually work.
+*/ 
+/*
+** Argument zSql points to a buffer containing an SQL script to execute 
+** against the database handle passed as the first argument. As well as
+** executing the SQL script, this function collects a changeset recording
+** all changes made to the "main" database file. Assuming no error occurs,
+** output variables (*ppChangeset) and (*pnChangeset) are set to point
+** to a buffer containing the changeset and the size of the changeset in
+** bytes before returning SQLITE_OK. In this case it is the responsibility
+** of the caller to eventually free the changeset blob by passing it to
+** the sqlite3_free function.
+**
+** Or, if an error does occur, return an SQLite error code. The final
+** value of (*pChangeset) and (*pnChangeset) are undefined in this case.
+*/
+int sql_exec_changeset(
+  sqlite3 *db,                  /* Database handle */
+  const char *zSql,             /* SQL script to execute */
+  int *pnChangeset,             /* OUT: Size of changeset blob in bytes */
+  void **ppChangeset            /* OUT: Pointer to changeset blob */
+){
+  sqlite3_session *pSession = 0;
+  int rc;
+
+  /* Create a new session object */
+  rc = sqlite3session_create(db, "main", &pSession);
+
+  /* Configure the session object to record changes to all tables */
+  if( rc==SQLITE_OK ) rc = sqlite3session_attach(pSession, NULL);
+
+  /* Execute the SQL script */
+  if( rc==SQLITE_OK ) rc = sqlite3_exec(db, zSql, 0, 0, 0);
+
+  /* Collect the changeset */
+  if( rc==SQLITE_OK ){
+    rc = sqlite3session_changeset(pSession, pnChangeset, ppChangeset);
+  }
+
+  /* Delete the session object */
+  sqlite3session_delete(pSession);
+
+  return rc;
+}
+/************************************************************************/
+
+/*
+** Tclcmd: sql_exec_changeset DB SQL
+*/
+static int SQLITE_TCLAPI test_sql_exec_changeset(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  const char *zSql;
+  sqlite3 *db;
+  void *pChangeset;
+  int nChangeset;
+  int rc;
+
+  if( objc!=3 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB SQL");
+    return TCL_ERROR;
+  }
+  if( dbHandleFromObj(interp, objv[1], &db) ) return TCL_ERROR;
+  zSql = (const char*)Tcl_GetString(objv[2]);
+
+  rc = sql_exec_changeset(db, zSql, &nChangeset, &pChangeset);
+  if( rc!=SQLITE_OK ){
+    Tcl_ResetResult(interp);
+    Tcl_AppendResult(interp, "error in sql_exec_changeset()", 0);
+    return TCL_ERROR;
+  }
+
+  Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pChangeset, nChangeset));
+  sqlite3_free(pChangeset);
+  return TCL_OK;
+}
+
+
+
 #define SESSION_STREAM_TCL_VAR "sqlite3session_streams"
 
 /*
@@ -919,23 +1020,26 @@ static int SQLITE_TCLAPI test_sqlite3session_foreach(
 }
 
 int TestSession_Init(Tcl_Interp *interp){
-  Tcl_CreateObjCommand(interp, "sqlite3session", test_sqlite3session, 0, 0);
-  Tcl_CreateObjCommand(
-      interp, "sqlite3session_foreach", test_sqlite3session_foreach, 0, 0
-  );
-  Tcl_CreateObjCommand(
-      interp, "sqlite3changeset_invert", test_sqlite3changeset_invert, 0, 0
-  );
-  Tcl_CreateObjCommand(
-      interp, "sqlite3changeset_concat", test_sqlite3changeset_concat, 0, 0
-  );
-  Tcl_CreateObjCommand(
-      interp, "sqlite3changeset_apply", test_sqlite3changeset_apply, 0, 0
-  );
-  Tcl_CreateObjCommand(
-      interp, "sqlite3changeset_apply_replace_all", 
-      test_sqlite3changeset_apply_replace_all, 0, 0
-  );
+  struct Cmd {
+    const char *zCmd;
+    Tcl_ObjCmdProc *xProc;
+  } aCmd[] = {
+    { "sqlite3session", test_sqlite3session },
+    { "sqlite3session_foreach", test_sqlite3session_foreach },
+    { "sqlite3changeset_invert", test_sqlite3changeset_invert },
+    { "sqlite3changeset_concat", test_sqlite3changeset_concat },
+    { "sqlite3changeset_apply", test_sqlite3changeset_apply },
+    { "sqlite3changeset_apply_replace_all", 
+      test_sqlite3changeset_apply_replace_all },
+    { "sql_exec_changeset", test_sql_exec_changeset },
+  };
+  int i;
+
+  for(i=0; i<sizeof(aCmd)/sizeof(struct Cmd); i++){
+    struct Cmd *p = &aCmd[i];
+    Tcl_CreateObjCommand(interp, p->zCmd, p->xProc, 0, 0);
+  }
+
   return TCL_OK;
 }
 
diff --git a/manifest b/manifest
index 669a25cf06..be047d09d2 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\stypos\sin\scomments.\s\sNo\schanges\sto\srunning\scode.
-D 2016-08-27T20:21:51.309
+C Use\ssome\sof\sthe\sexample\scode\sfrom\sthe\ssessions\sdocumenatation\sin\sthe\ssessions\stest\scases.
+D 2016-08-29T14:18:18.207
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -297,12 +297,12 @@ F ext/session/sessionD.test d4744c78334162851d2a2f285c7e603e31b49aa2
 F ext/session/sessionE.test e60a238c47f0feb3bb707e7f35e22be09c7e8f26
 F ext/session/sessionF.test c2f178d4dfd723a5fd94a730ea2ccb44c669e3ce
 F ext/session/sessionG.test 01ef705096a9d3984eebdcca79807a211dee1b60
-F ext/session/session_common.tcl a1293167d14774b5e728836720497f40fe4ea596
+F ext/session/session_common.tcl 9b696a341cf1d3744823715ed92bb19749b6c3d4
 F ext/session/sessionfault.test da273f2712b6411e85e71465a1733b8501dbf6f7
 F ext/session/sessionfault2.test 04aa0bc9aa70ea43d8de82c4f648db4de1e990b0
 F ext/session/sqlite3session.c 37485891b4add26cf61495df193c419f36556a32
 F ext/session/sqlite3session.h 7b9037818ee61f7429ca83e9866885ca6de5f764
-F ext/session/test_session.c 2caed9a659586428c63ca46e4900347b374487d4
+F ext/session/test_session.c eb0bd6c1ea791c1d66ee4ef94c16500dad936386
 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220
 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04
 F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 78cd64e202fcbe9ce69070b0f48ccd0c6b48538d
-R 3fe3d686aa6a61df97c243454c02cd64
-U drh
-Z 90feb3c09ae2ca31e13afbfbff662667
+P a07269f2a0f87e0b736127f528f6caf3b63f9052
+R e338ebda27cef4eaa61a39e0a172ba0c
+U dan
+Z 11041cb022515aa05751c4e3c90843c8
diff --git a/manifest.uuid b/manifest.uuid
index 640141a0eb..5cc9198912 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-a07269f2a0f87e0b736127f528f6caf3b63f9052
\ No newline at end of file
+6602974d17536bcb904a794bddedffd58926794b
\ No newline at end of file

From d0d49b9ca3c106e972ca724c7202a81182f4260f Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Sep 2016 09:35:20 +0000
Subject: [PATCH 10/14] If SQLITE_ENABLE_ZIPVFS is defined, journal_mode=off is
 configured and a savepoint or statement rollback is attempted, move the pager
 into the error state to prevent the transaction from being committed. This
 makes it safe to use journal_mode=off with zipvfs under some conditions.

FossilOrigin-Name: 38d31e189e7c7899e14455f2c083aa676ce4d4c0
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/pager.c   | 20 +++++++++++++++++++-
 3 files changed, 26 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index be047d09d2..c4f3856500 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Use\ssome\sof\sthe\sexample\scode\sfrom\sthe\ssessions\sdocumenatation\sin\sthe\ssessions\stest\scases.
-D 2016-08-29T14:18:18.207
+C If\sSQLITE_ENABLE_ZIPVFS\sis\sdefined,\sjournal_mode=off\sis\sconfigured\sand\sa\ssavepoint\sor\sstatement\srollback\sis\sattempted,\smove\sthe\spager\sinto\sthe\serror\sstate\sto\sprevent\sthe\stransaction\sfrom\sbeing\scommitted.\sThis\smakes\sit\ssafe\sto\suse\sjournal_mode=off\swith\szipvfs\sunder\ssome\sconditions.
+D 2016-09-01T09:35:20.703
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -371,7 +371,7 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586
 F src/os_unix.c be9ca0f901a2b6c1bc93dc338f4863675180c189
 F src/os_win.c 520f23475f1de530c435d30b67b7b15fe90874b0
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
-F src/pager.c 40928c450320da78bb4bd3ae82818f4239e19b7e
+F src/pager.c bf5b71bde3e9b6110e7d6990607db881f6a471a2
 F src/pager.h 966d2769e76ae347c8a32c4165faf6e6cb64546d
 F src/parse.y ed6990c2d41eb0302eda90d5009c51fec792c850
 F src/pcache.c 5583c8ade4b05075a60ba953ef471d1c1a9c05df
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a07269f2a0f87e0b736127f528f6caf3b63f9052
-R e338ebda27cef4eaa61a39e0a172ba0c
+P 6602974d17536bcb904a794bddedffd58926794b
+R 78dc0041209a92fe8b940a2665fbfc99
 U dan
-Z 11041cb022515aa05751c4e3c90843c8
+Z f1c8273131a5d41903cfd86ae1fce8b9
diff --git a/manifest.uuid b/manifest.uuid
index 5cc9198912..d765505e05 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-6602974d17536bcb904a794bddedffd58926794b
\ No newline at end of file
+38d31e189e7c7899e14455f2c083aa676ce4d4c0
\ No newline at end of file
diff --git a/src/pager.c b/src/pager.c
index cd8d1204b3..dfa512b485 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -6656,7 +6656,11 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){
 ** savepoint. If no errors occur, SQLITE_OK is returned.
 */ 
 int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){
-  int rc = pPager->errCode;       /* Return code */
+  int rc = pPager->errCode;
+  
+#ifdef SQLITE_ENABLE_ZIPVFS
+  if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK;
+#endif
 
   assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK );
   assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK );
@@ -6697,6 +6701,20 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){
       rc = pagerPlaybackSavepoint(pPager, pSavepoint);
       assert(rc!=SQLITE_DONE);
     }
+    
+#ifdef SQLITE_ENABLE_ZIPVFS
+    /* If the cache has been modified but the savepoint cannot be rolled 
+    ** back journal_mode=off, put the pager in the error state. This way,
+    ** if the VFS used by this pager includes ZipVFS, the entire transaction
+    ** can be rolled back at the ZipVFS level.  */
+    else if( 
+        pPager->journalMode==PAGER_JOURNALMODE_OFF 
+     && pPager->eState>=PAGER_WRITER_CACHEMOD
+    ){
+      pPager->errCode = SQLITE_ABORT;
+      pPager->eState = PAGER_ERROR;
+    }
+#endif
   }
 
   return rc;

From e5a0cfa161a77e8ab674fc53f30154c9129eff70 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Thu, 1 Sep 2016 14:03:28 +0000
Subject: [PATCH 11/14] Have "sqldiff --rbu" ignore rows with NULL values in
 primary key fields. RBU can't handle such rows and the documentation already
 says sqldiff ignores them. Because the code now uses "=" instead of "IS" to
 filter on primary key columns, diffs on virtual tables are faster now too.

FossilOrigin-Name: f4ba894a86aa195bcbe2fa69e91cd870ec3fb577
---
 ext/rbu/rbudiff.test | 10 ++++++++++
 manifest             | 14 +++++++-------
 manifest.uuid        |  2 +-
 tool/sqldiff.c       | 12 +++++++-----
 4 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test
index 041a242428..4a32009230 100644
--- a/ext/rbu/rbudiff.test
+++ b/ext/rbu/rbudiff.test
@@ -140,6 +140,15 @@ foreach {tn init mod} {
     );
   }
 
+  4 {
+    CREATE TABLE x1(a, b, c, PRIMARY KEY(a, b, c));
+    INSERT INTO x1 VALUES('u', 'v', NULL);
+    INSERT INTO x1 VALUES('x', 'y', 'z');
+    INSERT INTO x1 VALUES('a', NULL, 'b');
+  } {
+    INSERT INTO x1 VALUES('a', 'b', 'c');
+  }
+
 } {
   catch { db close }
 
@@ -280,5 +289,6 @@ tablE t1 USING FTs5(c);
   }
 }
 
+
 finish_test
 
diff --git a/manifest b/manifest
index c4f3856500..ed5afd047d 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C If\sSQLITE_ENABLE_ZIPVFS\sis\sdefined,\sjournal_mode=off\sis\sconfigured\sand\sa\ssavepoint\sor\sstatement\srollback\sis\sattempted,\smove\sthe\spager\sinto\sthe\serror\sstate\sto\sprevent\sthe\stransaction\sfrom\sbeing\scommitted.\sThis\smakes\sit\ssafe\sto\suse\sjournal_mode=off\swith\szipvfs\sunder\ssome\sconditions.
-D 2016-09-01T09:35:20.703
+C Have\s"sqldiff\s--rbu"\signore\srows\swith\sNULL\svalues\sin\sprimary\skey\sfields.\sRBU\scan't\shandle\ssuch\srows\sand\sthe\sdocumentation\salready\ssays\ssqldiff\signores\sthem.\sBecause\sthe\scode\snow\suses\s"="\sinstead\sof\s"IS"\sto\sfilter\son\sprimary\skey\scolumns,\sdiffs\son\svirtual\stables\sare\sfaster\snow\stoo.
+D 2016-09-01T14:03:28.041
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -245,7 +245,7 @@ F ext/rbu/rbuB.test c25bc325b8072a766e56bb76c001866b405925c2
 F ext/rbu/rbuC.test efe47db508a0269b683cb2a1913a425ffd39a831
 F ext/rbu/rbu_common.tcl a38e8e2d4a50fd6aaf151633714c1b1d2fae3ead
 F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695
-F ext/rbu/rbudiff.test b3c7675810b81de98a930a87fcd40d9ae545619d
+F ext/rbu/rbudiff.test d099b56b073a737cfe1b8e9f67b77940130719cb
 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89
 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06
 F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca
@@ -1478,7 +1478,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 7ce07da76b5e745783e703a834417d725b7d45fd
 F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
-F tool/sqldiff.c c965d49bf2677db06103854b47e105484b5b1b84
+F tool/sqldiff.c 7f567367d87fdb493e3e65169569a10d9f330c3f
 F tool/srcck1.c 371de5363b70154012955544f86fdee8f6e5326f
 F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43
 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 6602974d17536bcb904a794bddedffd58926794b
-R 78dc0041209a92fe8b940a2665fbfc99
+P 38d31e189e7c7899e14455f2c083aa676ce4d4c0
+R ca589e025240033a5748a5eba7e4950c
 U dan
-Z f1c8273131a5d41903cfd86ae1fce8b9
+Z 28ca6847fc75deee037b1b39928043e8
diff --git a/manifest.uuid b/manifest.uuid
index d765505e05..78d594bd6c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-38d31e189e7c7899e14455f2c083aa676ce4d4c0
\ No newline at end of file
+f4ba894a86aa195bcbe2fa69e91cd870ec3fb577
\ No newline at end of file
diff --git a/tool/sqldiff.c b/tool/sqldiff.c
index b1363bb341..cbb57e7774 100644
--- a/tool/sqldiff.c
+++ b/tool/sqldiff.c
@@ -1179,8 +1179,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM aux.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " main.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n)");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Deleted rows: */
   strPrintf(pSql, "\nUNION ALL\nSELECT ");
@@ -1194,8 +1195,9 @@ static void getRbudiffQuery(
   strPrintf(pSql, " FROM main.%Q AS n WHERE NOT EXISTS (\n", zTab);
   strPrintf(pSql, "    SELECT 1 FROM ", zTab);
   strPrintf(pSql, " aux.%Q AS o WHERE ", zTab);
-  strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
-  strPrintf(pSql, "\n) ");
+  strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
+  strPrintf(pSql, "\n) AND ");
+  strPrintfArray(pSql, " AND ", "(n.%Q IS NOT NULL)", azCol, nPK);
 
   /* Updated rows. If all table columns are part of the primary key, there 
   ** can be no updates. In this case this part of the compound SELECT can
@@ -1226,7 +1228,7 @@ static void getRbudiffQuery(
     );
 
     strPrintf(pSql, "\nFROM main.%Q AS o, aux.%Q AS n\nWHERE ", zTab, zTab);
-    strPrintfArray(pSql, " AND ", "(n.%Q IS o.%Q)", azCol, nPK);
+    strPrintfArray(pSql, " AND ", "(n.%Q = o.%Q)", azCol, nPK);
     strPrintf(pSql, " AND ota_control LIKE '%%x%%'");
   }
 

From 033564cca9d64f67ce6cb0df1b1ddf53d4a74f49 Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 2 Sep 2016 17:18:20 +0000
Subject: [PATCH 12/14] Within a backup operation, ensure that a
 read-transaction is opened on the source database before its page size is
 read. This ensures the page-size used to write to the backup database is the
 same as its actual page-size, which is important for ZipVFS databases.

FossilOrigin-Name: 7908fc604991d81140c182b97981fd724ad126ae
---
 manifest      | 12 ++++++------
 manifest.uuid |  2 +-
 src/backup.c  | 25 ++++++++++++++++---------
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/manifest b/manifest
index ed5afd047d..f6d9f3788a 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Have\s"sqldiff\s--rbu"\signore\srows\swith\sNULL\svalues\sin\sprimary\skey\sfields.\sRBU\scan't\shandle\ssuch\srows\sand\sthe\sdocumentation\salready\ssays\ssqldiff\signores\sthem.\sBecause\sthe\scode\snow\suses\s"="\sinstead\sof\s"IS"\sto\sfilter\son\sprimary\skey\scolumns,\sdiffs\son\svirtual\stables\sare\sfaster\snow\stoo.
-D 2016-09-01T14:03:28.041
+C Within\sa\sbackup\soperation,\sensure\sthat\sa\sread-transaction\sis\sopened\son\sthe\ssource\sdatabase\sbefore\sits\spage\ssize\sis\sread.\sThis\sensures\sthe\spage-size\sused\sto\swrite\sto\sthe\sbackup\sdatabase\sis\sthe\ssame\sas\sits\sactual\spage-size,\swhich\sis\simportant\sfor\sZipVFS\sdatabases.
+D 2016-09-02T17:18:20.100
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -325,7 +325,7 @@ F src/alter.c 299117695b1f21ac62dfc5b608588810ba22ed0d
 F src/analyze.c 8b62b2cf4da85451534ac0af82cafc418d837f68
 F src/attach.c 4711ff365df4072b8c3dcd55db5d12dcf8ffa0c6
 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
-F src/backup.c 17cd25a36d49330df2bacd2cadf2a61f3b525976
+F src/backup.c 7d986927896b70c29a313b9954ab36576aff113d
 F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63
 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
 F src/btree.c 2551bd3ecb8b8988fb8b23aabadfb214dbc38e46
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 38d31e189e7c7899e14455f2c083aa676ce4d4c0
-R ca589e025240033a5748a5eba7e4950c
+P f4ba894a86aa195bcbe2fa69e91cd870ec3fb577
+R 6268e0964e35cc05a56dd4669fe54aba
 U dan
-Z 28ca6847fc75deee037b1b39928043e8
+Z 246b0cb12b0fcdaba66d59db5334a074
diff --git a/manifest.uuid b/manifest.uuid
index 78d594bd6c..a145bb5aba 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-f4ba894a86aa195bcbe2fa69e91cd870ec3fb577
\ No newline at end of file
+7908fc604991d81140c182b97981fd724ad126ae
\ No newline at end of file
diff --git a/src/backup.c b/src/backup.c
index 19c3b2a647..0a236700b2 100644
--- a/src/backup.c
+++ b/src/backup.c
@@ -196,7 +196,6 @@ sqlite3_backup *sqlite3_backup_init(
     p->isAttached = 0;
 
     if( 0==p->pSrc || 0==p->pDest 
-     || setDestPgsz(p)==SQLITE_NOMEM 
      || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK 
      ){
       /* One (or both) of the named databases did not exist or an OOM
@@ -384,14 +383,6 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
       rc = SQLITE_OK;
     }
 
-    /* Lock the destination database, if it is not locked already. */
-    if( SQLITE_OK==rc && p->bDestLocked==0
-     && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) 
-    ){
-      p->bDestLocked = 1;
-      sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema);
-    }
-
     /* If there is no open read-transaction on the source database, open
     ** one now. If a transaction is opened here, then it will be closed
     ** before this function exits.
@@ -401,6 +392,22 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
       bCloseTrans = 1;
     }
 
+    /* If the destination database has not yet been locked (i.e. if this
+    ** is the first call to backup_step() for the current backup operation),
+    ** try to set its page size to the same as the source database. This
+    ** is especially important on ZipVFS systems, as in that case it is
+    ** not possible to create a database file that uses one page size by
+    ** writing to it with another.  */
+    if( p->bDestLocked==0 ) setDestPgsz(p);
+
+    /* Lock the destination database, if it is not locked already. */
+    if( SQLITE_OK==rc && p->bDestLocked==0
+     && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) 
+    ){
+      p->bDestLocked = 1;
+      sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema);
+    }
+
     /* Do not allow backup if the destination database is in WAL mode
     ** and the page sizes are different between source and destination */
     pgszSrc = sqlite3BtreeGetPageSize(p->pSrc);

From 76729ed4be600448eac38af45aafacdff116500f Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 2 Sep 2016 21:17:51 +0000
Subject: [PATCH 13/14] Correctly detect an OOM occurring in the setDestPgsz()
 subroutine of backup.

FossilOrigin-Name: d9d8a048d4b621435870b4f8dd13b2938ac2f8fd
---
 manifest      | 14 +++++++-------
 manifest.uuid |  2 +-
 src/backup.c  |  4 +++-
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/manifest b/manifest
index f6d9f3788a..4b8ebb6ca8 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Within\sa\sbackup\soperation,\sensure\sthat\sa\sread-transaction\sis\sopened\son\sthe\ssource\sdatabase\sbefore\sits\spage\ssize\sis\sread.\sThis\sensures\sthe\spage-size\sused\sto\swrite\sto\sthe\sbackup\sdatabase\sis\sthe\ssame\sas\sits\sactual\spage-size,\swhich\sis\simportant\sfor\sZipVFS\sdatabases.
-D 2016-09-02T17:18:20.100
+C Correctly\sdetect\san\sOOM\soccurring\sin\sthe\ssetDestPgsz()\ssubroutine\sof\nbackup.
+D 2016-09-02T21:17:51.499
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -325,7 +325,7 @@ F src/alter.c 299117695b1f21ac62dfc5b608588810ba22ed0d
 F src/analyze.c 8b62b2cf4da85451534ac0af82cafc418d837f68
 F src/attach.c 4711ff365df4072b8c3dcd55db5d12dcf8ffa0c6
 F src/auth.c 930b376a9c56998557367e6f7f8aaeac82a2a792
-F src/backup.c 7d986927896b70c29a313b9954ab36576aff113d
+F src/backup.c 92c2e3b5fcb47626413717138617f4d32f08aea4
 F src/bitvec.c 3ee4c8b2c94ed3a7377256e18199e6ff5cf33f63
 F src/btmutex.c bc87dd3b062cc26edfe79918de2200ccb8d41e73
 F src/btree.c 2551bd3ecb8b8988fb8b23aabadfb214dbc38e46
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P f4ba894a86aa195bcbe2fa69e91cd870ec3fb577
-R 6268e0964e35cc05a56dd4669fe54aba
-U dan
-Z 246b0cb12b0fcdaba66d59db5334a074
+P 7908fc604991d81140c182b97981fd724ad126ae
+R 927ab162f03d57784d15e59a76583f84
+U drh
+Z 3a29e1d29c21b4d8169c8797b79e666a
diff --git a/manifest.uuid b/manifest.uuid
index a145bb5aba..a3366a05c1 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7908fc604991d81140c182b97981fd724ad126ae
\ No newline at end of file
+d9d8a048d4b621435870b4f8dd13b2938ac2f8fd
\ No newline at end of file
diff --git a/src/backup.c b/src/backup.c
index 0a236700b2..eb8f15cc7e 100644
--- a/src/backup.c
+++ b/src/backup.c
@@ -398,7 +398,9 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){
     ** is especially important on ZipVFS systems, as in that case it is
     ** not possible to create a database file that uses one page size by
     ** writing to it with another.  */
-    if( p->bDestLocked==0 ) setDestPgsz(p);
+    if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){
+      rc = SQLITE_NOMEM;
+    }
 
     /* Lock the destination database, if it is not locked already. */
     if( SQLITE_OK==rc && p->bDestLocked==0

From c6952addf7c2c22c7345d5ae8564ab762a1bdb1b Mon Sep 17 00:00:00 2001
From: dan <dan@noemail.net>
Date: Fri, 2 Sep 2016 21:34:17 +0000
Subject: [PATCH 14/14] Add a test case for the OOM handled by the previous
 commit.

FossilOrigin-Name: 9bdf7ca1b317fe0ba7efea38fb395bf6130ac89a
---
 manifest                | 14 +++++++-------
 manifest.uuid           |  2 +-
 test/backup_malloc.test | 31 +++++++++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/manifest b/manifest
index 4b8ebb6ca8..c643135037 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Correctly\sdetect\san\sOOM\soccurring\sin\sthe\ssetDestPgsz()\ssubroutine\sof\nbackup.
-D 2016-09-02T21:17:51.499
+C Add\sa\stest\scase\sfor\sthe\sOOM\shandled\sby\sthe\sprevious\scommit.
+D 2016-09-02T21:34:17.199
 F Makefile.in cfd8fb987cd7a6af046daa87daa146d5aad0e088
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 5017381e4853b1472e01d5bb926be1268eba429c
@@ -524,7 +524,7 @@ F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf
 F test/backup4.test 8f6fd48e0dfde77b9a3bb26dc471ede3e101df32
 F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4
 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135
-F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450
+F test/backup_malloc.test 833d1b90561a6dbab00079b9591bd4fc90b7c2e1
 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f
 F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f
 F test/bc_common.tcl b5e42d80305be95697e6370e015af571e5333a1c
@@ -1511,7 +1511,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 7908fc604991d81140c182b97981fd724ad126ae
-R 927ab162f03d57784d15e59a76583f84
-U drh
-Z 3a29e1d29c21b4d8169c8797b79e666a
+P d9d8a048d4b621435870b4f8dd13b2938ac2f8fd
+R b03bceee349e62c9e4b232e967e75375
+U dan
+Z 94198719997091b0b262fcfd62b5468a
diff --git a/manifest.uuid b/manifest.uuid
index a3366a05c1..33cb4d6b33 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-d9d8a048d4b621435870b4f8dd13b2938ac2f8fd
\ No newline at end of file
+9bdf7ca1b317fe0ba7efea38fb395bf6130ac89a
\ No newline at end of file
diff --git a/test/backup_malloc.test b/test/backup_malloc.test
index f556861fb1..45cac44f18 100644
--- a/test/backup_malloc.test
+++ b/test/backup_malloc.test
@@ -84,4 +84,35 @@ do_malloc_test backup_malloc-2 -tclprep {
   db2 close
 }
 
+reset_db
+do_execsql_test 3.0 {
+  PRAGMA page_size = 16384;
+  BEGIN;
+  CREATE TABLE t1(a, b);
+  INSERT INTO t1 VALUES(1, 2);
+  COMMIT;
+}
+
+do_faultsim_test 3 -faults oom* -prep {
+  catch { db close }
+
+  forcedelete test2.db
+  sqlite3 db2 test2.db
+  sqlite3 db test.db
+  sqlite3_backup B db2 main db main
+} -body {
+
+  set rc [B step 50]
+  if {$rc == "SQLITE_NOMEM" || $rc == "SQLITE_IOERR_NOMEM"} {
+    error "out of memory"
+  }
+
+} -test {
+  faultsim_test_result {0 {}} 
+  faultsim_integrity_check
+  
+  # Finalize the backup.
+  catch { B finish }
+}
+
 finish_test