Support "ORDER BY rowid ASC".

FossilOrigin-Name: b96b5e166990e4ec363b24f66e04cfa5f00f6342
This commit is contained in:
dan 2014-07-10 20:21:12 +00:00
parent a2b569f955
commit 48eecfb8b9
8 changed files with 457 additions and 151 deletions

View File

@ -414,21 +414,25 @@ static int fts5ExprNearAdvanceAll(
*/
static int fts5ExprAdvanceto(
Fts5IndexIter *pIter, /* Iterator to advance */
i64 *piMin, /* IN/OUT: Minimum rowid seen so far */
int bAsc, /* True if iterator is "rowid ASC" */
i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
int *pRc, /* OUT: Error code */
int *pbEof /* OUT: Set to true if EOF */
){
i64 iMin = *piMin;
i64 iLast = *piLast;
i64 iRowid;
while( (iRowid = sqlite3Fts5IterRowid(pIter))>iMin ){
while( 1 ){
iRowid = sqlite3Fts5IterRowid(pIter);
if( (bAsc==0 && iRowid<=iLast) || (bAsc==1 && iRowid>=iLast) ) break;
sqlite3Fts5IterNext(pIter, 0);
if( sqlite3Fts5IterEof(pIter) ){
*pbEof = 1;
return 1;
}
}
if( iRowid<iMin ){
*piMin = iRowid;
if( iRowid!=iLast ){
assert( (bAsc==0 && iRowid<iLast) || (bAsc==1 && iRowid>iLast) );
*piLast = iRowid;
}
return 0;
@ -452,10 +456,15 @@ static int fts5ExprNearNextRowidMatch(
Fts5ExprNearset *pNear = pNode->pNear;
int rc = SQLITE_OK;
int i, j; /* Phrase and token index, respectively */
i64 iMin; /* Smallest rowid any iterator points to */
int bMatch;
i64 iLast; /* Lastest rowid any iterator points to */
int bMatch; /* True if all terms are at the same rowid */
/* Set iLast, the lastest rowid any iterator points to. If the iterator
** skips through rowids in the default descending order, this means the
** minimum rowid. Or, if the iterator is "ORDER BY rowid ASC", then it
** means the maximum rowid. */
iLast = sqlite3Fts5IterRowid(pNear->apPhrase[0]->aTerm[0].pIter);
iMin = sqlite3Fts5IterRowid(pNear->apPhrase[0]->aTerm[0].pIter);
do {
bMatch = 1;
for(i=0; i<pNear->nPhrase; i++){
@ -463,13 +472,15 @@ static int fts5ExprNearNextRowidMatch(
for(j=0; j<pPhrase->nTerm; j++){
Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
i64 iRowid = sqlite3Fts5IterRowid(pIter);
if( iRowid!=iMin ) bMatch = 0;
if( fts5ExprAdvanceto(pIter, &iMin, &rc, &pNode->bEof) ) return rc;
if( iRowid!=iLast ) bMatch = 0;
if( fts5ExprAdvanceto(pIter, pExpr->bAsc, &iLast, &rc, &pNode->bEof) ){
return rc;
}
}
}
}while( bMatch==0 );
pNode->iRowid = iMin;
pNode->iRowid = iLast;
return rc;
}
@ -555,7 +566,7 @@ static int fts5ExprNearInitAll(
(pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
(pExpr->bAsc ? FTS5INDEX_QUERY_ASC : 0)
);
if( sqlite3Fts5IterEof(pTerm->pIter) ){
if( pTerm->pIter && sqlite3Fts5IterEof(pTerm->pIter) ){
pNode->bEof = 1;
return SQLITE_OK;
}
@ -569,16 +580,31 @@ static int fts5ExprNearInitAll(
static int fts5ExprNodeNextMatch(Fts5Expr*, Fts5ExprNode*);
/*
** Nodes at EOF are considered larger than all other nodes. A node that
** points to a *smaller* rowid is considered larger.
**
** Compare the values currently indicated by the two nodes as follows:
**
** res = (*p1) - (*p2)
**
** Nodes that point to values that come later in the iteration order are
** considered to be larger. Nodes at EOF are the largest of all.
**
** This means that if the iteration order is ASC, then numerically larger
** rowids are considered larger. Or if it is the default DESC, numerically
** smaller rowids are larger.
*/
static int fts5NodeCompare(Fts5ExprNode *p1, Fts5ExprNode *p2){
static int fts5NodeCompare(
Fts5Expr *pExpr,
Fts5ExprNode *p1,
Fts5ExprNode *p2
){
if( p2->bEof ) return -1;
if( p1->bEof ) return +1;
if( p1->iRowid>p2->iRowid ) return -1;
return (p1->iRowid < p2->iRowid);
if( pExpr->bAsc ){
if( p1->iRowid<p2->iRowid ) return -1;
return (p1->iRowid > p2->iRowid);
}else{
if( p1->iRowid>p2->iRowid ) return -1;
return (p1->iRowid < p2->iRowid);
}
}
static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){
@ -600,7 +626,7 @@ static int fts5ExprNodeNext(Fts5Expr *pExpr, Fts5ExprNode *pNode){
case FTS5_OR: {
Fts5ExprNode *p1 = pNode->pLeft;
Fts5ExprNode *p2 = pNode->pRight;
int cmp = fts5NodeCompare(p1, p2);
int cmp = fts5NodeCompare(pExpr, p1, p2);
if( cmp==0 ){
rc = fts5ExprNodeNext(pExpr, p1);
@ -644,7 +670,12 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){
Fts5ExprNode *p2 = pNode->pRight;
while( p1->bEof==0 && p2->bEof==0 && p2->iRowid!=p1->iRowid ){
Fts5ExprNode *pAdv = (p1->iRowid > p2->iRowid) ? p1 : p2;
Fts5ExprNode *pAdv;
if( pExpr->bAsc ){
pAdv = (p1->iRowid < p2->iRowid) ? p1 : p2;
}else{
pAdv = (p1->iRowid > p2->iRowid) ? p1 : p2;
}
rc = fts5ExprNodeNext(pExpr, pAdv);
if( rc!=SQLITE_OK ) break;
}
@ -656,7 +687,7 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){
case FTS5_OR: {
Fts5ExprNode *p1 = pNode->pLeft;
Fts5ExprNode *p2 = pNode->pRight;
Fts5ExprNode *pNext = (fts5NodeCompare(p1, p2) > 0 ? p2 : p1);
Fts5ExprNode *pNext = (fts5NodeCompare(pExpr, p1, p2) > 0 ? p2 : p1);
pNode->bEof = pNext->bEof;
pNode->iRowid = pNext->iRowid;
break;
@ -667,7 +698,7 @@ static int fts5ExprNodeNextMatch(Fts5Expr *pExpr, Fts5ExprNode *pNode){
Fts5ExprNode *p2 = pNode->pRight;
while( rc==SQLITE_OK ){
int cmp;
while( rc==SQLITE_OK && (cmp = fts5NodeCompare(p1, p2))>0 ){
while( rc==SQLITE_OK && (cmp = fts5NodeCompare(pExpr, p1, p2))>0 ){
rc = fts5ExprNodeNext(pExpr, p2);
}
if( rc || cmp ) break;

View File

@ -299,6 +299,7 @@ struct Fts5Index {
};
struct Fts5DoclistIter {
int bAsc;
u8 *a;
int n;
int i;
@ -410,6 +411,7 @@ struct Fts5SegWriter {
*/
struct Fts5MultiSegIter {
int nSeg; /* Size of aSeg[] array */
int bRev; /* True to iterate in reverse order */
Fts5SegIter *aSeg; /* Array of segment iterators */
u16 *aFirst; /* Current merge state (see above) */
};
@ -443,6 +445,18 @@ struct Fts5MultiSegIter {
** FTS5_SEGITER_ONETERM:
** If set, set the iterator to point to EOF after the current doclist
** has been exhausted. Do not proceed to the next term in the segment.
**
** FTS5_SEGITER_REVERSE:
** This flag is only ever set if FTS5_SEGITER_ONETERM is also set. If
** it is set, iterate through docids in ascending order instead of the
** default descending order.
**
** iRowidOffset/nRowidOffset/aRowidOffset:
** These are used if the FTS5_SEGITER_REVERSE flag is set.
**
** Each time a new page is loaded, the iterator is set to point to the
** final rowid. Additionally, the aRowidOffset[] array is populated
** with the byte offsets of all relevant rowid fields on the page.
*/
struct Fts5SegIter {
Fts5StructureSegment *pSeg; /* Segment to iterate through */
@ -452,15 +466,23 @@ struct Fts5SegIter {
Fts5Data *pLeaf; /* Current leaf data */
int iLeafOffset; /* Byte offset within current leaf */
/* The page and offset from which the current term was read. The offset
** is the offset of the first rowid in the current doclist. */
int iTermLeafPgno;
int iTermLeafOffset;
/* The following are only used if the FTS5_SEGITER_REVERSE flag is set. */
int iRowidOffset; /* Current entry in aRowidOffset[] */
int nRowidOffset; /* Allocated size of aRowidOffset[] array */
int *aRowidOffset; /* Array of offset to rowid fields */
/* Variables populated based on current entry. */
Fts5Buffer term; /* Current term */
i64 iRowid; /* Current rowid */
};
#define FTS5_SEGITER_ONETERM 0x01
#define FTS5_SEGITER_REVERSE 0x02
/*
@ -1076,6 +1098,55 @@ static void fts5SegIterInit(
}
}
static void fts5LeafHeader(Fts5Data *pLeaf, int *piRowid, int *piTerm){
*piRowid = (int)fts5GetU16(&pLeaf->p[0]);
*piTerm = (int)fts5GetU16(&pLeaf->p[2]);
}
/*
** This function is only ever called on iterators created by calls to
** Fts5IndexQuery() with the FTS5INDEX_QUERY_ASC flag set.
**
** When this function is called, iterator pIter points to the first rowid
** on the current leaf associated with the term being queried. This function
** advances it to point to the last such rowid and, if necessary, initializes
** the aRowidOffset[] and iRowidOffset variables.
*/
static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){
int n = pIter->pLeaf->n;
int i = pIter->iLeafOffset;
u8 *a = pIter->pLeaf->p;
int iRowidOffset = 0;
while( p->rc==SQLITE_OK && i<n ){
i64 iDelta = 0;
int nPos;
i += getVarint32(&a[i], nPos);
i += nPos;
if( i>=n ) break;
i += getVarint(&a[i], (u64*)&iDelta);
if( iDelta==0 ) break;
pIter->iRowid -= iDelta;
if( iRowidOffset>=pIter->nRowidOffset ){
int nNew = pIter->nRowidOffset + 8;
int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int));
if( aNew==0 ){
p->rc = SQLITE_NOMEM;
break;
}
pIter->aRowidOffset = aNew;
pIter->nRowidOffset = nNew;
}
pIter->aRowidOffset[iRowidOffset++] = pIter->iLeafOffset;
pIter->iLeafOffset = i;
}
pIter->iRowidOffset = iRowidOffset;
}
/*
** Advance iterator pIter to the next entry.
**
@ -1088,68 +1159,198 @@ static void fts5SegIterNext(
Fts5SegIter *pIter /* Iterator to advance */
){
if( p->rc==SQLITE_OK ){
Fts5Data *pLeaf = pIter->pLeaf;
int iOff;
int bNewTerm = 0;
int nKeep = 0;
if( pIter->flags & FTS5_SEGITER_REVERSE ){
if( pIter->iRowidOffset>0 ){
u8 *a = pIter->pLeaf->p;
int iOff;
int nPos;
i64 iDelta;
pIter->iRowidOffset--;
/* Search for the end of the position list within the current page. */
u8 *a = pLeaf->p;
int n = pLeaf->n;
iOff = pIter->iLeafOffset;
if( iOff<n ){
int nPoslist;
iOff += getVarint32(&a[iOff], nPoslist);
iOff += nPoslist;
}
if( iOff<n ){
/* The next entry is on the current page */
u64 iDelta;
iOff += sqlite3GetVarint(&a[iOff], &iDelta);
pIter->iLeafOffset = iOff;
if( iDelta==0 ){
bNewTerm = 1;
if( iOff>=n ){
fts5SegIterNextPage(p, pIter);
pIter->iLeafOffset = 4;
}else if( iOff!=fts5GetU16(&a[2]) ){
pIter->iLeafOffset += getVarint32(&a[iOff], nKeep);
}
pIter->iLeafOffset = iOff = pIter->aRowidOffset[pIter->iRowidOffset];
iOff += getVarint32(&a[iOff], nPos);
iOff += nPos;
getVarint(&a[iOff], (u64*)&iDelta);
pIter->iRowid += iDelta;
}else{
pIter->iRowid -= iDelta;
}
}else{
iOff = 0;
/* Next entry is not on the current page */
while( iOff==0 ){
fts5SegIterNextPage(p, pIter);
pLeaf = pIter->pLeaf;
if( pLeaf==0 ) break;
if( (iOff = fts5GetU16(&pLeaf->p[0])) ){
iOff += sqlite3GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){
pIter->iLeafOffset = iOff;
bNewTerm = 1;
}
}
}
/* Check if the iterator is now at EOF. If so, return early. */
if( pIter->pLeaf && bNewTerm ){
if( pIter->flags & FTS5_SEGITER_ONETERM ){
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = 0;
while( p->rc==SQLITE_OK && pIter->iLeafPgno>pIter->iTermLeafPgno ){
Fts5Data *pNew;
pIter->iLeafPgno--;
pNew = fts5DataRead(p, FTS5_SEGMENT_ROWID(
pIter->iIdx, pIter->pSeg->iSegid, 0, pIter->iLeafPgno
));
if( pNew ){
if( pIter->iLeafPgno==pIter->iTermLeafPgno ){
if( pIter->iTermLeafOffset<pNew->n ){
pIter->pLeaf = pNew;
pIter->iLeafOffset = pIter->iTermLeafOffset;
}
}else{
int iRowidOff, dummy;
fts5LeafHeader(pNew, &iRowidOff, &dummy);
if( iRowidOff ){
pIter->pLeaf = pNew;
pIter->iLeafOffset = iRowidOff;
}
}
if( pIter->pLeaf ){
u8 *a = &pIter->pLeaf->p[pIter->iLeafOffset];
pIter->iLeafOffset += getVarint(a, (u64*)&pIter->iRowid);
break;
}else{
fts5DataRelease(pNew);
}
}
}
if( pIter->pLeaf ){
fts5SegIterReverseInitPage(p, pIter);
}
}
}else{
Fts5Data *pLeaf = pIter->pLeaf;
int iOff;
int bNewTerm = 0;
int nKeep = 0;
/* Search for the end of the position list within the current page. */
u8 *a = pLeaf->p;
int n = pLeaf->n;
iOff = pIter->iLeafOffset;
if( iOff<n ){
int nPoslist;
iOff += getVarint32(&a[iOff], nPoslist);
iOff += nPoslist;
}
if( iOff<n ){
/* The next entry is on the current page */
u64 iDelta;
iOff += sqlite3GetVarint(&a[iOff], &iDelta);
pIter->iLeafOffset = iOff;
if( iDelta==0 ){
bNewTerm = 1;
if( iOff>=n ){
fts5SegIterNextPage(p, pIter);
pIter->iLeafOffset = 4;
}else if( iOff!=fts5GetU16(&a[2]) ){
pIter->iLeafOffset += getVarint32(&a[iOff], nKeep);
}
}else{
pIter->iRowid -= iDelta;
}
}else{
fts5SegIterLoadTerm(p, pIter, nKeep);
iOff = 0;
/* Next entry is not on the current page */
while( iOff==0 ){
fts5SegIterNextPage(p, pIter);
pLeaf = pIter->pLeaf;
if( pLeaf==0 ) break;
if( (iOff = fts5GetU16(&pLeaf->p[0])) ){
iOff += sqlite3GetVarint(&pLeaf->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
else if( (iOff = fts5GetU16(&pLeaf->p[2])) ){
pIter->iLeafOffset = iOff;
bNewTerm = 1;
}
}
}
/* Check if the iterator is now at EOF. If so, return early. */
if( pIter->pLeaf && bNewTerm ){
if( pIter->flags & FTS5_SEGITER_ONETERM ){
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = 0;
}else{
fts5SegIterLoadTerm(p, pIter, nKeep);
}
}
}
}
}
/*
** Iterator pIter currently points to the first rowid in a doclist. This
** function sets the iterator up so that iterates in reverse order through
** the doclist.
*/
static void fts5SegIterReverse(Fts5Index *p, int iIdx, Fts5SegIter *pIter){
Fts5Data *pLeaf; /* Current leaf data */
int iOff = pIter->iLeafOffset; /* Byte offset within current leaf */
Fts5Data *pLast = 0;
int pgnoLast = 0;
/* Move to the page that contains the last rowid in this doclist. */
pLeaf = pIter->pLeaf;
while( iOff<pLeaf->n ){
int nPos;
i64 iDelta;
/* Position list size in bytes */
iOff += getVarint32(&pLeaf->p[iOff], nPos);
iOff += nPos;
if( iOff>=pLeaf->n ) break;
/* Rowid delta. Or, if 0x00, the end of doclist marker. */
nPos = getVarint(&pLeaf->p[iOff], (u64*)&iDelta);
if( iDelta==0 ) break;
iOff += nPos;
}
if( iOff>=pLeaf->n ){
Fts5StructureSegment *pSeg = pIter->pSeg;
i64 iAbs = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pIter->iLeafPgno);
i64 iLast = FTS5_SEGMENT_ROWID(iIdx, pSeg->iSegid, 0, pSeg->pgnoLast);
/* The last rowid in the doclist may not be on the current page. Search
** forward to find the page containing the last rowid. */
for(iAbs++; p->rc==SQLITE_OK && iAbs<=iLast; iAbs++){
Fts5Data *pNew = fts5DataRead(p, iAbs);
if( pNew ){
int iRowid, iTerm;
fts5LeafHeader(pNew, &iRowid, &iTerm);
if( iRowid ){
Fts5Data *pTmp = pLast;
pLast = pNew;
pNew = pTmp;
pgnoLast = iAbs & (((i64)1 << FTS5_DATA_PAGE_B) - 1);
}
if( iTerm ){
iAbs = iLast;
}
fts5DataRelease(pNew);
}
}
}
/* If pLast is NULL at this point, then the last rowid for this doclist
** lies on the page currently indicated by the iterator. In this case
** iLastOff is set to the value that pIter->iLeafOffset will take when
** the iterator points to that rowid.
**
** Or, if pLast is non-NULL, then it is the page that contains the last
** rowid.
*/
if( pLast ){
int dummy;
fts5DataRelease(pIter->pLeaf);
pIter->pLeaf = pLast;
pIter->iLeafPgno = pgnoLast;
fts5LeafHeader(pLast, &iOff, &dummy);
iOff += getVarint(&pLast->p[iOff], (u64*)&pIter->iRowid);
pIter->iLeafOffset = iOff;
}
fts5SegIterReverseInitPage(p, pIter);
pIter->flags |= FTS5_SEGITER_REVERSE;
}
/*
** Initialize the object pIter to point to term pTerm/nTerm within segment
** pSeg, index iIdx. If there is no such term in the index, the iterator
@ -1162,13 +1363,15 @@ static void fts5SegIterSeekInit(
Fts5Index *p, /* FTS5 backend */
int iIdx, /* Config.aHash[] index of FTS index */
const u8 *pTerm, int nTerm, /* Term to seek to */
int bGe, /* If true seek for >=. If false, == */
int flags, /* Mask of FTS5INDEX_XXX flags */
Fts5StructureSegment *pSeg, /* Description of segment */
Fts5SegIter *pIter /* Object to populate */
){
int iPg = 1;
int h;
int bGe = ((flags & FTS5INDEX_QUERY_PREFIX) && iIdx==0);
assert( bGe==0 || (flags & FTS5INDEX_QUERY_ASC)==0 );
assert( pTerm && nTerm );
memset(pIter, 0, sizeof(*pIter));
pIter->pSeg = pSeg;
@ -1220,7 +1423,12 @@ static void fts5SegIterSeekInit(
}
}
if( bGe==0 ) pIter->flags |= FTS5_SEGITER_ONETERM;
if( bGe==0 ){
pIter->flags |= FTS5_SEGITER_ONETERM;
if( pIter->pLeaf && (flags & FTS5INDEX_QUERY_ASC) ){
fts5SegIterReverse(p, iIdx, pIter);
}
}
}
/*
@ -1229,6 +1437,7 @@ static void fts5SegIterSeekInit(
static void fts5SegIterClear(Fts5SegIter *pIter){
fts5BufferFree(&pIter->term);
fts5DataRelease(pIter->pLeaf);
sqlite3_free(pIter->aRowidOffset);
memset(pIter, 0, sizeof(Fts5SegIter));
}
@ -1248,6 +1457,7 @@ static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){
Fts5SegIter *p2; /* Right-hand Fts5SegIter */
assert( iOut<pIter->nSeg && iOut>0 );
assert( pIter->bRev==0 || pIter->bRev==1 );
if( iOut>=(pIter->nSeg/2) ){
i1 = (iOut - pIter->nSeg/2) * 2;
@ -1269,7 +1479,7 @@ static int fts5MultiIterDoCompare(Fts5MultiSegIter *pIter, int iOut){
assert( i2>i1 );
assert( i2!=0 );
if( p1->iRowid==p2->iRowid ) return i2;
res = (p1->iRowid > p2->iRowid) ? -1 : +1;
res = ((p1->iRowid < p2->iRowid)==pIter->bRev) ? -1 : +1;
}
assert( res!=0 );
if( res<0 ){
@ -1342,7 +1552,7 @@ static void fts5MultiIterNew(
Fts5Index *p, /* FTS5 backend to iterate within */
Fts5Structure *pStruct, /* Structure of specific index */
int iIdx, /* Config.aHash[] index of FTS index */
int bGe, /* True for >= */
int flags, /* True for >= */
const u8 *pTerm, int nTerm, /* Term to seek to (or NULL/0) */
int iLevel, /* Level to iterate (-1 for all) */
int nSegment, /* Number of segments to merge (iLevel>=0) */
@ -1373,6 +1583,7 @@ static void fts5MultiIterNew(
pNew->nSeg = nSlot;
pNew->aSeg = (Fts5SegIter*)&pNew[1];
pNew->aFirst = (u16*)&pNew->aSeg[nSlot];
pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_ASC));
/* Initialize each of the component segment iterators. */
if( iLevel<0 ){
@ -1384,7 +1595,7 @@ static void fts5MultiIterNew(
if( pTerm==0 ){
fts5SegIterInit(p, iIdx, pSeg, pIter);
}else{
fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, bGe, pSeg, pIter);
fts5SegIterSeekInit(p, iIdx, pTerm, nTerm, flags, pSeg, pIter);
}
}
}
@ -3074,7 +3285,11 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
if( pIter->i ){
i64 iDelta;
pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&iDelta);
pIter->iRowid -= iDelta;
if( pIter->bAsc ){
pIter->iRowid += iDelta;
}else{
pIter->iRowid -= iDelta;
}
}else{
pIter->i += getVarint(&pIter->a[pIter->i], (u64*)&pIter->iRowid);
}
@ -3086,10 +3301,15 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){
}
}
static void fts5DoclistIterInit(Fts5Buffer *pBuf, Fts5DoclistIter *pIter){
static void fts5DoclistIterInit(
Fts5Buffer *pBuf,
int bAsc,
Fts5DoclistIter *pIter
){
memset(pIter, 0, sizeof(*pIter));
pIter->a = pBuf->p;
pIter->n = pBuf->n;
pIter->bAsc = bAsc;
fts5DoclistIterNext(pIter);
}
@ -3098,14 +3318,17 @@ static void fts5DoclistIterInit(Fts5Buffer *pBuf, Fts5DoclistIter *pIter){
*/
static void fts5MergeAppendDocid(
int *pRc, /* IN/OUT: Error code */
int bAsc,
Fts5Buffer *pBuf, /* Buffer to write to */
i64 *piLastRowid, /* IN/OUT: Previous rowid written (if any) */
i64 iRowid /* Rowid to append */
){
if( pBuf->n==0 ){
fts5BufferAppendVarint(pRc, pBuf, iRowid);
}else{
}else if( bAsc==0 ){
fts5BufferAppendVarint(pRc, pBuf, *piLastRowid - iRowid);
}else{
fts5BufferAppendVarint(pRc, pBuf, iRowid - *piLastRowid);
}
*piLastRowid = iRowid;
}
@ -3120,6 +3343,7 @@ static void fts5MergeAppendDocid(
*/
static void fts5MergePrefixLists(
Fts5Index *p, /* FTS5 backend object */
int bAsc,
Fts5Buffer *p1, /* First list to merge */
Fts5Buffer *p2 /* Second list to merge */
){
@ -3132,19 +3356,21 @@ static void fts5MergePrefixLists(
memset(&out, 0, sizeof(out));
memset(&tmp, 0, sizeof(tmp));
fts5DoclistIterInit(p1, &i1);
fts5DoclistIterInit(p2, &i2);
fts5DoclistIterInit(p1, bAsc, &i1);
fts5DoclistIterInit(p2, bAsc, &i2);
while( i1.aPoslist!=0 || i2.aPoslist!=0 ){
if( i2.aPoslist==0 || (i1.aPoslist && i1.iRowid>i2.iRowid) ){
if( i2.aPoslist==0 || (i1.aPoslist &&
( (!bAsc && i1.iRowid>i2.iRowid) || (bAsc && i1.iRowid<i2.iRowid) )
)){
/* Copy entry from i1 */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i1.iRowid);
fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i1.iRowid);
fts5BufferAppendVarint(&p->rc, &out, i1.nPoslist);
fts5BufferAppendBlob(&p->rc, &out, i1.nPoslist, i1.aPoslist);
fts5DoclistIterNext(&i1);
}
else if( i1.aPoslist==0 || i2.iRowid>i1.iRowid ){
else if( i1.aPoslist==0 || i2.iRowid!=i1.iRowid ){
/* Copy entry from i2 */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
fts5BufferAppendVarint(&p->rc, &out, i2.nPoslist);
fts5BufferAppendBlob(&p->rc, &out, i2.nPoslist, i2.aPoslist);
fts5DoclistIterNext(&i2);
@ -3157,7 +3383,7 @@ static void fts5MergePrefixLists(
memset(&writer, 0, sizeof(writer));
/* Merge the two position lists. */
fts5MergeAppendDocid(&p->rc, &out, &iLastRowid, i2.iRowid);
fts5MergeAppendDocid(&p->rc, bAsc, &out, &iLastRowid, i2.iRowid);
fts5BufferZero(&tmp);
sqlite3Fts5PoslistReaderInit(-1, i1.aPoslist, i1.nPoslist, &r1);
sqlite3Fts5PoslistReaderInit(-1, i2.aPoslist, i2.nPoslist, &r2);
@ -3195,6 +3421,7 @@ static void fts5BufferSwap(Fts5Buffer *p1, Fts5Buffer *p2){
static void fts5SetupPrefixIter(
Fts5Index *p, /* Index to read from */
int bAsc, /* True for "ORDER BY rowid ASC" */
const u8 *pToken, /* Buffer containing prefix to match */
int nToken, /* Size of buffer pToken in bytes */
Fts5IndexIter *pIter /* Populate this object */
@ -3203,7 +3430,6 @@ static void fts5SetupPrefixIter(
Fts5Buffer *aBuf;
const int nBuf = 32;
aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf);
pStruct = fts5StructureRead(p, 0);
@ -3225,29 +3451,34 @@ static void fts5SetupPrefixIter(
assert( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 );
if( nTerm<nToken || memcmp(pToken, pTerm, nToken) ) break;
if( doclist.n>0 && iRowid>=iLastRowid ){
if( doclist.n>0
&& ((!bAsc && iRowid>=iLastRowid) || (bAsc && iRowid<=iLastRowid))
){
for(i=0; doclist.n && p->rc==SQLITE_OK; i++){
assert( i<nBuf );
if( aBuf[i].n==0 ){
fts5BufferSwap(&doclist, &aBuf[i]);
fts5BufferZero(&doclist);
}else{
fts5MergePrefixLists(p, &doclist, &aBuf[i]);
fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
fts5BufferZero(&aBuf[i]);
}
}
}
if( doclist.n==0 ){
fts5BufferAppendVarint(&p->rc, &doclist, iRowid);
}else{
}else if( bAsc==0 ){
fts5BufferAppendVarint(&p->rc, &doclist, iLastRowid - iRowid);
}else{
fts5BufferAppendVarint(&p->rc, &doclist, iRowid - iLastRowid);
}
iLastRowid = iRowid;
fts5MultiIterPoslist(p, p1, 1, &doclist);
}
for(i=0; i<nBuf; i++){
fts5MergePrefixLists(p, &doclist, &aBuf[i]);
fts5MergePrefixLists(p, bAsc, &doclist, &aBuf[i]);
fts5BufferFree(&aBuf[i]);
}
fts5MultiIterFree(p, p1);
@ -3257,7 +3488,7 @@ static void fts5SetupPrefixIter(
fts5BufferFree(&doclist);
}else{
pIter->pDoclist = pDoclist;
fts5DoclistIterInit(&doclist, pIter->pDoclist);
fts5DoclistIterInit(&doclist, bAsc, pIter->pDoclist);
}
}
@ -3296,11 +3527,12 @@ Fts5IndexIter *sqlite3Fts5IndexQuery(
pRet->pStruct = fts5StructureRead(p, iIdx);
if( pRet->pStruct ){
fts5MultiIterNew(p, pRet->pStruct,
iIdx, 0, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
iIdx, flags, (const u8*)pToken, nToken, -1, 0, &pRet->pMulti
);
}
}else{
fts5SetupPrefixIter(p, (const u8*)pToken, nToken, pRet);
int bAsc = (flags & FTS5INDEX_QUERY_ASC)!=0;
fts5SetupPrefixIter(p, bAsc, (const u8*)pToken, nToken, pRet);
}
}
@ -3362,6 +3594,7 @@ const u8 *sqlite3Fts5IterPoslist(Fts5IndexIter *pIter, int *pn){
Fts5Index *p = pIter->pIndex;
fts5BufferZero(&pIter->poslist);
fts5MultiIterPoslist(p, pIter->pMulti, 0, &pIter->poslist);
assert( p->rc==SQLITE_OK );
if( p->rc ) return 0;
*pn = pIter->poslist.n;
return pIter->poslist.p;

View File

@ -1,5 +1,5 @@
C Add\ssupport\sfor\sprefix\squeries\sto\sfts5.
D 2014-07-08T16:27:37.120
C Support\s"ORDER\sBY\srowid\sASC".
D 2014-07-10T20:21:12.482
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -107,8 +107,8 @@ F ext/fts5/fts5.c 1af3184dd9c0e5c1686f71202d6b6cac8f225f05
F ext/fts5/fts5Int.h bb716a6e6a376a7c8211e55e5577c6c020d176c2
F ext/fts5/fts5_buffer.c 83b463a179ad4348fa87796fce78b0e4ef6b898a
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
F ext/fts5/fts5_expr.c 21351cdd256f8e561a57a38490d27f7922247696
F ext/fts5/fts5_index.c a3084168a384a9d43f7fb045511b386ccb6e55e8
F ext/fts5/fts5_expr.c 0dc31b06d444cad097bec05699797590729d2638
F ext/fts5/fts5_index.c 9ff3008e903aa9077b0a7a7aa76ab6080eb07a36
F ext/fts5/fts5_storage.c 7848d8f8528d798bba159900ea310a6d4a279da8
F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb
@ -593,9 +593,9 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b
F test/fts4noti.test 524807f0c36d49deea7920cdd4cd687408b58849
F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36
F test/fts5aa.test c8d3b9694f6b2864161c7437408464a535d19343
F test/fts5ab.test 4db86a9473ee2a8c2cb30e0d81df21c6022f99b6
F test/fts5ac.test d3aeb7a079d40093b34ac8053fc5e4c0ed7e88dc
F test/fts5ad.test a4d2f344c86a45ee53b424512585b3900ccb8cf3
F test/fts5ab.test dc04ed48cf93ca957d174406e6c192f2ff4f3397
F test/fts5ac.test 28203ba2334030514d7a6271c5fb1ba3cbc219b1
F test/fts5ad.test 2ed38bbc865678cb2905247120d02ebba7f20e07
F test/fts5ea.test ff43b40f8879ba50b82def70f2ab67c195d1a1d4
F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef
@ -763,7 +763,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0
F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
F test/permutations.test 43a4c2397b5e8a45c41fac20c7a8a2d4094f470f
F test/permutations.test 0b5333e5dcdeffba0ecbe5ee8dc7577029ffab6c
F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0
F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13
F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
@ -1191,7 +1191,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
P 8682b87e794767cefcaa080fd53c8973c24c556a
R dba83c3d230dbf413439715289f715cf
P 75ebd3cd5904a4f89f7f3a9b25d32b2a42a31310
R 5a76d2f2fc0d7fcaa9a60fabc7fdb146
U dan
Z 876bab147b2ac69a43a21b2ef49df211
Z 870004bd588f44c77d8063239acbea69

View File

@ -1 +1 @@
75ebd3cd5904a4f89f7f3a9b25d32b2a42a31310
b96b5e166990e4ec363b24f66e04cfa5f00f6342

View File

@ -124,7 +124,6 @@ foreach {tn expr res} {
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {8 5}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
@ -133,10 +132,23 @@ foreach {tn expr res} {
} $res
}
breakpoint
do_execsql_test 3.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'NEAR(aback abate, 2)'
} {6}
foreach {tn expr res} {
1 {abash} {1 3 5 9}
2 {abase} {1 3 4 9}
3 {abase + abash} {1}
4 {abash + abase} {9}
5 {abaft + abashing} {5 8}
6 {abandon + abandoning} {10}
7 {"abashing abases abasement abaft abashing"} {8}
} {
do_execsql_test 3.4.$tn {
SELECT rowid FROM t1 WHERE t1 MATCH $expr ORDER BY rowid ASC
} $res
}
finish_test

View File

@ -208,18 +208,24 @@ proc nearset {aCol args} {
return $bMatch
}
proc matchdata {expr {print 0}} {
proc matchdata {expr {bAsc 0}} {
set tclexpr [db one {SELECT fts5_expr_tcl($expr, 'nearset $cols', 'x', 'y')}]
set res [list]
foreach {id x y} $::data {
set cols [list $x $y]
if $tclexpr {
set res [concat $id $res]
lappend res $id
}
}
if {$print} {
puts $tclexpr
# puts $tclexpr
if {$bAsc} {
set res [lsort -integer -increasing $res]
} else {
set res [lsort -integer -decreasing $res]
}
return $res
}
@ -263,32 +269,37 @@ foreach {tn expr tclexpr} {
#-------------------------------------------------------------------------
#
foreach {tn expr} {
1 { NEAR(r c) }
2 { NEAR(r c, 5) }
3 { NEAR(r c, 3) }
4 { NEAR(r c, 2) }
5 { NEAR(r c, 0) }
6 { NEAR(a b c) }
7 { NEAR(a b c, 8) }
8 { x : NEAR(r c) }
9 { y : NEAR(r c) }
10 { x : "r c" }
11 { y : "r c" }
12 { a AND b }
13 { a AND b AND c }
14a { a }
14b { a OR b }
15 { a OR b AND c }
16 { c AND b OR a }
17 { c AND (b OR a) }
18 { c NOT (b OR a) }
19 { c NOT b OR a AND d }
foreach {bAsc sql} {
0 {SELECT rowid FROM xx WHERE xx MATCH $expr}
1 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid ASC}
} {
set res [matchdata $expr]
do_execsql_test 4.$tn.[llength $res] {
SELECT rowid FROM xx WHERE xx match $expr
} $res
foreach {tn expr} {
0.1 x
1 { NEAR(r c) }
2 { NEAR(r c, 5) }
3 { NEAR(r c, 3) }
4 { NEAR(r c, 2) }
5 { NEAR(r c, 0) }
6 { NEAR(a b c) }
7 { NEAR(a b c, 8) }
8 { x : NEAR(r c) }
9 { y : NEAR(r c) }
10 { x : "r c" }
11 { y : "r c" }
12 { a AND b }
13 { a AND b AND c }
14a { a }
14b { a OR b }
15 { a OR b AND c }
16 { c AND b OR a }
17 { c AND (b OR a) }
18 { c NOT (b OR a) }
19 { c NOT b OR a AND d }
} {
set res [matchdata $expr $bAsc]
do_execsql_test 4.$bAsc.$tn.[llength $res] $sql $res
}
}

View File

@ -41,6 +41,17 @@ foreach {tn match res} {
} $res
}
foreach {tn match res} {
5 {c*} {1}
6 {i*} {2 3}
7 {t*} {1 3}
8 {r*} {1 3}
} {
do_execsql_test 1.$tn {
SELECT rowid FROM yy WHERE yy MATCH $match ORDER BY rowid ASC
} $res
}
foreach {T create} {
2 {
CREATE VIRTUAL TABLE t1 USING fts5(a, b);
@ -178,17 +189,25 @@ foreach {T create} {
return $ret
}
foreach {tn prefix} {
1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*}
6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*}
11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*}
16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*}
21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
27 {x*}
foreach {bAsc sql} {
0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix}
1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid ASC}
} {
set res [prefix_query $prefix]
set n [llength $res]
do_execsql_test $T.$tn.$n {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} $res
foreach {tn prefix} {
1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*}
6 {f*} 7 {fg*} 8 {fgh*} 9 {fghi*} 10 {fghij*}
11 {k*} 12 {kl*} 13 {klm*} 14 {klmn*} 15 {klmno*}
16 {p*} 17 {pq*} 18 {pqr*} 19 {pqrs*} 20 {pqrst*}
21 {u*} 22 {uv*} 23 {uvw*} 24 {uvwx*} 25 {uvwxy*} 26 {uvwxyz*}
27 {x*}
} {
set res [prefix_query $prefix]
if {$bAsc} {
set res [lsort -integer -increasing $res]
}
set n [llength $res]
do_execsql_test $T.$bAsc.$tn.$n $sql $res
}
}
}

View File

@ -225,7 +225,7 @@ test_suite "fts3" -prefix "" -description {
test_suite "fts5" -prefix "" -description {
All FTS5 tests.
} -files {
fts5aa.test fts5ab.test fts5ea.test
fts5aa.test fts5ab.test fts5ac.test fts5ad.test fts5ea.test
}
test_suite "nofaultsim" -prefix "" -description {