diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index 50003632d1..d31ac07210 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -228,7 +228,7 @@ typedef struct Fts5IndexIter Fts5IndexIter; /* ** Values used as part of the flags argument passed to IndexQuery(). */ -#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ #define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ /* @@ -259,16 +259,13 @@ int sqlite3Fts5IndexQuery( ); /* -** Docid list iteration. +** The various operations on open token or token prefix iterators opened +** using sqlite3Fts5IndexQuery(). */ int sqlite3Fts5IterEof(Fts5IndexIter*); int sqlite3Fts5IterNext(Fts5IndexIter*); int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); i64 sqlite3Fts5IterRowid(Fts5IndexIter*); - -/* -** Obtain the position list that corresponds to the current position. -*/ int sqlite3Fts5IterPoslist(Fts5IndexIter*, const u8 **pp, int *pn); /* @@ -365,13 +362,6 @@ int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); */ typedef struct Fts5Hash Fts5Hash; -typedef struct Fts5Data Fts5Data; -struct Fts5Data { - u8 *p; /* Pointer to buffer containing record */ - int n; /* Size of record in bytes */ - int nRef; /* Ref count */ -}; - /* ** Create a hash table, free a hash table. */ @@ -405,7 +395,20 @@ int sqlite3Fts5HashIterate( int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ const char *pTerm, int nTerm, /* Query term */ - Fts5Data **ppData /* OUT: Query result */ + const char **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +); + +void sqlite3Fts5HashScanInit( + Fts5Hash*, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +); +void sqlite3Fts5HashScanNext(Fts5Hash*); +int sqlite3Fts5HashScanEof(Fts5Hash*); +void sqlite3Fts5HashScanEntry(Fts5Hash *, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const char **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ ); diff --git a/ext/fts5/fts5_buffer.c b/ext/fts5/fts5_buffer.c index b8f55694d8..53cb02f521 100644 --- a/ext/fts5/fts5_buffer.c +++ b/ext/fts5/fts5_buffer.c @@ -74,6 +74,7 @@ void sqlite3Fts5BufferAppendBlob( int nData, const u8 *pData ){ + assert( nData>=0 ); if( sqlite3Fts5BufferGrow(pRc, pBuf, nData) ) return; memcpy(&pBuf->p[pBuf->n], pData, nData); pBuf->n += nData; diff --git a/ext/fts5/fts5_hash.c b/ext/fts5/fts5_hash.c index 41b2eb774a..78ab2083c3 100644 --- a/ext/fts5/fts5_hash.c +++ b/ext/fts5/fts5_hash.c @@ -27,6 +27,7 @@ struct Fts5Hash { int *pnByte; /* Pointer to bytes counter */ int nEntry; /* Number of entries currently in hash */ int nSlot; /* Size of aSlot[] array */ + Fts5HashEntry *pScan; /* Current ordered scan item */ Fts5HashEntry **aSlot; /* Array of hash slots */ }; @@ -52,7 +53,8 @@ struct Fts5Hash { ** Bytes of data written since iRowidOff. */ struct Fts5HashEntry { - Fts5HashEntry *pNext; /* Next hash entry with same hash-key */ + Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ + Fts5HashEntry *pScanNext; /* Next entry in sorted order */ int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ @@ -124,7 +126,7 @@ void sqlite3Fts5HashClear(Fts5Hash *pHash){ Fts5HashEntry *pNext; Fts5HashEntry *pSlot; for(pSlot=pHash->aSlot[i]; pSlot; pSlot=pNext){ - pNext = pSlot->pNext; + pNext = pSlot->pHashNext; sqlite3_free(pSlot); } } @@ -158,9 +160,9 @@ static int fts5HashResize(Fts5Hash *pHash){ while( apOld[i] ){ int iHash; Fts5HashEntry *p = apOld[i]; - apOld[i] = p->pNext; + apOld[i] = p->pHashNext; iHash = fts5HashKey(nNew, p->zKey, strlen(p->zKey)); - p->pNext = apNew[iHash]; + p->pHashNext = apNew[iHash]; apNew[iHash] = p; } } @@ -184,7 +186,7 @@ int sqlite3Fts5HashWrite( int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ /* Attempt to locate an existing hash entry */ - for(p=pHash->aSlot[iHash]; p; p=p->pNext){ + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ if( memcmp(p->zKey, pToken, nToken)==0 && p->zKey[nToken]==0 ) break; } @@ -210,7 +212,7 @@ int sqlite3Fts5HashWrite( p->iSzPoslist = p->nData; p->nData += 4; p->iRowid = iRowid; - p->pNext = pHash->aSlot[iHash]; + p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; nIncr += p->nData; @@ -232,7 +234,7 @@ int sqlite3Fts5HashWrite( pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); if( pNew==0 ) return SQLITE_NOMEM; pNew->nAlloc = nNew; - for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pNext); + for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); *pp = pNew; p = pNew; } @@ -301,13 +303,13 @@ static Fts5HashEntry *fts5HashEntryMerge( if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){ /* p2 is smaller */ *ppOut = p2; - ppOut = &p2->pNext; - p2 = p2->pNext; + ppOut = &p2->pScanNext; + p2 = p2->pScanNext; }else{ /* p1 is smaller */ *ppOut = p1; - ppOut = &p1->pNext; - p1 = p1->pNext; + ppOut = &p1->pScanNext; + p1 = p1->pScanNext; } *ppOut = 0; } @@ -322,7 +324,11 @@ static Fts5HashEntry *fts5HashEntryMerge( ** the responsibility of the caller to free the elements of the returned ** list. */ -static int fts5HashEntrySort(Fts5Hash *pHash, Fts5HashEntry **ppSorted){ +static int fts5HashEntrySort( + Fts5Hash *pHash, + const char *pTerm, int nTerm, /* Query prefix, if any */ + Fts5HashEntry **ppSorted +){ const int nMergeSlot = 32; Fts5HashEntry **ap; Fts5HashEntry *pList; @@ -335,15 +341,17 @@ static int fts5HashEntrySort(Fts5Hash *pHash, Fts5HashEntry **ppSorted){ memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); for(iSlot=0; iSlotnSlot; iSlot++){ - while( pHash->aSlot[iSlot] ){ - Fts5HashEntry *pEntry = pHash->aSlot[iSlot]; - pHash->aSlot[iSlot] = pEntry->pNext; - pEntry->pNext = 0; - for(i=0; ap[i]; i++){ - pEntry = fts5HashEntryMerge(pEntry, ap[i]); - ap[i] = 0; + Fts5HashEntry *pIter; + for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ + if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){ + Fts5HashEntry *pEntry = pIter; + pEntry->pScanNext = 0; + for(i=0; ap[i]; i++){ + pEntry = fts5HashEntryMerge(pEntry, ap[i]); + ap[i] = 0; + } + ap[i] = pEntry; } - ap[i] = pEntry; } } @@ -368,10 +376,11 @@ int sqlite3Fts5HashIterate( Fts5HashEntry *pList; int rc; - rc = fts5HashEntrySort(pHash, &pList); + rc = fts5HashEntrySort(pHash, 0, 0, &pList); if( rc==SQLITE_OK ){ + memset(pHash->aSlot, 0, sizeof(Fts5HashEntry*) * pHash->nSlot); while( pList ){ - Fts5HashEntry *pNext = pList->pNext; + Fts5HashEntry *pNext = pList->pScanNext; if( rc==SQLITE_OK ){ const int nSz = pList->nData - pList->iSzPoslist - 4; const int nKey = strlen(pList->zKey); @@ -406,3 +415,70 @@ int sqlite3Fts5HashIterate( return rc; } +/* +** Query the hash table for a doclist associated with term pTerm/nTerm. +*/ +int sqlite3Fts5HashQuery( + Fts5Hash *pHash, /* Hash table to query */ + const char *pTerm, int nTerm, /* Query term */ + const char **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + int *pnDoclist /* OUT: Size of doclist in bytes */ +){ + unsigned int iHash = fts5HashKey(pHash->nSlot, pTerm, nTerm); + Fts5HashEntry *p; + + for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ + if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; + } + + if( p ){ + u8 *pPtr = (u8*)p; + fts5Put4ByteVarint(&pPtr[p->iSzPoslist], p->nData - p->iSzPoslist - 4); + *ppDoclist = &p->zKey[nTerm+1]; + *pnDoclist = p->nData - (sizeof(*p) + nTerm + 1); + }else{ + *ppDoclist = 0; + *pnDoclist = 0; + } + + return SQLITE_OK; +} + +void sqlite3Fts5HashScanInit( + Fts5Hash *p, /* Hash table to query */ + const char *pTerm, int nTerm /* Query prefix */ +){ + fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); +} + +void sqlite3Fts5HashScanNext(Fts5Hash *p){ + if( p->pScan ){ + p->pScan = p->pScan->pScanNext; + } +} + +int sqlite3Fts5HashScanEof(Fts5Hash *p){ + return (p->pScan==0); +} + +void sqlite3Fts5HashScanEntry( + Fts5Hash *pHash, + const char **pzTerm, /* OUT: term (nul-terminated) */ + const char **ppDoclist, /* OUT: pointer to doclist */ + int *pnDoclist /* OUT: size of doclist in bytes */ +){ + Fts5HashEntry *p; + if( (p = pHash->pScan) ){ + u8 *pPtr = (u8*)p; + int nTerm = strlen(p->zKey); + fts5Put4ByteVarint(&pPtr[p->iSzPoslist], p->nData - p->iSzPoslist - 4); + *pzTerm = p->zKey; + *ppDoclist = &p->zKey[nTerm+1]; + *pnDoclist = p->nData - (sizeof(*p) + nTerm + 1); + }else{ + *pzTerm = 0; + *ppDoclist = 0; + *pnDoclist = 0; + } +} + diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 01ad055b0d..1731d3ed70 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -274,6 +274,7 @@ int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } typedef struct Fts5BtreeIter Fts5BtreeIter; typedef struct Fts5BtreeIterLevel Fts5BtreeIterLevel; typedef struct Fts5ChunkIter Fts5ChunkIter; +typedef struct Fts5Data Fts5Data; typedef struct Fts5DlidxIter Fts5DlidxIter; typedef struct Fts5MultiSegIter Fts5MultiSegIter; typedef struct Fts5NodeIter Fts5NodeIter; @@ -286,6 +287,12 @@ typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; +struct Fts5Data { + u8 *p; /* Pointer to buffer containing record */ + int n; /* Size of record in bytes */ + int nRef; /* Ref count */ +}; + /* ** One object per %_data table. */ @@ -1514,7 +1521,7 @@ static void fts5SegIterNextPage( Fts5SegIter *pIter /* Iterator to advance to next page */ ){ Fts5StructureSegment *pSeg = pIter->pSeg; - if( pIter->pLeaf ) fts5DataRelease(pIter->pLeaf); + fts5DataRelease(pIter->pLeaf); pIter->iLeafPgno++; if( pIter->iLeafPgno<=pSeg->pgnoLast ){ pIter->pLeaf = fts5DataRead(p, @@ -1775,6 +1782,26 @@ static void fts5SegIterNext( }else{ pIter->iRowid += iDelta; } + }else if( pIter->pSeg==0 ){ + const char *pList = 0; + const char *zTerm; + int nList; + if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ + sqlite3Fts5HashScanNext(p->apHash[0]); + sqlite3Fts5HashScanEntry(p->apHash[0], &zTerm, &pList, &nList); + } + if( pList==0 ){ + fts5DataRelease(pIter->pLeaf); + pIter->pLeaf = 0; + }else{ + pIter->pLeaf->p = (u8*)pList; + pIter->pLeaf->n = nList; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, strlen(zTerm), (u8*)zTerm); + pIter->iLeafOffset = getVarint((u8*)pList, (u64*)&pIter->iRowid); + if( pIter->flags & FTS5_SEGITER_REVERSE ){ + fts5SegIterReverseInitPage(p, pIter); + } + } }else{ iOff = 0; /* Next entry is not on the current page */ @@ -2018,6 +2045,58 @@ static void fts5SegIterSeekInit( } } +/* +** Initialize the object pIter to point to term pTerm/nTerm within the +** in-memory hash table iIdx. If there is no such term in the table, the +** iterator is set to EOF. +** +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** an error has already occurred when this function is called, it is a no-op. +*/ +static void fts5SegIterHashInit( + Fts5Index *p, /* FTS5 backend */ + int iIdx, /* Config.aHash[] index of FTS index */ + const u8 *pTerm, int nTerm, /* Term to seek to */ + int flags, /* Mask of FTS5INDEX_XXX flags */ + Fts5SegIter *pIter /* Object to populate */ +){ + Fts5Hash *pHash = p->apHash[iIdx]; + const char *pList = 0; + int nList = 0; + const u8 *z = 0; + int n = 0; + + assert( pHash ); + + if( pTerm==0 || (iIdx==0 && (flags & FTS5INDEX_QUERY_PREFIX)) ){ + sqlite3Fts5HashScanInit(pHash, (const char*)pTerm, nTerm); + sqlite3Fts5HashScanEntry(pHash, (const char**)&z, &pList, &nList); + n = (z ? strlen((const char*)z) : 0); + }else{ + pIter->flags |= FTS5_SEGITER_ONETERM; + sqlite3Fts5HashQuery(pHash, (const char*)pTerm, nTerm, &pList, &nList); + z = pTerm; + n = nTerm; + } + + if( pList ){ + Fts5Data *pLeaf; + sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf==0 ) return; + pLeaf->nRef = 1; + pLeaf->p = (u8*)pList; + pLeaf->n = nList; + pIter->pLeaf = pLeaf; + pIter->iLeafOffset = getVarint(pLeaf->p, (u64*)&pIter->iRowid); + + if( flags & FTS5INDEX_QUERY_DESC ){ + pIter->flags |= FTS5_SEGITER_REVERSE; + fts5SegIterReverseInitPage(p, pIter); + } + } +} + /* ** Zero the iterator passed as the only argument. */ @@ -2261,6 +2340,7 @@ static void fts5MultiIterNew( /* Allocate space for the new multi-seg-iterator. */ if( iLevel<0 ){ nSeg = fts5StructureCountSegments(pStruct); + nSeg += (p->apHash ? 1 : 0); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } @@ -2280,6 +2360,11 @@ static void fts5MultiIterNew( /* Initialize each of the component segment iterators. */ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; + if( p->apHash ){ + /* Add a segment iterator for the current contents of the hash table. */ + Fts5SegIter *pIter = &pNew->aSeg[iIter++]; + fts5SegIterHashInit(p, iIdx, pTerm, nTerm, flags, pIter); + } for(pLvl=&pStruct->aLevel[0]; pLvlnSeg-1; iSeg>=0; iSeg--){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; @@ -2406,13 +2491,19 @@ static void fts5ChunkIterInit( Fts5SegIter *pSeg, /* Segment iterator to read poslist from */ Fts5ChunkIter *pIter /* Initialize this object */ ){ - int iId = pSeg->pSeg->iSegid; - i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno); Fts5Data *pLeaf = pSeg->pLeaf; int iOff = pSeg->iLeafOffset; memset(pIter, 0, sizeof(*pIter)); - pIter->iLeafRowid = rowid; + /* If Fts5SegIter.pSeg is NULL, then this iterator iterates through data + ** currently stored in a hash table. In this case there is no leaf-rowid + ** to calculate. */ + if( pSeg->pSeg ){ + int iId = pSeg->pSeg->iSegid; + i64 rowid = FTS5_SEGMENT_ROWID(pSeg->iIdx, iId, 0, pSeg->iLeafPgno); + pIter->iLeafRowid = rowid; + } + if( iOffn ){ fts5DataReference(pLeaf); pIter->pLeaf = pLeaf; @@ -3100,6 +3191,7 @@ fprintf(stdout, "merging %d segments from level %d!", nInput, iLvl); fflush(stdout); #endif + assert( iLvl>=0 ); for(fts5MultiIterNew(p, pStruct, iIdx, 0, 0, 0, 0, iLvl, nInput, &pIter); fts5MultiIterEof(p, pIter)==0; fts5MultiIterNext(p, pIter, 0, 0) diff --git a/ext/fts5/test/fts5ab.test b/ext/fts5/test/fts5ab.test index d68240901e..23fdec0dfa 100644 --- a/ext/fts5/test/fts5ab.test +++ b/ext/fts5/test/fts5ab.test @@ -247,14 +247,21 @@ do_execsql_test 6.0 { INSERT INTO s3 VALUES('A B C'); } -do_execsql_test 6.1 { +do_execsql_test 6.1.1 { SELECT rowid FROM s3 WHERE s3 MATCH 'a' +} {1 2} + +do_execsql_test 6.1.2 { + SELECT rowid FROM s3 WHERE s3 MATCH 'a' ORDER BY rowid DESC } {2 1} do_execsql_test 6.2 { COMMIT; +} + +do_execsql_test 6.3 { SELECT rowid FROM s3 WHERE s3 MATCH 'a' -} {2 1} +} {1 2} finish_test diff --git a/ext/fts5/test/fts5ac.test b/ext/fts5/test/fts5ac.test index 999b0788f0..b061d0bf6f 100644 --- a/ext/fts5/test/fts5ac.test +++ b/ext/fts5/test/fts5ac.test @@ -22,11 +22,6 @@ ifcapable !fts5 { return } -do_execsql_test 1.0 { - CREATE VIRTUAL TABLE xx USING fts5(x,y); - INSERT INTO xx(xx, rank) VALUES('pgsz', 32); -} - set data { 0 {p o q e z k z p n f y u z y n y} {l o o l v v k} 1 {p k h h p y l l h i p v n} {p p l u r i f a j g e r r x w} @@ -130,13 +125,6 @@ set data { 99 {r c v w i v h a t a c v c r e} {h h u m g o f b a e o} } -do_test 1.1 { - foreach {id x y} $data { - execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } - } - execsql { INSERT INTO xx(xx) VALUES('integrity-check') } -} {} - # Usage: # # poslist aCol ?-pc VARNAME? ?-near N? ?-col C? -- phrase1 phrase2... @@ -302,112 +290,94 @@ proc fts5_test_poslist {cmd} { set res } -sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist - -#------------------------------------------------------------------------- -# Test phrase queries. -# -foreach {tn phrase} { - 1 "o" - 2 "b q" - 3 "e a e" - 4 "m d g q q b k b w f q q p p" - 5 "l o o l v v k" - 6 "a" - 7 "b" - 8 "c" - 9 "no" - 10 "L O O L V V K" +foreach {tn2 sql} { + 1 {} + 2 {BEGIN} } { - set expr "\"$phrase\"" - set res [matchdata 1 $expr] + reset_db + sqlite3_fts5_create_function db fts5_test_poslist fts5_test_poslist - do_execsql_test 1.2.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res -} + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE xx USING fts5(x,y); + INSERT INTO xx(xx, rank) VALUES('pgsz', 32); + } -#------------------------------------------------------------------------- -# Test some AND and OR queries. -# -foreach {tn expr} { - 1.1 "a AND b" - 1.2 "a+b AND c" - 1.3 "d+c AND u" - 1.4 "d+c AND u+d" + execsql $sql - 2.1 "a OR b" - 2.2 "a+b OR c" - 2.3 "d+c OR u" - 2.4 "d+c OR u+d" + do_test $tn2.1.1 { + foreach {id x y} $data { + execsql { INSERT INTO xx(rowid, x, y) VALUES($id, $x, $y) } + } + execsql { INSERT INTO xx(xx) VALUES('integrity-check') } + } {} - 3.1 { a AND b AND c } -} { - set res [matchdata 1 $expr] - do_execsql_test 2.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res -} + #------------------------------------------------------------------------- + # Test phrase queries. + # + foreach {tn phrase} { + 1 "o" + 2 "b q" + 3 "e a e" + 4 "m d g q q b k b w f q q p p" + 5 "l o o l v v k" + 6 "a" + 7 "b" + 8 "c" + 9 "no" + 10 "L O O L V V K" + } { + set expr "\"$phrase\"" + set res [matchdata 1 $expr] -#------------------------------------------------------------------------- -# Queries on a specific column. -# -foreach {tn expr} { - 1 "x:a" - 2 "y:a" - 3 "x:b" - 4 "y:b" -} { - set res [matchdata 1 $expr] - do_execsql_test 3.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res -} + do_execsql_test $tn2.1.2.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + } -#------------------------------------------------------------------------- -# Some NEAR queries. -# -foreach {tn expr} { - 1 "NEAR(a b)" - 2 "NEAR(r c)" - 2 { NEAR(r c, 5) } - 3 { NEAR(r c, 3) } - 4 { NEAR(r c, 2) } - 5 { NEAR(r c, 0) } - 6 { NEAR(a b c) } - 7 { NEAR(a b c, 8) } - 8 { x : NEAR(r c) } - 9 { y : NEAR(r c) } -} { - set res [matchdata 1 $expr] - do_execsql_test 4.1.$tn.[llength $res] { - SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr - } $res -} - -do_test 4.1 { poslist {{a b c}} -- a } {0.0.0} -do_test 4.2 { poslist {{a b c}} -- c } {0.0.2} - -foreach {tn expr tclexpr} { - 1 {a b} {[N $x -- {a}] && [N $x -- {b}]} -} { - do_execsql_test 5.$tn {SELECT fts5_expr_tcl($expr, 'N $x')} [list $tclexpr] -} - -#------------------------------------------------------------------------- -# -do_execsql_test 6.integrity { - INSERT INTO xx(xx) VALUES('integrity-check'); -} -#db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} -foreach {bAsc sql} { - 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} - 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} -} { + #------------------------------------------------------------------------- + # Test some AND and OR queries. + # foreach {tn expr} { - 0.1 x - 1 { NEAR(r c) } + 1.1 "a AND b" + 1.2 "a+b AND c" + 1.3 "d+c AND u" + 1.4 "d+c AND u+d" + + 2.1 "a OR b" + 2.2 "a+b OR c" + 2.3 "d+c OR u" + 2.4 "d+c OR u+d" + + 3.1 { a AND b AND c } + } { + set res [matchdata 1 $expr] + do_execsql_test $tn2.2.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + } + + #------------------------------------------------------------------------- + # Queries on a specific column. + # + foreach {tn expr} { + 1 "x:a" + 2 "y:a" + 3 "x:b" + 4 "y:b" + } { + set res [matchdata 1 $expr] + do_execsql_test $tn2.3.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + } + + #------------------------------------------------------------------------- + # Some NEAR queries. + # + foreach {tn expr} { + 1 "NEAR(a b)" + 2 "NEAR(r c)" 2 { NEAR(r c, 5) } 3 { NEAR(r c, 3) } 4 { NEAR(r c, 2) } @@ -416,20 +386,60 @@ foreach {bAsc sql} { 7 { NEAR(a b c, 8) } 8 { x : NEAR(r c) } 9 { y : NEAR(r c) } - 10 { x : "r c" } - 11 { y : "r c" } - 12 { a AND b } - 13 { a AND b AND c } - 14a { a } - 14b { a OR b } - 15 { a OR b AND c } - 16 { c AND b OR a } - 17 { c AND (b OR a) } - 18 { c NOT (b OR a) } - 19 { c NOT b OR a AND d } } { - set res [matchdata 0 $expr $bAsc] - do_execsql_test 6.$bAsc.$tn.[llength $res] $sql $res + set res [matchdata 1 $expr] + do_execsql_test $tn2.4.1.$tn.[llength $res] { + SELECT rowid, fts5_test_poslist(xx) FROM xx WHERE xx match $expr + } $res + } + + do_test $tn2.4.1 { poslist {{a b c}} -- a } {0.0.0} + do_test $tn2.4.2 { poslist {{a b c}} -- c } {0.0.2} + + foreach {tn expr tclexpr} { + 1 {a b} {[N $x -- {a}] && [N $x -- {b}]} + } { + do_execsql_test $tn2.5.$tn { + SELECT fts5_expr_tcl($expr, 'N $x') + } [list $tclexpr] + } + + #------------------------------------------------------------------------- + # + do_execsql_test $tn2.6.integrity { + INSERT INTO xx(xx) VALUES('integrity-check'); + } + #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM xx_data} {puts $r} + foreach {bAsc sql} { + 1 {SELECT rowid FROM xx WHERE xx MATCH $expr} + 0 {SELECT rowid FROM xx WHERE xx MATCH $expr ORDER BY rowid DESC} + } { + foreach {tn expr} { + 0.1 x + 1 { NEAR(r c) } + 2 { NEAR(r c, 5) } + 3 { NEAR(r c, 3) } + 4 { NEAR(r c, 2) } + 5 { NEAR(r c, 0) } + 6 { NEAR(a b c) } + 7 { NEAR(a b c, 8) } + 8 { x : NEAR(r c) } + 9 { y : NEAR(r c) } + 10 { x : "r c" } + 11 { y : "r c" } + 12 { a AND b } + 13 { a AND b AND c } + 14a { a } + 14b { a OR b } + 15 { a OR b AND c } + 16 { c AND b OR a } + 17 { c AND (b OR a) } + 18 { c NOT (b OR a) } + 19 { c NOT b OR a AND d } + } { + set res [matchdata 0 $expr $bAsc] + do_execsql_test $tn2.6.$bAsc.$tn.[llength $res] $sql $res + } } } diff --git a/ext/fts5/test/fts5ad.test b/ext/fts5/test/fts5ad.test index 89a5078816..461fe41e50 100644 --- a/ext/fts5/test/fts5ad.test +++ b/ext/fts5/test/fts5ad.test @@ -62,6 +62,18 @@ foreach {T create} { INSERT INTO t1(t1, rank) VALUES('pgsz', 32); } + 4 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + } + + 5 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, prefix=1,2,3,4,5); + INSERT INTO t1(t1, rank) VALUES('pgsz', 32); + BEGIN; + } + } { do_test $T.1 { @@ -194,8 +206,8 @@ foreach {T create} { } foreach {bAsc sql} { - 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} 0 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix ORDER BY rowid DESC} + 1 {SELECT rowid FROM t1 WHERE t1 MATCH $prefix} } { foreach {tn prefix} { 1 {a*} 2 {ab*} 3 {abc*} 4 {abcd*} 5 {abcde*} @@ -211,9 +223,12 @@ foreach {T create} { set res [lsort -integer -increasing $res] } set n [llength $res] + if {$T==5} breakpoint do_execsql_test $T.$bAsc.$tn.$n $sql $res } } + + catchsql COMMIT } finish_test diff --git a/manifest b/manifest index 6b87d2a44b..d807680b8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sfts5\sdoclist-indexes\sthat\soccured\sif\sthe\sfirst\srowid\sof\sthe\sfirst\snon-term\spage\sof\sa\sdoclist\sis\szero. -D 2015-01-27T20:41:00.681 +C Fix\ssome\sproblems\swith\stransactions\sthat\sboth\sread\sand\swrite\san\sfts5\stable. +D 2015-01-29T20:59:34.380 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5407a688f4d77a05c18a8142be8ae5a2829dd610 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -106,13 +106,13 @@ F ext/fts3/unicode/mkunicode.tcl 4199cb887040ee3c3cd59a5171ddb0566904586e F ext/fts5/extract_api_docs.tcl 55a6d648d516f35d9a1e580ac00de27154e1904a F ext/fts5/fts5.c f2e899fba27ca33c8897635752c4c83a40dcb18d F ext/fts5/fts5.h f931954065693898d26c51f23f1d27200184a69a -F ext/fts5/fts5Int.h e3b9344d8209c9639825c711662d5d039eb70322 +F ext/fts5/fts5Int.h f7cf9331f34c5a5a83a88f43148161daa4cc0233 F ext/fts5/fts5_aux.c 549aef152b0fd46020f5595d861b1fd60b3f9b4f -F ext/fts5/fts5_buffer.c 32dd3c950392346ca69a0f1803501766c5c954f9 +F ext/fts5/fts5_buffer.c b92ba0eb67532d174934087f93716caf9a2168c7 F ext/fts5/fts5_config.c e3421a76c2abd33a05ac09df0c97c64952d1e700 F ext/fts5/fts5_expr.c 473e3428a9a637fa6e61d64d8ca3796ec57a58e9 -F ext/fts5/fts5_hash.c 4ab952b75f27d5ed3ef0f3b4f7fa1464744483e8 -F ext/fts5/fts5_index.c ef6c7764a9f4968465936839c8f7e7423d8458c2 +F ext/fts5/fts5_hash.c b54822ca901fb76d79c6a09daecbc464e5fe02c1 +F ext/fts5/fts5_index.c 1550befd9622d009520fdadfa0b42154e0ac54c0 F ext/fts5/fts5_storage.c f7c12c9f454b2a525827b3d85fd222789236f548 F ext/fts5/fts5_tcl.c 1293fac2bb26903fd3d5cdee59c5885ba7e620d5 F ext/fts5/fts5_tokenize.c 7c61d5c35c3449597bdeaa54dd48afe26852c7b0 @@ -121,9 +121,9 @@ F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba F ext/fts5/test/fts5_common.tcl d9ea79fdbc9ecbb3541bf89d13ee0e03a8dc3d32 F ext/fts5/test/fts5aa.test 065767c60ad301f77ad95f24369305e13347aa00 -F ext/fts5/test/fts5ab.test 127769288519ed549c57d7e11628dbe5b9952ad5 -F ext/fts5/test/fts5ac.test cc39f7debda6f10ca2422e17163f9b6f078d5560 -F ext/fts5/test/fts5ad.test 6c970531caf865b65f4e1dd9d6d43bd6ea37d754 +F ext/fts5/test/fts5ab.test 5da2e92a8047860b9e22b6fd3990549639d631b1 +F ext/fts5/test/fts5ac.test 8b3c2938840da8f3f6a53b1324fb03e0bac12d1e +F ext/fts5/test/fts5ad.test 2141b0360dc4397bfed30f0b0d700fa64b44835d F ext/fts5/test/fts5ae.test 347c96db06aab23ff00cf6a6b4064a8dbb182e42 F ext/fts5/test/fts5af.test c2501ec2b61d6b179c305f5d2b8782ab3d4f832a F ext/fts5/test/fts5ag.test ec3e119b728196620a31507ef503c455a7a73505 @@ -1283,7 +1283,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5206ca6005bfa9dfc7346d4b89430c9748d32c10 -R dc023966ceb63b949d8070662e553f89 +P f704bc059e06b01f1d68fa7dad89e33eace6c389 +R f45496311c450f4a551203517eb9c071 U dan -Z 99344f3fa1c5e2c02514e48da6c76a56 +Z d89173c476e3f912e9f3a6ccba8c9b1b diff --git a/manifest.uuid b/manifest.uuid index 10e9759b14..a041d59a2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f704bc059e06b01f1d68fa7dad89e33eace6c389 \ No newline at end of file +0e225b15357765f132c3364b222f9931a608a5b2 \ No newline at end of file