mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add some missing comments and fix some other issues in fts3 code.

Browse Source 
FossilOrigin-Name: 2fe579e778b75fbf503c02e01e5424c1926f2b49
This commit is contained in:
dan 2009-11-18 15:35:58 +00:00
parent f13b704ee6
commit bd61689382
5 changed files with 125 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
ext/fts3/fts3.c
View File

@ -895,38 +895,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){
}
/* TODO(shess) If we pushed LeafReader to the top of the file, or to
** another file, term_select() could be pushed above
** docListOfTerm().
*/
/*
** Read a single block from the %_segments table.
*/
static int fts3ReadBlock(
Fts3Table *p,
sqlite3_int64 iBlock,
char const **pzBlock,
int *pnBlock
){
sqlite3_stmt *pStmt;
int rc = sqlite3Fts3SqlStmt(p, FTS3_SQL_GET_BLOCK, &pStmt);
if( rc!=SQLITE_OK ) return rc;
sqlite3_reset(pStmt);
sqlite3_bind_int64(pStmt, 1, iBlock);
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
return SQLITE_CORRUPT;
}
*pnBlock = sqlite3_column_bytes(pStmt, 0);
*pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
if( !*pzBlock ){
return SQLITE_NOMEM;
}
return SQLITE_OK;
}
/*
** The buffer pointed to by argument zNode (size nNode bytes) contains the
** root node of a b-tree segment. The segment is guaranteed to be at least
@ -1013,7 +981,7 @@ static int fts3SelectLeaf(
}
/* Descend to interior node iChild. */
rc = fts3ReadBlock(p, iChild, &zCsr, &nBlock);
rc = sqlite3Fts3ReadBlock(p, iChild, &zCsr, &nBlock);
if( rc!=SQLITE_OK ) break;
zEnd = &zCsr[nBlock];
}
@ -1524,7 +1492,7 @@ static int fts3TermSelect(
** (unless the root node happens to be a leaf). It simply examines the
** b-tree structure to determine which leaves need to be inspected.
*/
rc = sqlite3Fts3SqlStmt(p, FTS3_SQL_GET_ALL_SEGDIRS, &pStmt);
rc = sqlite3Fts3AllSegdirs(p, &pStmt);
while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
Fts3SegReader *pNew = 0;
int nRoot = sqlite3_column_bytes(pStmt, 4);

24
ext/fts3/fts3Int.h
View File

@ -174,6 +174,15 @@ int sqlite3Fts3UpdateMethod(sqlite3_vtab*,int,sqlite3_value**,sqlite3_int64*);
int sqlite3Fts3PendingTermsFlush(Fts3Table *);
void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);
int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char const**, int*);
int sqlite3Fts3AllSegdirs(Fts3Table*, sqlite3_stmt **);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@ -181,6 +190,7 @@ int sqlite3Fts3Optimize(Fts3Table *);
#define FTS3_SEGMENT_COLUMN_FILTER 0x00000004
#define FTS3_SEGMENT_PREFIX 0x00000008
/* Type passed as 4th argument to SegmentReaderIterate() */
struct Fts3SegFilter {
const char *zTerm;
int nTerm;
@ -188,15 +198,6 @@ struct Fts3SegFilter {
int flags;
};
int sqlite3Fts3SegReaderNew(Fts3Table *,int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3SegReaderIterate(
Fts3Table *, Fts3SegReader **, int, Fts3SegFilter *,
int (*)(Fts3Table *, void *, char *, int, char *, int), void *
);
/* fts3.c */
int sqlite3Fts3PutVarint(char *, sqlite3_int64);
int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
@ -204,11 +205,6 @@ int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
/* Valid arguments for the second argument to sqlite3Fts3SqlStmt() */
#define FTS3_SQL_GET_ALL_SEGDIRS 11
#define FTS3_SQL_GET_BLOCK 17
int sqlite3Fts3SqlStmt(Fts3Table *, int, sqlite3_stmt **);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *);

142
ext/fts3/fts3_write.c
View File

@ -29,6 +29,11 @@ typedef struct PendingList PendingList;
typedef struct SegmentNode SegmentNode;
typedef struct SegmentWriter SegmentWriter;
/*
** Data structure used while accumulating terms in the pending-terms hash
** table. The hash table entry maps from term (a string) to a malloced
** instance of this structure.
*/
struct PendingList {
int nData;
char *aData;
@ -39,9 +44,15 @@ struct PendingList {
};
/*
** fts3SegReaderNew()
** fts3SegReaderNext()
** sqlite3Fts3SegReaderFree()
** An instance of this structure is used to iterate through the terms on
** a contiguous set of segment b-tree leaf nodes. Although the details of
** this structure are only manipulated by code in this file, opaque handles
** of type Fts3SegReader* are also used by code in fts3.c to iterate through
** terms when querying the full-text index. See functions:
**
** sqlite3Fts3SegReaderNew()
** sqlite3Fts3SegReaderFree()
** sqlite3Fts3SegReaderIterate()
*/
struct Fts3SegReader {
int iIdx; /* Index within level */
@ -68,9 +79,13 @@ struct Fts3SegReader {
};
/*
** fts3LeafAdd()
** fts3LeafWrite()
** fts3LeafFree()
** An instance of this structure is used to create a segment b-tree in the
** database. The internal details of this type are only accessed by the
** following functions:
**
** fts3SegWriterAdd()
** fts3SegWriterFlush()
** fts3SegWriterFree()
*/
struct SegmentWriter {
SegmentNode *pTree; /* Pointer to interior tree structure */
@ -88,8 +103,8 @@ struct SegmentWriter {
/*
** Type SegmentNode is used by the following three functions to create
** the interior part of the segment b+-tree structures (everything except
** the leaf nodes. These functions and type are only ever used by code
** within the fts3LeafXXX() family of functions described above.
** the leaf nodes). These functions and type are only ever used by code
** within the fts3SegWriterXXX() family of functions described above.
**
** fts3NodeAddTerm()
** fts3NodeWrite()
@ -109,23 +124,8 @@ struct SegmentNode {
};
/*
** This is a comparison function used as a qsort() callback when sorting
** an array of pending terms by term.
** Valid values for the second argument to fts3SqlStmt().
*/
static int qsortCompare(const void *lhs, const void *rhs){
char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
int n = (n1<n2 ? n1 : n2);
int c = memcmp(z1, z2, n);
if( c==0 ){
c = n1 - n2;
}
return c;
}
#define SQL_DELETE_CONTENT 0
#define SQL_IS_EMPTY 1
#define SQL_DELETE_ALL_CONTENT 2
@ -136,15 +136,14 @@ static int qsortCompare(const void *lhs, const void *rhs){
#define SQL_INSERT_SEGMENTS 7
#define SQL_NEXT_SEGMENTS_ID 8
#define SQL_INSERT_SEGDIR 9
#define SQL_SELECT_LEVEL 10
#define SQL_SELECT_ALL_LEVEL 11
#define SQL_SELECT_LEVEL_COUNT 12
#define SQL_SELECT_SEGDIR_COUNT_MAX 13
#define SQL_DELETE_SEGDIR_BY_LEVEL 14
#define SQL_DELETE_SEGMENTS_RANGE 15
#define SQL_CONTENT_INSERT 16
#define SQL_GET_BLOCK 17
static int fts3SqlStmt(
Fts3Table *p,
@ -207,10 +206,57 @@ static int fts3SqlStmt(
return rc;
}
int sqlite3Fts3SqlStmt(Fts3Table *p, int eStmt, sqlite3_stmt **ppStmt){
return fts3SqlStmt(p, eStmt, ppStmt, 0);
/*
** Read a single block from the %_segments table. If the specified block
** does not exist, return SQLITE_CORRUPT. If some other error (malloc, IO
** etc.) occurs, return the appropriate SQLite error code.
**
** Otherwise, if successful, set *pzBlock to point to a buffer containing
** the block read from the database, and *pnBlock to the size of the read
** block in bytes.
**
** WARNING: The returned buffer is only valid until the next call to
** sqlite3Fts3ReadBlock().
*/
int sqlite3Fts3ReadBlock(
Fts3Table *p,
sqlite3_int64 iBlock,
char const **pzBlock,
int *pnBlock
){
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_GET_BLOCK, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_reset(pStmt);
sqlite3_bind_int64(pStmt, 1, iBlock);
rc = sqlite3_step(pStmt);
if( rc!=SQLITE_ROW ){
return SQLITE_CORRUPT;
}
*pnBlock = sqlite3_column_bytes(pStmt, 0);
*pzBlock = (char *)sqlite3_column_blob(pStmt, 0);
if( !*pzBlock ){
return SQLITE_NOMEM;
}
return SQLITE_OK;
}
/*
** Set *ppStmt to a statement handle that may be used to iterate through
** all rows in the %_segdir table, from oldest to newest. If successful,
** return SQLITE_OK. If an error occurs while preparing the statement,
** return an SQLite error code.
**
** There is only ever one instance of this SQL statement compiled for
** each FTS3 table.
*/
int sqlite3Fts3AllSegdirs(Fts3Table *p, sqlite3_stmt **ppStmt){
return fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, ppStmt, 0);
}
static int fts3SqlExec(Fts3Table *p, int eStmt, sqlite3_value **apVal){
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, eStmt, &pStmt, apVal);
@ -1191,7 +1237,7 @@ static void fts3NodeFree(SegmentNode *pTree){
}
}
static int fts3LeafAdd(
static int fts3SegWriterAdd(
Fts3Table *p, /* Virtual table handle */
SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */
int isCopyTerm, /* True if buffer zTerm must be copied */
@ -1323,7 +1369,7 @@ static int fts3LeafAdd(
return SQLITE_OK;
}
static int fts3LeafWrite(
static int fts3SegWriterFlush(
Fts3Table *p,
SegmentWriter *pWriter,
int iLevel,
@ -1352,7 +1398,7 @@ static int fts3LeafWrite(
return rc;
}
static void fts3LeafFree(SegmentWriter *pWriter){
static void fts3SegWriterFree(SegmentWriter *pWriter){
if( pWriter ){
sqlite3_free(pWriter->aData);
sqlite3_free(pWriter->zMalloc);
@ -1488,7 +1534,7 @@ static int fts3MergeCallback(
int nDoclist
){
SegmentWriter **ppW = (SegmentWriter **)pContext;
return fts3LeafAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
return fts3SegWriterAdd(p, ppW, 1, zTerm, nTerm, aDoclist, nDoclist);
}
int sqlite3Fts3SegReaderIterate(
@ -1517,7 +1563,7 @@ int sqlite3Fts3SegReaderIterate(
*/
if( pFilter->zTerm ){
int nTerm = pFilter->nTerm;
char *zTerm = pFilter->zTerm;
const char *zTerm = pFilter->zTerm;
for(i=0; i<nSegment; i++){
Fts3SegReader *pSeg = apSegment[i];
while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ){
@ -1726,11 +1772,11 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
rc = fts3DeleteSegdir(p, iLevel, apSegment, nSegment);
if( rc==SQLITE_OK ){
rc = fts3LeafWrite(p, pWriter, iNewLevel, iIdx);
rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx);
}
finished:
fts3LeafFree(pWriter);
fts3SegWriterFree(pWriter);
if( apSegment ){
for(i=0; i<nSegment; i++){
sqlite3Fts3SegReaderFree(apSegment[i]);
@ -1741,6 +1787,26 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){
return rc;
}
/*
** This is a comparison function used as a qsort() callback when sorting
** an array of pending terms by term. This occurs as part of flushing
** the contents of the pending-terms hash table to the database.
*/
static int qsortCompare(const void *lhs, const void *rhs){
char *z1 = fts3HashKey(*(Fts3HashElem **)lhs);
char *z2 = fts3HashKey(*(Fts3HashElem **)rhs);
int n1 = fts3HashKeysize(*(Fts3HashElem **)lhs);
int n2 = fts3HashKeysize(*(Fts3HashElem **)rhs);
int n = (n1<n2 ? n1 : n2);
int c = memcmp(z1, z2, n);
if( c==0 ){
c = n1 - n2;
}
return c;
}
/*
** Flush the contents of pendingTerms to a level 0 segment.
*/
@ -1791,14 +1857,14 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
const char *z = fts3HashKey(apElem[i]);
int n = fts3HashKeysize(apElem[i]);
PendingList *pList = fts3HashData(apElem[i]);
rc = fts3LeafAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
rc = fts3SegWriterAdd(p, &pWriter, 0, z, n, pList->aData, pList->nData+1);
}
if( rc==SQLITE_OK ){
rc = fts3LeafWrite(p, pWriter, 0, idx);
rc = fts3SegWriterFlush(p, pWriter, 0, idx);
}
/* Free all allocated resources before returning */
fts3LeafFree(pWriter);
fts3SegWriterFree(pWriter);
sqlite3_free(apElem);
sqlite3Fts3PendingTermsClear(p);
return rc;

16
manifest
View File

@ -1,5 +1,5 @@
C Improvements\sto\sthe\sway\sfts3\sreads\sthe\sfull-text\sindex.
D 2009-11-17T12:52:10
C Add\ssome\smissing\scomments\sand\sfix\ssome\sother\sissues\sin\sfts3\scode.
D 2009-11-18T15:35:59
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in 53f3dfa49f28ab5b80cb083fb7c9051e596bcfa1
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@ -56,9 +56,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c a72c19fa6270b5f88ad9b1215c821f7082164655
F ext/fts3/fts3.c 2aa2c3f7b4d753f048af1c777b2c5fdd7e52c560
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h 5c040c0fb47ed81aaba589a55a7455c980592bea
F ext/fts3/fts3Int.h f7488bbc9fd12bd3843f59265227321a92b39ed4
F ext/fts3/fts3_expr.c bdf11f3602f62f36f0e42823680bf22033dae0de
F ext/fts3/fts3_hash.c 1af1833a4d581ee8d668bb71f5a500f7a0104982
F ext/fts3/fts3_hash.h 39524725425078bf9e814e9569c74a8e5a21b9fb
@ -68,7 +68,7 @@ F ext/fts3/fts3_snippet.c 8ea9619247ac61c79aca650fc3307b8b4097b5f3
F ext/fts3/fts3_tokenizer.c 185a212670a9bbdeb5cad6942305e681bce5c87b
F ext/fts3/fts3_tokenizer.h 7ff73caa3327589bf6550f60d93ebdd1f6a0fb5c
F ext/fts3/fts3_tokenizer1.c 0a5bcc579f35de5d24a9345d7908dc25ae403ee7
F ext/fts3/fts3_write.c edf123f978fca3d26707452a380fa169849eb655
F ext/fts3/fts3_write.c ec6bbf26bfa21e64b059535e46b3a8e57fb21f5c
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt 3b130aa66e7a681136f6add198b076a2f90d1e33
F ext/icu/icu.c 12e763d288d23b5a49de37caa30737b971a2f1e2
@ -772,7 +772,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P f29c8fcade4aadeae3824975cf59f306c11c906b
R a419bf1bcdd810be07b904f21b0a1ef0
P 45c051e78651d8204c17cecdda2bde705698881f
R f882843f6b95cfb6a8f2fd130e122bb3
U dan
Z 4e40dbade582f444c0b93ffcad88c385
Z b8988750cf7cc171c22183eb295bebe6

2
manifest.uuid
View File

@ -1 +1 @@
45c051e78651d8204c17cecdda2bde705698881f
2fe579e778b75fbf503c02e01e5424c1926f2b49