Add the "loadfts" program, for performance testing the loading of data into fts3/fts4/fts5 tables.
FossilOrigin-Name: 770b9540c19ad1e3d24adff382332bf032065efd
This commit is contained in:
dan
parent
454b5ce524
commit
92e497e517
@ -14,26 +14,26 @@
|
||||
#include "fts5Int.h"
|
||||
#include <math.h>
|
||||
|
||||
typedef struct SnippetPhrase SnippetPhrase;
|
||||
typedef struct SnippetIter SnippetIter;
|
||||
typedef struct SnipPhrase SnipPhrase;
|
||||
typedef struct SnipIter SnipIter;
|
||||
typedef struct SnippetCtx SnippetCtx;
|
||||
|
||||
struct SnippetPhrase {
|
||||
struct SnipPhrase {
|
||||
u64 mask; /* Current mask */
|
||||
int nToken; /* Tokens in this phrase */
|
||||
int i; /* Current offset in phrase poslist */
|
||||
i64 iPos; /* Next position in phrase (-ve -> EOF) */
|
||||
};
|
||||
|
||||
struct SnippetIter {
|
||||
struct SnipIter {
|
||||
i64 iLast; /* Last token position of current snippet */
|
||||
int nScore; /* Score of current snippet */
|
||||
|
||||
const Fts5ExtensionApi *pApi;
|
||||
Fts5Context *pFts;
|
||||
u64 szmask; /* Mask used to on SnippetPhrase.mask */
|
||||
u64 szmask; /* Mask used to on SnipPhrase.mask */
|
||||
int nPhrase; /* Number of phrases */
|
||||
SnippetPhrase aPhrase[0]; /* Array of size nPhrase */
|
||||
SnipPhrase aPhrase[0]; /* Array of size nPhrase */
|
||||
};
|
||||
|
||||
struct SnippetCtx {
|
||||
@ -71,13 +71,13 @@ static int fts5SnippetCallback(
|
||||
/*
|
||||
** Set pIter->nScore to the score for the current entry.
|
||||
*/
|
||||
static void fts5SnippetCalculateScore(SnippetIter *pIter){
|
||||
static void fts5SnippetCalculateScore(SnipIter *pIter){
|
||||
int i;
|
||||
int nScore = 0;
|
||||
assert( pIter->iLast>=0 );
|
||||
|
||||
for(i=0; i<pIter->nPhrase; i++){
|
||||
SnippetPhrase *p = &pIter->aPhrase[i];
|
||||
SnipPhrase *p = &pIter->aPhrase[i];
|
||||
u64 mask = p->mask;
|
||||
if( mask ){
|
||||
u64 j;
|
||||
@ -94,21 +94,21 @@ static void fts5SnippetCalculateScore(SnippetIter *pIter){
|
||||
/*
|
||||
** Allocate a new snippet iter.
|
||||
*/
|
||||
static int fts5SnippetIterNew(
|
||||
static int fts5SnipIterNew(
|
||||
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
||||
Fts5Context *pFts, /* First arg to pass to pApi functions */
|
||||
int nToken, /* Number of tokens in snippets */
|
||||
SnippetIter **ppIter /* OUT: New object */
|
||||
SnipIter **ppIter /* OUT: New object */
|
||||
){
|
||||
int i; /* Counter variable */
|
||||
SnippetIter *pIter; /* New iterator object */
|
||||
SnipIter *pIter; /* New iterator object */
|
||||
int nByte; /* Bytes of space to allocate */
|
||||
int nPhrase; /* Number of phrases in query */
|
||||
|
||||
*ppIter = 0;
|
||||
nPhrase = pApi->xPhraseCount(pFts);
|
||||
nByte = sizeof(SnippetIter) + nPhrase * sizeof(SnippetPhrase);
|
||||
pIter = (SnippetIter*)sqlite3_malloc(nByte);
|
||||
nByte = sizeof(SnipIter) + nPhrase * sizeof(SnipPhrase);
|
||||
pIter = (SnipIter*)sqlite3_malloc(nByte);
|
||||
if( pIter==0 ) return SQLITE_NOMEM;
|
||||
memset(pIter, 0, nByte);
|
||||
|
||||
@ -129,16 +129,16 @@ static int fts5SnippetIterNew(
|
||||
/*
|
||||
** Set the iterator to point to the first candidate snippet.
|
||||
*/
|
||||
static void fts5SnippetIterFirst(SnippetIter *pIter){
|
||||
static void fts5SnipIterFirst(SnipIter *pIter){
|
||||
const Fts5ExtensionApi *pApi = pIter->pApi;
|
||||
Fts5Context *pFts = pIter->pFts;
|
||||
int i; /* Used to iterate through phrases */
|
||||
SnippetPhrase *pMin = 0; /* Phrase with first match */
|
||||
SnipPhrase *pMin = 0; /* Phrase with first match */
|
||||
|
||||
memset(pIter->aPhrase, 0, sizeof(SnippetPhrase) * pIter->nPhrase);
|
||||
memset(pIter->aPhrase, 0, sizeof(SnipPhrase) * pIter->nPhrase);
|
||||
|
||||
for(i=0; i<pIter->nPhrase; i++){
|
||||
SnippetPhrase *p = &pIter->aPhrase[i];
|
||||
SnipPhrase *p = &pIter->aPhrase[i];
|
||||
p->nToken = pApi->xPhraseSize(pFts, i);
|
||||
pApi->xPoslist(pFts, i, &p->i, &p->iPos);
|
||||
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
|
||||
@ -156,26 +156,26 @@ static void fts5SnippetIterFirst(SnippetIter *pIter){
|
||||
/*
|
||||
** Advance the snippet iterator to the next candidate snippet.
|
||||
*/
|
||||
static void fts5SnippetIterNext(SnippetIter *pIter){
|
||||
static void fts5SnipIterNext(SnipIter *pIter){
|
||||
const Fts5ExtensionApi *pApi = pIter->pApi;
|
||||
Fts5Context *pFts = pIter->pFts;
|
||||
int nPhrase = pIter->nPhrase;
|
||||
int i; /* Used to iterate through phrases */
|
||||
SnippetPhrase *pMin = 0;
|
||||
SnipPhrase *pMin = 0;
|
||||
|
||||
for(i=0; i<nPhrase; i++){
|
||||
SnippetPhrase *p = &pIter->aPhrase[i];
|
||||
SnipPhrase *p = &pIter->aPhrase[i];
|
||||
if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
|
||||
}
|
||||
|
||||
if( pMin==0 ){
|
||||
/* pMin==0 indicates that the SnippetIter is at EOF. */
|
||||
/* pMin==0 indicates that the SnipIter is at EOF. */
|
||||
pIter->iLast = -1;
|
||||
}else{
|
||||
i64 nShift = pMin->iPos - pIter->iLast;
|
||||
assert( nShift>=0 );
|
||||
for(i=0; i<nPhrase; i++){
|
||||
SnippetPhrase *p = &pIter->aPhrase[i];
|
||||
SnipPhrase *p = &pIter->aPhrase[i];
|
||||
if( nShift>=63 ){
|
||||
p->mask = 0;
|
||||
}else{
|
||||
@ -191,7 +191,7 @@ static void fts5SnippetIterNext(SnippetIter *pIter){
|
||||
}
|
||||
}
|
||||
|
||||
static void fts5SnippetIterFree(SnippetIter *pIter){
|
||||
static void fts5SnipIterFree(SnipIter *pIter){
|
||||
if( pIter ){
|
||||
sqlite3_free(pIter);
|
||||
}
|
||||
@ -200,7 +200,7 @@ static void fts5SnippetIterFree(SnippetIter *pIter){
|
||||
static int fts5SnippetText(
|
||||
const Fts5ExtensionApi *pApi, /* API offered by current FTS version */
|
||||
Fts5Context *pFts, /* First arg to pass to pApi functions */
|
||||
SnippetIter *pIter, /* Snippet to write to buffer */
|
||||
SnipIter *pIter, /* Snippet to write to buffer */
|
||||
int nToken, /* Size of desired snippet in tokens */
|
||||
const char *zStart,
|
||||
const char *zFinal,
|
||||
@ -299,7 +299,7 @@ static int fts5SnippetText(
|
||||
/* Check if this is the first token of any phrase match. */
|
||||
int ip;
|
||||
for(ip=0; ip<pIter->nPhrase; ip++){
|
||||
SnippetPhrase *pPhrase = &pIter->aPhrase[ip];
|
||||
SnipPhrase *pPhrase = &pIter->aPhrase[ip];
|
||||
u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
|
||||
|
||||
if( i<=iLast && (pPhrase->mask & m) ){
|
||||
@ -368,7 +368,7 @@ static void fts5SnippetFunction(
|
||||
int nToken = -15;
|
||||
int nAbs;
|
||||
int rc;
|
||||
SnippetIter *pIter = 0;
|
||||
SnipIter *pIter = 0;
|
||||
|
||||
if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
|
||||
if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
|
||||
@ -379,20 +379,20 @@ static void fts5SnippetFunction(
|
||||
}
|
||||
nAbs = nToken * (nToken<0 ? -1 : 1);
|
||||
|
||||
rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
|
||||
rc = fts5SnipIterNew(pApi, pFts, nAbs, &pIter);
|
||||
if( rc==SQLITE_OK ){
|
||||
Fts5Buffer buf; /* Result buffer */
|
||||
int nBestScore = 0; /* Score of best snippet found */
|
||||
|
||||
for(fts5SnippetIterFirst(pIter);
|
||||
for(fts5SnipIterFirst(pIter);
|
||||
pIter->iLast>=0;
|
||||
fts5SnippetIterNext(pIter)
|
||||
fts5SnipIterNext(pIter)
|
||||
){
|
||||
if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
|
||||
}
|
||||
for(fts5SnippetIterFirst(pIter);
|
||||
for(fts5SnipIterFirst(pIter);
|
||||
pIter->iLast>=0;
|
||||
fts5SnippetIterNext(pIter)
|
||||
fts5SnipIterNext(pIter)
|
||||
){
|
||||
if( pIter->nScore==nBestScore ) break;
|
||||
}
|
||||
@ -405,7 +405,7 @@ static void fts5SnippetFunction(
|
||||
sqlite3_free(buf.p);
|
||||
}
|
||||
|
||||
fts5SnippetIterFree(pIter);
|
||||
fts5SnipIterFree(pIter);
|
||||
if( rc!=SQLITE_OK ){
|
||||
sqlite3_result_error_code(pCtx, rc);
|
||||
}
|
||||
|
||||
@ -113,7 +113,7 @@ static char *fts5Strdup(const char *z){
|
||||
return sqlite3_mprintf("%s", z);
|
||||
}
|
||||
|
||||
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
|
||||
void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
|
||||
|
||||
/*
|
||||
** Allocate an instance of the default tokenizer ("simple") at
|
||||
@ -121,7 +121,7 @@ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
|
||||
** code if an error occurs.
|
||||
*/
|
||||
static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
|
||||
sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
|
||||
const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
|
||||
sqlite3_tokenizer *pTokenizer; /* Tokenizer instance */
|
||||
int rc; /* Return code */
|
||||
|
||||
|
||||
@ -997,7 +997,7 @@ i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
|
||||
** It is the responsibility of the caller to eventually free the returned
|
||||
** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned.
|
||||
*/
|
||||
static char *fts5Strdup(const char *pIn, int nIn){
|
||||
static char *fts5Strndup(const char *pIn, int nIn){
|
||||
char *zRet = (char*)sqlite3_malloc(nIn+1);
|
||||
if( zRet ){
|
||||
memcpy(zRet, pIn, nIn);
|
||||
@ -1007,7 +1007,7 @@ static char *fts5Strdup(const char *pIn, int nIn){
|
||||
}
|
||||
|
||||
static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
|
||||
*pz = fts5Strdup(pToken->p, pToken->n);
|
||||
*pz = fts5Strndup(pToken->p, pToken->n);
|
||||
if( *pz==0 ) return SQLITE_NOMEM;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
@ -1115,7 +1115,7 @@ static int fts5ParseTokenize(
|
||||
|
||||
pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
|
||||
memset(pTerm, 0, sizeof(Fts5ExprTerm));
|
||||
pTerm->zTerm = fts5Strdup(pToken, nToken);
|
||||
pTerm->zTerm = fts5Strndup(pToken, nToken);
|
||||
|
||||
return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
|
||||
}
|
||||
|
||||
15
main.mk
15
main.mk
@ -224,6 +224,18 @@ SRC += \
|
||||
$(TOP)/ext/rtree/rtree.h \
|
||||
$(TOP)/ext/rtree/rtree.c
|
||||
|
||||
SRC += \
|
||||
$(TOP)/ext/fts5/fts5.h \
|
||||
$(TOP)/ext/fts5/fts5Int.h \
|
||||
$(TOP)/ext/fts5/fts5_aux.c \
|
||||
$(TOP)/ext/fts5/fts5_buffer.c \
|
||||
$(TOP)/ext/fts5/fts5.c \
|
||||
$(TOP)/ext/fts5/fts5_config.c \
|
||||
$(TOP)/ext/fts5/fts5_expr.c \
|
||||
$(TOP)/ext/fts5/fts5_index.c \
|
||||
fts5parse.c \
|
||||
$(TOP)/ext/fts5/fts5_storage.c
|
||||
|
||||
|
||||
# Generated source code files
|
||||
#
|
||||
@ -684,6 +696,9 @@ wordcount$(EXE): $(TOP)/test/wordcount.c sqlite3.c
|
||||
speedtest1$(EXE): $(TOP)/test/speedtest1.c sqlite3.o
|
||||
$(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB)
|
||||
|
||||
loadfts: $(TOP)/tool/loadfts.c libsqlite3.a
|
||||
$(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB)
|
||||
|
||||
# This target will fail if the SQLite amalgamation contains any exported
|
||||
# symbols that do not begin with "sqlite3_". It is run as part of the
|
||||
# releasetest.tcl script.
|
||||
|
||||
21
manifest
21
manifest
@ -1,5 +1,5 @@
|
||||
C Add\stests\sand\sfixes\sfor\sbm25()\sfunction.
|
||||
D 2014-07-26T18:38:51.294
|
||||
C Add\sthe\s"loadfts"\sprogram,\sfor\sperformance\stesting\sthe\sloading\sof\sdata\sinto\sfts3/fts4/fts5\stables.
|
||||
D 2014-07-28T20:14:02.001
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
@ -106,10 +106,10 @@ F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
|
||||
F ext/fts5/fts5.c 1496aff16dd9b0a013d14b6c8cf5b7df8c170abe
|
||||
F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a
|
||||
F ext/fts5/fts5Int.h 92fb9c4f759674ef569aebc338f363e167a8933c
|
||||
F ext/fts5/fts5_aux.c 78adc5db0ff4d6834df220ba6b3caa351d98b971
|
||||
F ext/fts5/fts5_aux.c 243156c197384e17983d6a3ed149fa2270b5bb85
|
||||
F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
|
||||
F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
|
||||
F ext/fts5/fts5_expr.c 65c1918002f2ec1755e4c0c28bf007659409fbd8
|
||||
F ext/fts5/fts5_config.c 2138741013e189724b5d40ea7af0f48952a44916
|
||||
F ext/fts5/fts5_expr.c e426baa54b9473cb31b8d891d7d1b923bfb5d017
|
||||
F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
|
||||
F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7
|
||||
F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
|
||||
@ -156,7 +156,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
|
||||
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
|
||||
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
|
||||
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
|
||||
F main.mk cffc02a30f1af82d35410674f70a0286587add81
|
||||
F main.mk 8118631727a27fa88eb38a07ac3b86ecb86e9eb0
|
||||
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
|
||||
F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
|
||||
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
|
||||
@ -1158,6 +1158,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
|
||||
F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
|
||||
F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6
|
||||
F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
|
||||
F tool/loadfts.c 3bdd46090112c84df44a4fbae740af3836108b3f
|
||||
F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
|
||||
F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
|
||||
F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
|
||||
@ -1165,7 +1166,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
|
||||
F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460
|
||||
F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
|
||||
F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d
|
||||
F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6
|
||||
F tool/mksqlite3c.tcl becaa9d5617dfe137e73dddda9dab8f58bc71e8c
|
||||
F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12
|
||||
F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1
|
||||
F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105
|
||||
@ -1196,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
|
||||
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
|
||||
P c4d50428ab97f77e6721c4f8d03eaaf3ea91f3eb
|
||||
R 3301ccb2b839356242606883792ca77e
|
||||
P 71d32f53e81921e43c933cc968cb1c18d83fe1e0
|
||||
R 378763b2640fc19d9f72a0522c9f77b1
|
||||
U dan
|
||||
Z 456b4a2f1abc554b124e25c35490489e
|
||||
Z 3cf4ed481646bab9077300595c244e00
|
||||
|
||||
@ -1 +1 @@
|
||||
71d32f53e81921e43c933cc968cb1c18d83fe1e0
|
||||
770b9540c19ad1e3d24adff382332bf032065efd
|
||||
204
tool/loadfts.c
Normal file
204
tool/loadfts.c
Normal file
@ -0,0 +1,204 @@
|
||||
/*
|
||||
** 2013-06-10
|
||||
**
|
||||
** The author disclaims copyright to this source code. In place of
|
||||
** a legal notice, here is a blessing:
|
||||
**
|
||||
** May you do good and not evil.
|
||||
** May you find forgiveness for yourself and forgive others.
|
||||
** May you share freely, never taking more than you give.
|
||||
**
|
||||
*************************************************************************
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <ctype.h>
|
||||
#include <assert.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <dirent.h>
|
||||
#include "sqlite3.h"
|
||||
|
||||
/*
|
||||
** Implementation of the "readtext(X)" SQL function. The entire content
|
||||
** of the file named X is read and returned as a TEXT value. It is assumed
|
||||
** the file contains UTF-8 text. NULL is returned if the file does not
|
||||
** exist or is unreadable.
|
||||
*/
|
||||
static void readfileFunc(
|
||||
sqlite3_context *context,
|
||||
int argc,
|
||||
sqlite3_value **argv
|
||||
){
|
||||
const char *zName;
|
||||
FILE *in;
|
||||
long nIn;
|
||||
void *pBuf;
|
||||
|
||||
zName = (const char*)sqlite3_value_text(argv[0]);
|
||||
if( zName==0 ) return;
|
||||
in = fopen(zName, "rb");
|
||||
if( in==0 ) return;
|
||||
fseek(in, 0, SEEK_END);
|
||||
nIn = ftell(in);
|
||||
rewind(in);
|
||||
pBuf = sqlite3_malloc( nIn );
|
||||
if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
|
||||
sqlite3_result_text(context, pBuf, nIn, sqlite3_free);
|
||||
}else{
|
||||
sqlite3_free(pBuf);
|
||||
}
|
||||
fclose(in);
|
||||
}
|
||||
|
||||
/*
|
||||
** Print usage text for this program and exit.
|
||||
*/
|
||||
static void showHelp(const char *zArgv0){
|
||||
printf("\n"
|
||||
"Usage: %s SWITCHES... DB\n"
|
||||
"\n"
|
||||
" This program opens the database named on the command line and attempts to\n"
|
||||
" create an FTS table named \"fts\" with a single column. If successful, it\n"
|
||||
" recursively traverses the directory named by the -dir option and inserts\n"
|
||||
" the contents of each file into the fts table. All files are assumed to\n"
|
||||
" contain UTF-8 text.\n"
|
||||
"\n"
|
||||
"Switches are:\n"
|
||||
" -fts [345] FTS version to use (default=5)\n"
|
||||
" -idx [01] Create a mapping from filename to rowid (default=0)\n"
|
||||
" -dir <path> Root of directory tree to load data from (default=.)\n"
|
||||
, zArgv0
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/*
|
||||
** Exit with a message based on the argument and the current value of errno.
|
||||
*/
|
||||
static void error_out(const char *zText){
|
||||
fprintf(stderr, "%s: %s\n", zText, strerror(errno));
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
/*
|
||||
** Exit with a message based on the first argument and the error message
|
||||
** currently stored in database handle db.
|
||||
*/
|
||||
static void sqlite_error_out(const char *zText, sqlite3 *db){
|
||||
fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db));
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
/*
|
||||
** Context object for visit_file().
|
||||
*/
|
||||
typedef struct VisitContext VisitContext;
|
||||
struct VisitContext {
|
||||
sqlite3 *db; /* Database handle */
|
||||
sqlite3_stmt *pInsert; /* INSERT INTO fts VALUES(readtext(:1)) */
|
||||
};
|
||||
|
||||
/*
|
||||
** Callback used with traverse(). The first argument points to an object
|
||||
** of type VisitContext. This function inserts the contents of the text
|
||||
** file zPath into the FTS table.
|
||||
*/
|
||||
void visit_file(void *pCtx, const char *zPath){
|
||||
int rc;
|
||||
VisitContext *p = (VisitContext*)pCtx;
|
||||
/* printf("%s\n", zPath); */
|
||||
sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC);
|
||||
sqlite3_step(p->pInsert);
|
||||
rc = sqlite3_reset(p->pInsert);
|
||||
if( rc!=SQLITE_OK ) sqlite_error_out("insert", p->db);
|
||||
}
|
||||
|
||||
/*
|
||||
** Recursively traverse directory zDir. For each file that is not a
|
||||
** directory, invoke the supplied callback with its path.
|
||||
*/
|
||||
static void traverse(
|
||||
const char *zDir, /* Directory to traverse */
|
||||
void *pCtx, /* First argument passed to callback */
|
||||
void (*xCallback)(void*, const char *zPath)
|
||||
){
|
||||
DIR *d;
|
||||
struct dirent *e;
|
||||
|
||||
d = opendir(zDir);
|
||||
if( d==0 ) error_out("opendir()");
|
||||
|
||||
for(e=readdir(d); e; e=readdir(d)){
|
||||
if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
|
||||
char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
|
||||
if (e->d_type & DT_DIR) {
|
||||
traverse(zPath, pCtx, xCallback);
|
||||
}else{
|
||||
xCallback(pCtx, zPath);
|
||||
}
|
||||
sqlite3_free(zPath);
|
||||
}
|
||||
|
||||
closedir(d);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv){
|
||||
int iFts = 5; /* Value of -fts option */
|
||||
int bMap = 0; /* True to create mapping table */
|
||||
const char *zDir = "."; /* Directory to scan */
|
||||
int i;
|
||||
int rc;
|
||||
sqlite3 *db;
|
||||
char *zSql;
|
||||
VisitContext sCtx;
|
||||
|
||||
if( argc % 2 ) showHelp(argv[0]);
|
||||
|
||||
for(i=1; i<(argc-1); i+=2){
|
||||
char *zOpt = argv[i];
|
||||
char *zArg = argv[i+1];
|
||||
if( strcmp(zOpt, "-fts")==0 ){
|
||||
iFts = atoi(zArg);
|
||||
if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
|
||||
}
|
||||
else if( strcmp(zOpt, "-idx")==0 ){
|
||||
bMap = atoi(zArg);
|
||||
if( bMap!=0 && bMap!=1 ) showHelp(argv[0]);
|
||||
}
|
||||
else if( strcmp(zOpt, "-dir")==0 ){
|
||||
zDir = zArg;
|
||||
}
|
||||
}
|
||||
|
||||
/* Open the database file */
|
||||
rc = sqlite3_open(argv[argc-1], &db);
|
||||
if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db);
|
||||
|
||||
rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0,
|
||||
readfileFunc, 0, 0);
|
||||
if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db);
|
||||
|
||||
/* Create the FTS table */
|
||||
zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts);
|
||||
rc = sqlite3_exec(db, zSql, 0, 0, 0);
|
||||
if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
|
||||
sqlite3_free(zSql);
|
||||
|
||||
/* Compile the INSERT statement to write data to the FTS table. */
|
||||
memset(&sCtx, 0, sizeof(VisitContext));
|
||||
sCtx.db = db;
|
||||
rc = sqlite3_prepare_v2(db,
|
||||
"INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0
|
||||
);
|
||||
if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db);
|
||||
|
||||
/* Load all files in the directory hierarchy into the FTS table. */
|
||||
traverse(zDir, (void*)&sCtx, visit_file);
|
||||
|
||||
/* Clean up and exit. */
|
||||
sqlite3_finalize(sCtx.pInsert);
|
||||
sqlite3_close(db);
|
||||
return 0;
|
||||
}
|
||||
@ -97,6 +97,8 @@ foreach hdr {
|
||||
fts3Int.h
|
||||
fts3_hash.h
|
||||
fts3_tokenizer.h
|
||||
fts5.h
|
||||
fts5Int.h
|
||||
hash.h
|
||||
hwtime.h
|
||||
keywordhash.h
|
||||
@ -328,6 +330,15 @@ foreach file {
|
||||
fts3_unicode.c
|
||||
fts3_unicode2.c
|
||||
|
||||
fts5_aux.c
|
||||
fts5_buffer.c
|
||||
fts5.c
|
||||
fts5_config.c
|
||||
fts5_expr.c
|
||||
fts5_index.c
|
||||
fts5parse.c
|
||||
fts5_storage.c
|
||||
|
||||
rtree.c
|
||||
icu.c
|
||||
fts3_icu.c
|
||||
|
||||
Loading…
Reference in New Issue
Block a user