Add the "loadfts" program, for performance testing the loading of data into fts3/fts4/fts5 tables.

FossilOrigin-Name: 770b9540c19ad1e3d24adff382332bf032065efd

ext/fts5/fts5_aux.c

@@ -14,26 +14,26 @@
 #include "fts5Int.h"
 #include <math.h>
 
-typedef struct SnippetPhrase SnippetPhrase;
+typedef struct SnipPhrase SnipPhrase;
-typedef struct SnippetIter SnippetIter;
+typedef struct SnipIter SnipIter;
 typedef struct SnippetCtx SnippetCtx;
 
-struct SnippetPhrase {
+struct SnipPhrase {
   u64 mask;                       /* Current mask */
   int nToken;                     /* Tokens in this phrase */
   int i;                          /* Current offset in phrase poslist */
   i64 iPos;                       /* Next position in phrase (-ve -> EOF) */
 };
 
-struct SnippetIter {
+struct SnipIter {
   i64 iLast;                      /* Last token position of current snippet */
   int nScore;                     /* Score of current snippet */
 
   const Fts5ExtensionApi *pApi;
   Fts5Context *pFts;
-  u64 szmask;                     /* Mask used to on SnippetPhrase.mask */
+  u64 szmask;                     /* Mask used to on SnipPhrase.mask */
   int nPhrase;                    /* Number of phrases */
-  SnippetPhrase aPhrase[0];       /* Array of size nPhrase */
+  SnipPhrase aPhrase[0];       /* Array of size nPhrase */
 };
 
 struct SnippetCtx {
@@ -71,13 +71,13 @@ static int fts5SnippetCallback(
 /*
 ** Set pIter->nScore to the score for the current entry.
 */
-static void fts5SnippetCalculateScore(SnippetIter *pIter){
+static void fts5SnippetCalculateScore(SnipIter *pIter){
   int i;
   int nScore = 0;
   assert( pIter->iLast>=0 );
 
   for(i=0; i<pIter->nPhrase; i++){
-    SnippetPhrase *p = &pIter->aPhrase[i];
+    SnipPhrase *p = &pIter->aPhrase[i];
     u64 mask = p->mask;
     if( mask ){
       u64 j;
@@ -94,21 +94,21 @@ static void fts5SnippetCalculateScore(SnippetIter *pIter){
 /*
 ** Allocate a new snippet iter.
 */
-static int fts5SnippetIterNew(
+static int fts5SnipIterNew(
   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   Fts5Context *pFts,              /* First arg to pass to pApi functions */
   int nToken,                     /* Number of tokens in snippets */
-  SnippetIter **ppIter            /* OUT: New object */
+  SnipIter **ppIter            /* OUT: New object */
 ){
   int i;                          /* Counter variable */
-  SnippetIter *pIter;             /* New iterator object */
+  SnipIter *pIter;             /* New iterator object */
   int nByte;                      /* Bytes of space to allocate */
   int nPhrase;                    /* Number of phrases in query */
 
   *ppIter = 0;
   nPhrase = pApi->xPhraseCount(pFts);
-  nByte = sizeof(SnippetIter) + nPhrase * sizeof(SnippetPhrase);
+  nByte = sizeof(SnipIter) + nPhrase * sizeof(SnipPhrase);
-  pIter = (SnippetIter*)sqlite3_malloc(nByte);
+  pIter = (SnipIter*)sqlite3_malloc(nByte);
   if( pIter==0 ) return SQLITE_NOMEM;
   memset(pIter, 0, nByte);
 
@@ -129,16 +129,16 @@ static int fts5SnippetIterNew(
 /*
 ** Set the iterator to point to the first candidate snippet.
 */
-static void fts5SnippetIterFirst(SnippetIter *pIter){
+static void fts5SnipIterFirst(SnipIter *pIter){
   const Fts5ExtensionApi *pApi = pIter->pApi;
   Fts5Context *pFts = pIter->pFts;
   int i;                          /* Used to iterate through phrases */
-  SnippetPhrase *pMin = 0;        /* Phrase with first match */
+  SnipPhrase *pMin = 0;        /* Phrase with first match */
 
-  memset(pIter->aPhrase, 0, sizeof(SnippetPhrase) * pIter->nPhrase);
+  memset(pIter->aPhrase, 0, sizeof(SnipPhrase) * pIter->nPhrase);
 
   for(i=0; i<pIter->nPhrase; i++){
-    SnippetPhrase *p = &pIter->aPhrase[i];
+    SnipPhrase *p = &pIter->aPhrase[i];
     p->nToken = pApi->xPhraseSize(pFts, i);
     pApi->xPoslist(pFts, i, &p->i, &p->iPos);
     if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ){
@@ -156,26 +156,26 @@ static void fts5SnippetIterFirst(SnippetIter *pIter){
 /*
 ** Advance the snippet iterator to the next candidate snippet.
 */
-static void fts5SnippetIterNext(SnippetIter *pIter){
+static void fts5SnipIterNext(SnipIter *pIter){
   const Fts5ExtensionApi *pApi = pIter->pApi;
   Fts5Context *pFts = pIter->pFts;
   int nPhrase = pIter->nPhrase;
   int i;                          /* Used to iterate through phrases */
-  SnippetPhrase *pMin = 0;
+  SnipPhrase *pMin = 0;
 
   for(i=0; i<nPhrase; i++){
-    SnippetPhrase *p = &pIter->aPhrase[i];
+    SnipPhrase *p = &pIter->aPhrase[i];
     if( p->iPos>=0 && (pMin==0 || p->iPos<pMin->iPos) ) pMin = p;
   }
 
   if( pMin==0 ){
-    /* pMin==0 indicates that the SnippetIter is at EOF. */
+    /* pMin==0 indicates that the SnipIter is at EOF. */
     pIter->iLast = -1;
   }else{
     i64 nShift = pMin->iPos - pIter->iLast;
     assert( nShift>=0 );
     for(i=0; i<nPhrase; i++){
-      SnippetPhrase *p = &pIter->aPhrase[i];
+      SnipPhrase *p = &pIter->aPhrase[i];
       if( nShift>=63 ){
         p->mask = 0;
       }else{
@@ -191,7 +191,7 @@ static void fts5SnippetIterNext(SnippetIter *pIter){
   }
 }
 
-static void fts5SnippetIterFree(SnippetIter *pIter){
+static void fts5SnipIterFree(SnipIter *pIter){
   if( pIter ){
     sqlite3_free(pIter);
   }
@@ -200,7 +200,7 @@ static void fts5SnippetIterFree(SnippetIter *pIter){
 static int fts5SnippetText(
   const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
   Fts5Context *pFts,              /* First arg to pass to pApi functions */
-  SnippetIter *pIter,             /* Snippet to write to buffer */
+  SnipIter *pIter,             /* Snippet to write to buffer */
   int nToken,                     /* Size of desired snippet in tokens */
   const char *zStart,
   const char *zFinal,
@@ -299,7 +299,7 @@ static int fts5SnippetText(
       /* Check if this is the first token of any phrase match. */
       int ip;
       for(ip=0; ip<pIter->nPhrase; ip++){
-        SnippetPhrase *pPhrase = &pIter->aPhrase[ip];
+        SnipPhrase *pPhrase = &pIter->aPhrase[ip];
         u64 m = (1 << (iLast - i - pPhrase->nToken + 1));
 
         if( i<=iLast && (pPhrase->mask & m) ){
@@ -368,7 +368,7 @@ static void fts5SnippetFunction(
   int nToken = -15;
   int nAbs;
   int rc;
-  SnippetIter *pIter = 0;
+  SnipIter *pIter = 0;
 
   if( nVal>=1 ) zStart = (const char*)sqlite3_value_text(apVal[0]);
   if( nVal>=2 ) zFinal = (const char*)sqlite3_value_text(apVal[1]);
@@ -379,20 +379,20 @@ static void fts5SnippetFunction(
   }
   nAbs = nToken * (nToken<0 ? -1 : 1);
 
-  rc = fts5SnippetIterNew(pApi, pFts, nAbs, &pIter);
+  rc = fts5SnipIterNew(pApi, pFts, nAbs, &pIter);
   if( rc==SQLITE_OK ){
     Fts5Buffer buf;               /* Result buffer */
     int nBestScore = 0;           /* Score of best snippet found */
 
-    for(fts5SnippetIterFirst(pIter); 
+    for(fts5SnipIterFirst(pIter); 
         pIter->iLast>=0; 
-        fts5SnippetIterNext(pIter)
+        fts5SnipIterNext(pIter)
     ){
       if( pIter->nScore>nBestScore ) nBestScore = pIter->nScore;
     }
-    for(fts5SnippetIterFirst(pIter); 
+    for(fts5SnipIterFirst(pIter); 
         pIter->iLast>=0; 
-        fts5SnippetIterNext(pIter)
+        fts5SnipIterNext(pIter)
     ){
       if( pIter->nScore==nBestScore ) break;
     }
@@ -405,7 +405,7 @@ static void fts5SnippetFunction(
     sqlite3_free(buf.p);
   }
 
-  fts5SnippetIterFree(pIter);
+  fts5SnipIterFree(pIter);
   if( rc!=SQLITE_OK ){
     sqlite3_result_error_code(pCtx, rc);
   }

ext/fts5/fts5_config.c

@@ -113,7 +113,7 @@ static char *fts5Strdup(const char *z){
   return sqlite3_mprintf("%s", z);
 }
 
-void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
+void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**);
 
 /*
 ** Allocate an instance of the default tokenizer ("simple") at 
@@ -121,7 +121,7 @@ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module**);
 ** code if an error occurs.
 */
 static int fts5ConfigDefaultTokenizer(Fts5Config *pConfig){
-  sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
+  const sqlite3_tokenizer_module *pMod; /* Tokenizer module "simple" */
   sqlite3_tokenizer *pTokenizer;  /* Tokenizer instance */
   int rc;                         /* Return code */
 

ext/fts5/fts5_expr.c

@@ -997,7 +997,7 @@ i64 sqlite3Fts5ExprRowid(Fts5Expr *p){
 ** It is the responsibility of the caller to eventually free the returned
 ** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. 
 */
-static char *fts5Strdup(const char *pIn, int nIn){
+static char *fts5Strndup(const char *pIn, int nIn){
   char *zRet = (char*)sqlite3_malloc(nIn+1);
   if( zRet ){
     memcpy(zRet, pIn, nIn);
@@ -1007,7 +1007,7 @@ static char *fts5Strdup(const char *pIn, int nIn){
 }
 
 static int fts5ParseStringFromToken(Fts5Token *pToken, char **pz){
-  *pz = fts5Strdup(pToken->p, pToken->n);
+  *pz = fts5Strndup(pToken->p, pToken->n);
   if( *pz==0 ) return SQLITE_NOMEM;
   return SQLITE_OK;
 }
@@ -1115,7 +1115,7 @@ static int fts5ParseTokenize(
 
   pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
   memset(pTerm, 0, sizeof(Fts5ExprTerm));
-  pTerm->zTerm = fts5Strdup(pToken, nToken);
+  pTerm->zTerm = fts5Strndup(pToken, nToken);
 
   return pTerm->zTerm ? SQLITE_OK : SQLITE_NOMEM;
 }

main.mk

@@ -224,6 +224,18 @@ SRC += \
   $(TOP)/ext/rtree/rtree.h \
   $(TOP)/ext/rtree/rtree.c
 
+SRC += \
+   $(TOP)/ext/fts5/fts5.h \
+   $(TOP)/ext/fts5/fts5Int.h \
+   $(TOP)/ext/fts5/fts5_aux.c \
+   $(TOP)/ext/fts5/fts5_buffer.c \
+   $(TOP)/ext/fts5/fts5.c \
+   $(TOP)/ext/fts5/fts5_config.c \
+   $(TOP)/ext/fts5/fts5_expr.c \
+   $(TOP)/ext/fts5/fts5_index.c \
+   fts5parse.c \
+   $(TOP)/ext/fts5/fts5_storage.c 
+
 
 # Generated source code files
 #
@@ -684,6 +696,9 @@ wordcount$(EXE):	$(TOP)/test/wordcount.c sqlite3.c
 speedtest1$(EXE):	$(TOP)/test/speedtest1.c sqlite3.o
 	$(TCC) -I. -o speedtest1$(EXE) $(TOP)/test/speedtest1.c sqlite3.o $(THREADLIB)
 
+loadfts: $(TOP)/tool/loadfts.c libsqlite3.a
+	$(TCC) $(TOP)/tool/loadfts.c libsqlite3.a -o loadfts $(THREADLIB)
+
 # This target will fail if the SQLite amalgamation contains any exported
 # symbols that do not begin with "sqlite3_". It is run as part of the
 # releasetest.tcl script.

manifest

@@ -1,5 +1,5 @@
-C Add\stests\sand\sfixes\sfor\sbm25()\sfunction.
+C Add\sthe\s"loadfts"\sprogram,\sfor\sperformance\stesting\sthe\sloading\sof\sdata\sinto\sfts3/fts4/fts5\stables.
-D 2014-07-26T18:38:51.294
+D 2014-07-28T20:14:02.001
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -106,10 +106,10 @@ F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
 F ext/fts5/fts5.c 1496aff16dd9b0a013d14b6c8cf5b7df8c170abe
 F ext/fts5/fts5.h 8ace10d5b249a3baa983c79e7a1306d2a79cfd6a
 F ext/fts5/fts5Int.h 92fb9c4f759674ef569aebc338f363e167a8933c
-F ext/fts5/fts5_aux.c 78adc5db0ff4d6834df220ba6b3caa351d98b971
+F ext/fts5/fts5_aux.c 243156c197384e17983d6a3ed149fa2270b5bb85
 F ext/fts5/fts5_buffer.c 248c61ac9fec001602efc72a45704f3b8d367c00
-F ext/fts5/fts5_config.c 94f1b4cb4de6a7cd5780c14adb0198e289df8cef
+F ext/fts5/fts5_config.c 2138741013e189724b5d40ea7af0f48952a44916
-F ext/fts5/fts5_expr.c 65c1918002f2ec1755e4c0c28bf007659409fbd8
+F ext/fts5/fts5_expr.c e426baa54b9473cb31b8d891d7d1b923bfb5d017
 F ext/fts5/fts5_index.c 68d2d41b5c6d2f8838c3d6ebdc8b242718b8e997
 F ext/fts5/fts5_storage.c 2866e7e1de9dc851756c3a9c76b6e1d75e0facb7
 F ext/fts5/fts5parse.y 777da8e5819f75c217982c79c29d014c293acac9
@@ -156,7 +156,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk cffc02a30f1af82d35410674f70a0286587add81
+F main.mk 8118631727a27fa88eb38a07ac3b86ecb86e9eb0
 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
 F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5
 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1158,6 +1158,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
+F tool/loadfts.c 3bdd46090112c84df44a4fbae740af3836108b3f
 F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
 F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670
@@ -1165,7 +1166,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 1712d3d71256ca1f297046619c89e77a4d7c8f6d
-F tool/mksqlite3c.tcl ba274df71f5e6534b0a913c7c48eabfcbd0934b6
+F tool/mksqlite3c.tcl becaa9d5617dfe137e73dddda9dab8f58bc71e8c
 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12
 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1
 F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105
@@ -1196,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P c4d50428ab97f77e6721c4f8d03eaaf3ea91f3eb
+P 71d32f53e81921e43c933cc968cb1c18d83fe1e0
-R 3301ccb2b839356242606883792ca77e
+R 378763b2640fc19d9f72a0522c9f77b1
 U dan
-Z 456b4a2f1abc554b124e25c35490489e
+Z 3cf4ed481646bab9077300595c244e00

manifest.uuid

@@ -1 +1 @@
-71d32f53e81921e43c933cc968cb1c18d83fe1e0
+770b9540c19ad1e3d24adff382332bf032065efd

tool/loadfts.c

@@ -0,0 +1,204 @@
+/*
+** 2013-06-10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <assert.h>
+#include <string.h>
+#include <errno.h>
+#include <dirent.h>
+#include "sqlite3.h"
+
+/*
+** Implementation of the "readtext(X)" SQL function.  The entire content
+** of the file named X is read and returned as a TEXT value. It is assumed
+** the file contains UTF-8 text. NULL is returned if the file does not 
+** exist or is unreadable.
+*/
+static void readfileFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  const char *zName;
+  FILE *in;
+  long nIn;
+  void *pBuf;
+
+  zName = (const char*)sqlite3_value_text(argv[0]);
+  if( zName==0 ) return;
+  in = fopen(zName, "rb");
+  if( in==0 ) return;
+  fseek(in, 0, SEEK_END);
+  nIn = ftell(in);
+  rewind(in);
+  pBuf = sqlite3_malloc( nIn );
+  if( pBuf && 1==fread(pBuf, nIn, 1, in) ){
+    sqlite3_result_text(context, pBuf, nIn, sqlite3_free);
+  }else{
+    sqlite3_free(pBuf);
+  }
+  fclose(in);
+}
+
+/*
+** Print usage text for this program and exit.
+*/
+static void showHelp(const char *zArgv0){
+  printf("\n"
+"Usage: %s SWITCHES... DB\n"
+"\n"
+"  This program opens the database named on the command line and attempts to\n"
+"  create an FTS table named \"fts\" with a single column. If successful, it\n"
+"  recursively traverses the directory named by the -dir option and inserts\n"
+"  the contents of each file into the fts table. All files are assumed to\n"
+"  contain UTF-8 text.\n"
+"\n"
+"Switches are:\n"
+"  -fts [345]       FTS version to use (default=5)\n"
+"  -idx [01]        Create a mapping from filename to rowid (default=0)\n"
+"  -dir <path>      Root of directory tree to load data from (default=.)\n"
+, zArgv0
+);
+  exit(1);
+}
+
+/*
+** Exit with a message based on the argument and the current value of errno.
+*/
+static void error_out(const char *zText){
+  fprintf(stderr, "%s: %s\n", zText, strerror(errno));
+  exit(-1);
+}
+
+/*
+** Exit with a message based on the first argument and the error message
+** currently stored in database handle db.
+*/
+static void sqlite_error_out(const char *zText, sqlite3 *db){
+  fprintf(stderr, "%s: %s\n", zText, sqlite3_errmsg(db));
+  exit(-1);
+}
+
+/*
+** Context object for visit_file().
+*/
+typedef struct VisitContext VisitContext;
+struct VisitContext {
+  sqlite3 *db;                    /* Database handle */
+  sqlite3_stmt *pInsert;          /* INSERT INTO fts VALUES(readtext(:1)) */
+};
+
+/*
+** Callback used with traverse(). The first argument points to an object
+** of type VisitContext. This function inserts the contents of the text
+** file zPath into the FTS table.
+*/
+void visit_file(void *pCtx, const char *zPath){
+  int rc;
+  VisitContext *p = (VisitContext*)pCtx;
+  /* printf("%s\n", zPath); */
+  sqlite3_bind_text(p->pInsert, 1, zPath, -1, SQLITE_STATIC);
+  sqlite3_step(p->pInsert);
+  rc = sqlite3_reset(p->pInsert);
+  if( rc!=SQLITE_OK ) sqlite_error_out("insert", p->db);
+}
+
+/*
+** Recursively traverse directory zDir. For each file that is not a 
+** directory, invoke the supplied callback with its path.
+*/
+static void traverse(
+  const char *zDir,               /* Directory to traverse */
+  void *pCtx,                     /* First argument passed to callback */
+  void (*xCallback)(void*, const char *zPath)
+){
+  DIR *d;
+  struct dirent *e;
+
+  d = opendir(zDir);
+  if( d==0 ) error_out("opendir()");
+
+  for(e=readdir(d); e; e=readdir(d)){
+    if( strcmp(e->d_name, ".")==0 || strcmp(e->d_name, "..")==0 ) continue;
+    char *zPath = sqlite3_mprintf("%s/%s", zDir, e->d_name);
+    if (e->d_type & DT_DIR) {
+      traverse(zPath, pCtx, xCallback);
+    }else{
+      xCallback(pCtx, zPath);
+    }
+    sqlite3_free(zPath);
+  }
+
+  closedir(d);
+}
+
+int main(int argc, char **argv){
+  int iFts = 5;                   /* Value of -fts option */
+  int bMap = 0;                   /* True to create mapping table */
+  const char *zDir = ".";         /* Directory to scan */
+  int i;
+  int rc;
+  sqlite3 *db;
+  char *zSql;
+  VisitContext sCtx;
+
+  if( argc % 2 ) showHelp(argv[0]);
+
+  for(i=1; i<(argc-1); i+=2){
+    char *zOpt = argv[i];
+    char *zArg = argv[i+1];
+    if( strcmp(zOpt, "-fts")==0 ){
+      iFts = atoi(zArg);
+      if( iFts!=3 && iFts!=4 && iFts!= 5) showHelp(argv[0]);
+    }
+    else if( strcmp(zOpt, "-idx")==0 ){
+      bMap = atoi(zArg);
+      if( bMap!=0 && bMap!=1 ) showHelp(argv[0]);
+    }
+    else if( strcmp(zOpt, "-dir")==0 ){
+      zDir = zArg;
+    }
+  }
+
+  /* Open the database file */
+  rc = sqlite3_open(argv[argc-1], &db);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_open()", db);
+
+  rc = sqlite3_create_function(db, "readtext", 1, SQLITE_UTF8, 0,
+                               readfileFunc, 0, 0);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_create_function()", db);
+
+  /* Create the FTS table */
+  zSql = sqlite3_mprintf("CREATE VIRTUAL TABLE fts USING fts%d(content)", iFts);
+  rc = sqlite3_exec(db, zSql, 0, 0, 0);
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_exec(1)", db);
+  sqlite3_free(zSql);
+
+  /* Compile the INSERT statement to write data to the FTS table. */
+  memset(&sCtx, 0, sizeof(VisitContext));
+  sCtx.db = db;
+  rc = sqlite3_prepare_v2(db, 
+      "INSERT INTO fts VALUES(readtext(?))", -1, &sCtx.pInsert, 0
+  );
+  if( rc!=SQLITE_OK ) sqlite_error_out("sqlite3_prepare_v2(1)", db);
+
+  /* Load all files in the directory hierarchy into the FTS table. */
+  traverse(zDir, (void*)&sCtx, visit_file);
+
+  /* Clean up and exit. */
+  sqlite3_finalize(sCtx.pInsert);
+  sqlite3_close(db);
+  return 0;
+}

tool/mksqlite3c.tcl

@@ -97,6 +97,8 @@ foreach hdr {
    fts3Int.h
    fts3_hash.h
    fts3_tokenizer.h
+   fts5.h
+   fts5Int.h
    hash.h
    hwtime.h
    keywordhash.h
@@ -328,6 +330,15 @@ foreach file {
    fts3_unicode.c
    fts3_unicode2.c
 
+   fts5_aux.c
+   fts5_buffer.c
+   fts5.c
+   fts5_config.c
+   fts5_expr.c
+   fts5_index.c
+   fts5parse.c
+   fts5_storage.c
+
    rtree.c
    icu.c
    fts3_icu.c