From 6beeb0329a7839d8f3d9aee07a59aad35a2b82a6 Mon Sep 17 00:00:00 2001 From: shess Date: Thu, 23 Aug 2007 20:23:37 +0000 Subject: [PATCH] Fix fts3 to not have the VACUUM bug from fts2. %_content.docid is an alias to fix the rowid for documents, %_segments.blockid is an alias to fix the rowid for segment blocks. Unit test for the problem. (CVS 4280) FossilOrigin-Name: 6eb2d74a8cfce322930f05c97d4ec255f3711efb --- ext/fts3/fts3.c | 55 +++++++++++++++++-------- manifest | 15 +++---- manifest.uuid | 2 +- test/fts3b.test | 106 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 24 deletions(-) create mode 100644 test/fts3b.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index fe2b32f50e..64b0113951 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1772,13 +1772,14 @@ typedef enum fulltext_statement { */ static const char *const fulltext_zStatement[MAX_STMT] = { /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */ - /* CONTENT_SELECT */ "select * from %_content where rowid = ?", + /* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */ /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */ - /* CONTENT_DELETE */ "delete from %_content where rowid = ?", + /* CONTENT_DELETE */ "delete from %_content where docid = ?", - /* BLOCK_INSERT */ "insert into %_segments values (?)", - /* BLOCK_SELECT */ "select block from %_segments where rowid = ?", - /* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?", + /* BLOCK_INSERT */ + "insert into %_segments (blockid, block) values (null, ?)", + /* BLOCK_SELECT */ "select block from %_segments where blockid = ?", + /* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?", /* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?", /* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)", @@ -1821,7 +1822,7 @@ struct fulltext_vtab { sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT]; /* The statement used to prepare pLeafSelectStmts. */ #define LEAF_SELECT \ - "select block from %_segments where rowid between ? and ? order by rowid" + "select block from %_segments where blockid between ? and ? order by blockid" /* These buffer pending index updates during transactions. ** nPendingData estimates the memory size of the pending data. It @@ -1863,14 +1864,14 @@ static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){ static const sqlite3_module fts3Module; /* forward declaration */ /* Return a dynamically generated statement of the form - * insert into %_content (rowid, ...) values (?, ...) + * insert into %_content (docid, ...) values (?, ...) */ static const char *contentInsertStatement(fulltext_vtab *v){ StringBuffer sb; int i; initStringBuffer(&sb); - append(&sb, "insert into %_content (rowid, "); + append(&sb, "insert into %_content (docid, "); appendList(&sb, v->nColumn, v->azContentColumn); append(&sb, ") values (?"); for(i=0; inColumn; ++i) @@ -1879,9 +1880,21 @@ static const char *contentInsertStatement(fulltext_vtab *v){ return stringBufferData(&sb); } +/* Return a dynamically generated statement of the form + * select from %_content where docid = ? + */ +static const char *contentSelectStatement(fulltext_vtab *v){ + StringBuffer sb; + initStringBuffer(&sb); + append(&sb, "SELECT "); + appendList(&sb, v->nColumn, v->azContentColumn); + append(&sb, " FROM %_content WHERE docid = ?"); + return stringBufferData(&sb); +} + /* Return a dynamically generated statement of the form * update %_content set [col_0] = ?, [col_1] = ?, ... - * where rowid = ? + * where docid = ? */ static const char *contentUpdateStatement(fulltext_vtab *v){ StringBuffer sb; @@ -1896,7 +1909,7 @@ static const char *contentUpdateStatement(fulltext_vtab *v){ append(&sb, v->azContentColumn[i]); append(&sb, " = ?"); } - append(&sb, " where rowid = ?"); + append(&sb, " where docid = ?"); return stringBufferData(&sb); } @@ -1913,6 +1926,8 @@ static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, switch( iStmt ){ case CONTENT_INSERT_STMT: zStmt = contentInsertStatement(v); break; + case CONTENT_SELECT_STMT: + zStmt = contentSelectStatement(v); break; case CONTENT_UPDATE_STMT: zStmt = contentUpdateStatement(v); break; default: @@ -2806,6 +2821,7 @@ static int fulltextCreate(sqlite3 *db, void *pAux, initStringBuffer(&schema); append(&schema, "CREATE TABLE %_content("); + append(&schema, " docid INTEGER PRIMARY KEY,"); appendList(&schema, spec.nColumn, spec.azContentColumn); append(&schema, ")"); rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema)); @@ -2813,7 +2829,11 @@ static int fulltextCreate(sqlite3 *db, void *pAux, if( rc!=SQLITE_OK ) goto out; rc = sql_exec(db, spec.zDb, spec.zName, - "create table %_segments(block blob);"); + "create table %_segments(" + " blockid INTEGER PRIMARY KEY," + " block blob" + ");" + ); if( rc!=SQLITE_OK ) goto out; rc = sql_exec(db, spec.zDb, spec.zName, @@ -3662,15 +3682,18 @@ static int fulltextFilter( fulltext_cursor *c = (fulltext_cursor *) pCursor; fulltext_vtab *v = cursor_vtab(c); int rc; - char *zSql; + StringBuffer sb; TRACE(("FTS3 Filter %p\n",pCursor)); - zSql = sqlite3_mprintf("select rowid, * from %%_content %s", - idxNum==QUERY_GENERIC ? "" : "where rowid=?"); + initStringBuffer(&sb); + append(&sb, "SELECT docid, "); + appendList(&sb, v->nColumn, v->azContentColumn); + append(&sb, " FROM %_content"); + if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?"); sqlite3_finalize(c->pStmt); - rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql); - sqlite3_free(zSql); + rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, stringBufferData(&sb)); + stringBufferDestroy(&sb); if( rc!=SQLITE_OK ) return rc; c->iCursorType = idxNum; diff --git a/manifest b/manifest index c2b64c433d..b66492160a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\swarm-body\stests\sfor\srollback\sjournal\si/o\senhancements.\s(CVS\s4279) -D 2007-08-23T16:27:21 +C Fix\sfts3\sto\snot\shave\sthe\sVACUUM\sbug\sfrom\sfts2.\s\s%_content.docid\sis\san\nalias\sto\sfix\sthe\srowid\sfor\sdocuments,\s%_segments.blockid\sis\san\salias\nto\sfix\sthe\srowid\sfor\ssegment\sblocks.\s\sUnit\stest\sfor\sthe\sproblem.\s(CVS\s4280) +D 2007-08-23T20:23:37 F Makefile.in 0c0e53720f658c7a551046442dd7afba0b72bfbe F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028 @@ -49,7 +49,7 @@ F ext/fts2/fts2_tokenizer1.c 8a545c232bdffafd117c4eeaf59789691909f26a F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.tokenizers a97c9a55b3422f6cb04af9de9296fe2447ea4a78 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 64ebaf649e779ef22cade60e973090e0a924746a +F ext/fts3/fts3.c 2098c9b08503b70d4f0c60c5a4665d7413c31a97 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3_hash.c 84654768178452b00bbc986dd878a8299dc1e3dc F ext/fts3/fts3_hash.h af585d6867d478fc0457f64cfaae60e09541e63a @@ -296,6 +296,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test 2da4df52fe8ea8389f6fa7a01e4c1a0f091118d6 F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9 F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa +F test/fts3b.test 94cd8a2fb709c99c1617df01f6908de77892d8bc F test/func.test 605989453d1b42cec1d05c17aa232dc98e3e04e6 F test/fuzz.test 62fc19dd36a427777fd671b569df07166548628a F test/fuzz2.test ea38692ce2da99ad79fe0be5eb1a452c1c4d37bb @@ -560,7 +561,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5 -P 595568492e63822caed5b6970542dcee4615dc4d -R 481110817a2ed016cc610c9d83519f44 -U danielk1977 -Z bffb4d15c81e758cd2ec1b776561f96d +P ff3770f855c1dd75025b1f2496f8c75e9f17ee44 +R d6799fcfef1763dfc3645c2d81885228 +U shess +Z 2f4ef0343bc2683827077b02b21cd810 diff --git a/manifest.uuid b/manifest.uuid index ed827a6813..a6cc1966ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff3770f855c1dd75025b1f2496f8c75e9f17ee44 \ No newline at end of file +6eb2d74a8cfce322930f05c97d4ec255f3711efb \ No newline at end of file diff --git a/test/fts3b.test b/test/fts3b.test new file mode 100644 index 0000000000..9f1c18e4a3 --- /dev/null +++ b/test/fts3b.test @@ -0,0 +1,106 @@ +# 2007 August 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. This +# script tests for the fts2 rowid-versus-vacuum problem (ticket #2566). +# +# $Id: fts3b.test,v 1.1 2007/08/23 20:23:37 shess Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If SQLITE_ENABLE_FTS3 is not defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +db eval { + CREATE VIRTUAL TABLE t1 USING fts3(c); + INSERT INTO t1 (c) VALUES('this is a test'); + INSERT INTO t1 (c) VALUES('that was a test'); + INSERT INTO t1 (c) VALUES('this is fun'); + DELETE FROM t1 WHERE c = 'that was a test'; +} + +# Baseline test. +do_test fts3b-1.1 { + execsql { + SELECT rowid FROM t1 WHERE c MATCH 'this'; + } +} {1 3} + +db eval {VACUUM} + +# The VACUUM renumbered the t1_content table in fts2, which breaks +# this. +do_test fts3b-1.2 { + execsql { + SELECT rowid FROM t1 WHERE c MATCH 'this'; + } +} {1 3} + +# The t2 table is unfortunately pretty contrived. We need documents +# that are bigger than ROOT_MAX (1024) to force segments out of the +# segdir and into %_segments. We also need to force segment merging +# to generate a hole in the %_segments table, which needs more than 16 +# docs. Beyond that, to test correct operation of BLOCK_SELECT_STMT, +# we need to merge a mult-level tree, which is where the 10,000 comes +# from. Which is slow, thus the set of transactions, with the 500 +# being a number such that 10,000/500 > 16. +set text { + Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Maecenas + iaculis mollis ipsum. Praesent rhoncus placerat justo. Duis non quam + sed turpis posuere placerat. Curabitur et lorem in lorem porttitor + aliquet. Pellentesque bibendum tincidunt diam. Vestibulum blandit + ante nec elit. In sapien diam, facilisis eget, dictum sed, viverra + at, felis. Vestibulum magna. Sed magna dolor, vestibulum rhoncus, + ornare vel, vulputate sit amet, felis. Integer malesuada, tellus at + luctus gravida, diam nunc porta nibh, nec imperdiet massa metus eu + lectus. Aliquam nisi. Nunc fringilla nulla at lectus. Suspendisse + potenti. Cum sociis natoque penatibus et magnis dis parturient + montes, nascetur ridiculus mus. Pellentesque odio nulla, feugiat eu, + suscipit nec, consequat quis, risus. +} +append text $text + +db eval {CREATE VIRTUAL TABLE t2 USING fts3(c)} +set res {} +db eval {BEGIN} +for {set ii 0} {$ii<10000} {incr ii} { + db eval {INSERT INTO t2 (c) VALUES ($text)} + lappend res [expr {$ii+1}] + if {($ii%500)==0} { + db eval { + COMMIT; + BEGIN; + } + } +} +db eval {COMMIT} + +do_test fts3b-2.1 { + execsql { + SELECT rowid FROM t2 WHERE c MATCH 'lorem'; + } +} $res + +db eval {VACUUM} + +# The VACUUM renumbered the t2_segment table in fts2, which would +# break the following. +do_test fts3b-2.2 { + execsql { + SELECT rowid FROM t2 WHERE c MATCH 'lorem'; + } +} $res + +finish_test