Fix fts3 to not have the VACUUM bug from fts2. %_content.docid is an
alias to fix the rowid for documents, %_segments.blockid is an alias to fix the rowid for segment blocks. Unit test for the problem. (CVS 4280) FossilOrigin-Name: 6eb2d74a8cfce322930f05c97d4ec255f3711efb
This commit is contained in:
shess
parent
6897ca30b5
commit
6beeb0329a
@ -1772,13 +1772,14 @@ typedef enum fulltext_statement {
|
||||
*/
|
||||
static const char *const fulltext_zStatement[MAX_STMT] = {
|
||||
/* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */
|
||||
/* CONTENT_SELECT */ "select * from %_content where rowid = ?",
|
||||
/* CONTENT_SELECT */ NULL, /* generated in contentSelectStatement() */
|
||||
/* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */
|
||||
/* CONTENT_DELETE */ "delete from %_content where rowid = ?",
|
||||
/* CONTENT_DELETE */ "delete from %_content where docid = ?",
|
||||
|
||||
/* BLOCK_INSERT */ "insert into %_segments values (?)",
|
||||
/* BLOCK_SELECT */ "select block from %_segments where rowid = ?",
|
||||
/* BLOCK_DELETE */ "delete from %_segments where rowid between ? and ?",
|
||||
/* BLOCK_INSERT */
|
||||
"insert into %_segments (blockid, block) values (null, ?)",
|
||||
/* BLOCK_SELECT */ "select block from %_segments where blockid = ?",
|
||||
/* BLOCK_DELETE */ "delete from %_segments where blockid between ? and ?",
|
||||
|
||||
/* SEGDIR_MAX_INDEX */ "select max(idx) from %_segdir where level = ?",
|
||||
/* SEGDIR_SET */ "insert into %_segdir values (?, ?, ?, ?, ?, ?)",
|
||||
@ -1821,7 +1822,7 @@ struct fulltext_vtab {
|
||||
sqlite3_stmt *pLeafSelectStmts[MERGE_COUNT];
|
||||
/* The statement used to prepare pLeafSelectStmts. */
|
||||
#define LEAF_SELECT \
|
||||
"select block from %_segments where rowid between ? and ? order by rowid"
|
||||
"select block from %_segments where blockid between ? and ? order by blockid"
|
||||
|
||||
/* These buffer pending index updates during transactions.
|
||||
** nPendingData estimates the memory size of the pending data. It
|
||||
@ -1863,14 +1864,14 @@ static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){
|
||||
static const sqlite3_module fts3Module; /* forward declaration */
|
||||
|
||||
/* Return a dynamically generated statement of the form
|
||||
* insert into %_content (rowid, ...) values (?, ...)
|
||||
* insert into %_content (docid, ...) values (?, ...)
|
||||
*/
|
||||
static const char *contentInsertStatement(fulltext_vtab *v){
|
||||
StringBuffer sb;
|
||||
int i;
|
||||
|
||||
initStringBuffer(&sb);
|
||||
append(&sb, "insert into %_content (rowid, ");
|
||||
append(&sb, "insert into %_content (docid, ");
|
||||
appendList(&sb, v->nColumn, v->azContentColumn);
|
||||
append(&sb, ") values (?");
|
||||
for(i=0; i<v->nColumn; ++i)
|
||||
@ -1879,9 +1880,21 @@ static const char *contentInsertStatement(fulltext_vtab *v){
|
||||
return stringBufferData(&sb);
|
||||
}
|
||||
|
||||
/* Return a dynamically generated statement of the form
|
||||
* select <content columns> from %_content where docid = ?
|
||||
*/
|
||||
static const char *contentSelectStatement(fulltext_vtab *v){
|
||||
StringBuffer sb;
|
||||
initStringBuffer(&sb);
|
||||
append(&sb, "SELECT ");
|
||||
appendList(&sb, v->nColumn, v->azContentColumn);
|
||||
append(&sb, " FROM %_content WHERE docid = ?");
|
||||
return stringBufferData(&sb);
|
||||
}
|
||||
|
||||
/* Return a dynamically generated statement of the form
|
||||
* update %_content set [col_0] = ?, [col_1] = ?, ...
|
||||
* where rowid = ?
|
||||
* where docid = ?
|
||||
*/
|
||||
static const char *contentUpdateStatement(fulltext_vtab *v){
|
||||
StringBuffer sb;
|
||||
@ -1896,7 +1909,7 @@ static const char *contentUpdateStatement(fulltext_vtab *v){
|
||||
append(&sb, v->azContentColumn[i]);
|
||||
append(&sb, " = ?");
|
||||
}
|
||||
append(&sb, " where rowid = ?");
|
||||
append(&sb, " where docid = ?");
|
||||
return stringBufferData(&sb);
|
||||
}
|
||||
|
||||
@ -1913,6 +1926,8 @@ static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt,
|
||||
switch( iStmt ){
|
||||
case CONTENT_INSERT_STMT:
|
||||
zStmt = contentInsertStatement(v); break;
|
||||
case CONTENT_SELECT_STMT:
|
||||
zStmt = contentSelectStatement(v); break;
|
||||
case CONTENT_UPDATE_STMT:
|
||||
zStmt = contentUpdateStatement(v); break;
|
||||
default:
|
||||
@ -2806,6 +2821,7 @@ static int fulltextCreate(sqlite3 *db, void *pAux,
|
||||
|
||||
initStringBuffer(&schema);
|
||||
append(&schema, "CREATE TABLE %_content(");
|
||||
append(&schema, " docid INTEGER PRIMARY KEY,");
|
||||
appendList(&schema, spec.nColumn, spec.azContentColumn);
|
||||
append(&schema, ")");
|
||||
rc = sql_exec(db, spec.zDb, spec.zName, stringBufferData(&schema));
|
||||
@ -2813,7 +2829,11 @@ static int fulltextCreate(sqlite3 *db, void *pAux,
|
||||
if( rc!=SQLITE_OK ) goto out;
|
||||
|
||||
rc = sql_exec(db, spec.zDb, spec.zName,
|
||||
"create table %_segments(block blob);");
|
||||
"create table %_segments("
|
||||
" blockid INTEGER PRIMARY KEY,"
|
||||
" block blob"
|
||||
");"
|
||||
);
|
||||
if( rc!=SQLITE_OK ) goto out;
|
||||
|
||||
rc = sql_exec(db, spec.zDb, spec.zName,
|
||||
@ -3662,15 +3682,18 @@ static int fulltextFilter(
|
||||
fulltext_cursor *c = (fulltext_cursor *) pCursor;
|
||||
fulltext_vtab *v = cursor_vtab(c);
|
||||
int rc;
|
||||
char *zSql;
|
||||
StringBuffer sb;
|
||||
|
||||
TRACE(("FTS3 Filter %p\n",pCursor));
|
||||
|
||||
zSql = sqlite3_mprintf("select rowid, * from %%_content %s",
|
||||
idxNum==QUERY_GENERIC ? "" : "where rowid=?");
|
||||
initStringBuffer(&sb);
|
||||
append(&sb, "SELECT docid, ");
|
||||
appendList(&sb, v->nColumn, v->azContentColumn);
|
||||
append(&sb, " FROM %_content");
|
||||
if( idxNum!=QUERY_GENERIC ) append(&sb, " WHERE docid = ?");
|
||||
sqlite3_finalize(c->pStmt);
|
||||
rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql);
|
||||
sqlite3_free(zSql);
|
||||
rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, stringBufferData(&sb));
|
||||
stringBufferDestroy(&sb);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
|
||||
c->iCursorType = idxNum;
|
||||
|
||||
15
manifest
15
manifest
@ -1,5 +1,5 @@
|
||||
C Add\ssome\swarm-body\stests\sfor\srollback\sjournal\si/o\senhancements.\s(CVS\s4279)
|
||||
D 2007-08-23T16:27:21
|
||||
C Fix\sfts3\sto\snot\shave\sthe\sVACUUM\sbug\sfrom\sfts2.\s\s%_content.docid\sis\san\nalias\sto\sfix\sthe\srowid\sfor\sdocuments,\s%_segments.blockid\sis\san\salias\nto\sfix\sthe\srowid\sfor\ssegment\sblocks.\s\sUnit\stest\sfor\sthe\sproblem.\s(CVS\s4280)
|
||||
D 2007-08-23T20:23:37
|
||||
F Makefile.in 0c0e53720f658c7a551046442dd7afba0b72bfbe
|
||||
F Makefile.linux-gcc 65241babba6faf1152bf86574477baab19190499
|
||||
F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
|
||||
@ -49,7 +49,7 @@ F ext/fts2/fts2_tokenizer1.c 8a545c232bdffafd117c4eeaf59789691909f26a
|
||||
F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
|
||||
F ext/fts3/README.tokenizers a97c9a55b3422f6cb04af9de9296fe2447ea4a78
|
||||
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
|
||||
F ext/fts3/fts3.c 64ebaf649e779ef22cade60e973090e0a924746a
|
||||
F ext/fts3/fts3.c 2098c9b08503b70d4f0c60c5a4665d7413c31a97
|
||||
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
|
||||
F ext/fts3/fts3_hash.c 84654768178452b00bbc986dd878a8299dc1e3dc
|
||||
F ext/fts3/fts3_hash.h af585d6867d478fc0457f64cfaae60e09541e63a
|
||||
@ -296,6 +296,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
|
||||
F test/fts3an.test 2da4df52fe8ea8389f6fa7a01e4c1a0f091118d6
|
||||
F test/fts3ao.test 0aa29dd4fc1c8d46b1f7cfe5926f7ac97551bea9
|
||||
F test/fts3atoken.test 25c2070e1e8755d414bf9c8200427b277a9f99fa
|
||||
F test/fts3b.test 94cd8a2fb709c99c1617df01f6908de77892d8bc
|
||||
F test/func.test 605989453d1b42cec1d05c17aa232dc98e3e04e6
|
||||
F test/fuzz.test 62fc19dd36a427777fd671b569df07166548628a
|
||||
F test/fuzz2.test ea38692ce2da99ad79fe0be5eb1a452c1c4d37bb
|
||||
@ -560,7 +561,7 @@ F www/tclsqlite.tcl 8be95ee6dba05eabcd27a9d91331c803f2ce2130
|
||||
F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
|
||||
F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
|
||||
F www/whentouse.tcl fc46eae081251c3c181bd79c5faef8195d7991a5
|
||||
P 595568492e63822caed5b6970542dcee4615dc4d
|
||||
R 481110817a2ed016cc610c9d83519f44
|
||||
U danielk1977
|
||||
Z bffb4d15c81e758cd2ec1b776561f96d
|
||||
P ff3770f855c1dd75025b1f2496f8c75e9f17ee44
|
||||
R d6799fcfef1763dfc3645c2d81885228
|
||||
U shess
|
||||
Z 2f4ef0343bc2683827077b02b21cd810
|
||||
|
||||
@ -1 +1 @@
|
||||
ff3770f855c1dd75025b1f2496f8c75e9f17ee44
|
||||
6eb2d74a8cfce322930f05c97d4ec255f3711efb
|
||||
106
test/fts3b.test
Normal file
106
test/fts3b.test
Normal file
@ -0,0 +1,106 @@
|
||||
# 2007 August 20
|
||||
#
|
||||
# The author disclaims copyright to this source code. In place of
|
||||
# a legal notice, here is a blessing:
|
||||
#
|
||||
# May you do good and not evil.
|
||||
# May you find forgiveness for yourself and forgive others.
|
||||
# May you share freely, never taking more than you give.
|
||||
#
|
||||
#*************************************************************************
|
||||
# This file implements regression tests for SQLite library. This
|
||||
# script tests for the fts2 rowid-versus-vacuum problem (ticket #2566).
|
||||
#
|
||||
# $Id: fts3b.test,v 1.1 2007/08/23 20:23:37 shess Exp $
|
||||
#
|
||||
|
||||
set testdir [file dirname $argv0]
|
||||
source $testdir/tester.tcl
|
||||
|
||||
# If SQLITE_ENABLE_FTS3 is not defined, omit this file.
|
||||
ifcapable !fts3 {
|
||||
finish_test
|
||||
return
|
||||
}
|
||||
|
||||
db eval {
|
||||
CREATE VIRTUAL TABLE t1 USING fts3(c);
|
||||
INSERT INTO t1 (c) VALUES('this is a test');
|
||||
INSERT INTO t1 (c) VALUES('that was a test');
|
||||
INSERT INTO t1 (c) VALUES('this is fun');
|
||||
DELETE FROM t1 WHERE c = 'that was a test';
|
||||
}
|
||||
|
||||
# Baseline test.
|
||||
do_test fts3b-1.1 {
|
||||
execsql {
|
||||
SELECT rowid FROM t1 WHERE c MATCH 'this';
|
||||
}
|
||||
} {1 3}
|
||||
|
||||
db eval {VACUUM}
|
||||
|
||||
# The VACUUM renumbered the t1_content table in fts2, which breaks
|
||||
# this.
|
||||
do_test fts3b-1.2 {
|
||||
execsql {
|
||||
SELECT rowid FROM t1 WHERE c MATCH 'this';
|
||||
}
|
||||
} {1 3}
|
||||
|
||||
# The t2 table is unfortunately pretty contrived. We need documents
|
||||
# that are bigger than ROOT_MAX (1024) to force segments out of the
|
||||
# segdir and into %_segments. We also need to force segment merging
|
||||
# to generate a hole in the %_segments table, which needs more than 16
|
||||
# docs. Beyond that, to test correct operation of BLOCK_SELECT_STMT,
|
||||
# we need to merge a mult-level tree, which is where the 10,000 comes
|
||||
# from. Which is slow, thus the set of transactions, with the 500
|
||||
# being a number such that 10,000/500 > 16.
|
||||
set text {
|
||||
Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Maecenas
|
||||
iaculis mollis ipsum. Praesent rhoncus placerat justo. Duis non quam
|
||||
sed turpis posuere placerat. Curabitur et lorem in lorem porttitor
|
||||
aliquet. Pellentesque bibendum tincidunt diam. Vestibulum blandit
|
||||
ante nec elit. In sapien diam, facilisis eget, dictum sed, viverra
|
||||
at, felis. Vestibulum magna. Sed magna dolor, vestibulum rhoncus,
|
||||
ornare vel, vulputate sit amet, felis. Integer malesuada, tellus at
|
||||
luctus gravida, diam nunc porta nibh, nec imperdiet massa metus eu
|
||||
lectus. Aliquam nisi. Nunc fringilla nulla at lectus. Suspendisse
|
||||
potenti. Cum sociis natoque penatibus et magnis dis parturient
|
||||
montes, nascetur ridiculus mus. Pellentesque odio nulla, feugiat eu,
|
||||
suscipit nec, consequat quis, risus.
|
||||
}
|
||||
append text $text
|
||||
|
||||
db eval {CREATE VIRTUAL TABLE t2 USING fts3(c)}
|
||||
set res {}
|
||||
db eval {BEGIN}
|
||||
for {set ii 0} {$ii<10000} {incr ii} {
|
||||
db eval {INSERT INTO t2 (c) VALUES ($text)}
|
||||
lappend res [expr {$ii+1}]
|
||||
if {($ii%500)==0} {
|
||||
db eval {
|
||||
COMMIT;
|
||||
BEGIN;
|
||||
}
|
||||
}
|
||||
}
|
||||
db eval {COMMIT}
|
||||
|
||||
do_test fts3b-2.1 {
|
||||
execsql {
|
||||
SELECT rowid FROM t2 WHERE c MATCH 'lorem';
|
||||
}
|
||||
} $res
|
||||
|
||||
db eval {VACUUM}
|
||||
|
||||
# The VACUUM renumbered the t2_segment table in fts2, which would
|
||||
# break the following.
|
||||
do_test fts3b-2.2 {
|
||||
execsql {
|
||||
SELECT rowid FROM t2 WHERE c MATCH 'lorem';
|
||||
}
|
||||
} $res
|
||||
|
||||
finish_test
|
||||
Loading…
x
Reference in New Issue
Block a user