mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cherrypick the [3475092cff] fix for 32-bit overflow with large language-ids

Browse Source 
into trunk.

FossilOrigin-Name: 2755edc7f1186ba8955f2c3550a90f40a55ddae2
This commit is contained in:
drh 2012-03-16 16:52:54 +00:00
parent 7a48480d07
commit c8d7b24e93
4 changed files with 152 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
ext/fts3/fts3_write.c
View File

@ -434,16 +434,42 @@ int sqlite3Fts3ReadLock(Fts3Table *p){
return rc;
}
/*
** FTS maintains a separate indexes for each language-id (a 32-bit integer).
** Within each language id, a separate index is maintained to store the
** document terms, and each configured prefix size (configured the FTS
** "prefix=" option). And each index consists of multiple levels ("relative
** levels").
**
** All three of these values (the language id, the specific index and the
** level within the index) are encoded in 64-bit integer values stored
** in the %_segdir table on disk. This function is used to convert three
** separate component values into the single 64-bit integer value that
** can be used to query the %_segdir table.
**
** Specifically, each language-id/index combination is allocated 1024
** 64-bit integer level values ("absolute levels"). The main terms index
** for language-id 0 is allocate values 0-1023. The first prefix index
** (if any) for language-id 0 is allocated values 1024-2047. And so on.
** Language 1 indexes are allocated immediately following language 0.
**
** So, for a system with nPrefix prefix indexes configured, the block of
** absolute levels that corresponds to language-id iLangid and index
** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024).
*/
static sqlite3_int64 getAbsoluteLevel(
Fts3Table *p,
int iLangid,
int iIndex,
int iLevel
){
sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */
assert( iLangid>=0 );
assert( p->nIndex>0 );
assert( iIndex>=0 && iIndex<p->nIndex );
return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel;
iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
return iBase + iLevel;
}
@ -468,7 +494,7 @@ int sqlite3Fts3AllSegdirs(
Fts3Table *p, /* FTS3 table */
int iLangid, /* Language being queried */
int iIndex, /* Index for p->aIndex[] */
int iLevel, /* Level to select */
int iLevel, /* Level to select (relative level) */
sqlite3_stmt **ppStmt /* OUT: Compiled statement */
){
int rc;
@ -483,7 +509,7 @@ int sqlite3Fts3AllSegdirs(
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
sqlite3_bind_int(pStmt, 2,
sqlite3_bind_int64(pStmt, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
}
@ -491,7 +517,7 @@ int sqlite3Fts3AllSegdirs(
/* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
}
}
*ppStmt = pStmt;
@ -1763,7 +1789,7 @@ static int fts3WriteSegment(
*/
static int fts3WriteSegdir(
Fts3Table *p, /* Virtual table handle */
int iLevel, /* Value for "level" field */
sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */
int iIdx, /* Value for "idx" field */
sqlite3_int64 iStartBlock, /* Value for "start_block" field */
sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
@ -1774,7 +1800,7 @@ static int fts3WriteSegdir(
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pStmt, 1, iLevel);
sqlite3_bind_int64(pStmt, 1, iLevel);
sqlite3_bind_int(pStmt, 2, iIdx);
sqlite3_bind_int64(pStmt, 3, iStartBlock);
sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
@ -2157,7 +2183,7 @@ static int fts3SegWriterAdd(
static int fts3SegWriterFlush(
Fts3Table *p, /* Virtual table handle */
SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
int iLevel, /* Value for 'level' column of %_segdir */
sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */
int iIdx /* Value for 'idx' column of %_segdir */
){
int rc; /* Return code */
@ -2239,7 +2265,7 @@ static int fts3SegmentMaxLevel(
Fts3Table *p,
int iLangid,
int iIndex,
int *pnMax
sqlite3_int64 *pnMax
){
sqlite3_stmt *pStmt;
int rc;
@ -2253,12 +2279,12 @@ static int fts3SegmentMaxLevel(
*/
rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
sqlite3_bind_int(pStmt, 2,
sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
sqlite3_bind_int64(pStmt, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
*pnMax = sqlite3_column_int(pStmt, 0);
*pnMax = sqlite3_column_int64(pStmt, 0);
}
return sqlite3_reset(pStmt);
}
@ -2307,15 +2333,17 @@ static int fts3DeleteSegdir(
if( iLevel==FTS3_SEGCURSOR_ALL ){
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
sqlite3_bind_int(pDelete, 2,
sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
sqlite3_bind_int64(pDelete, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
}
}else{
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
sqlite3_bind_int64(
pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
);
}
}
@ -2792,7 +2820,7 @@ static int fts3SegmentMerge(
){
int rc; /* Return code */
int iIdx = 0; /* Index of new segment */
int iNewLevel = 0; /* Level/index to create new segment at */
sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
Fts3SegFilter filter; /* Segment term filter condition */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */

14
manifest
View File

@ -1,5 +1,5 @@
C Fix\sharmless\scompiler\swarnings.
D 2012-03-16T00:28:11.282
C Cherrypick\sthe\s[3475092cff]\sfix\sfor\s32-bit\soverflow\swith\slarge\slanguage-ids\ninto\strunk.
D 2012-03-16T16:52:54.032
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4
F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073
F ext/fts3/fts3_write.c 8eedfeb2e61114cfd94fd1d917daf3658203332c
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a
F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
P fed134a5f395533de0c9d76950697c060ddb126d
R 9a71f3ab5326ee35b9bbd87536e2a1d5
P 74eadeec34c4b19cf5f8b7f648db3b7ad601a00e
R da59c6040c440e1d087c29c77d94171a
U drh
Z 60cd8a049ba54606f3a162ca33790b57
Z 86f53efe49c877ef67da98d4e974121b

2
manifest.uuid
View File

@ -1 +1 @@
74eadeec34c4b19cf5f8b7f648db3b7ad601a00e
2755edc7f1186ba8955f2c3550a90f40a55ddae2

101
test/fts4langid.test
View File

@ -382,4 +382,105 @@ do_catchsql_test 4.1.5 {
INSERT INTO t4(content, lid) VALUES('hello world', 101)
} {1 {SQL logic error or missing database}}
#-------------------------------------------------------------------------
# Test cases 5.*
#
# The following test cases are designed to detect a 32-bit overflow bug
# that existed at one point.
#
proc build_multilingual_db_3 {db} {
$db eval {
CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid);
}
set languages [list 0 1 2 [expr 1<<30]]
foreach lid $languages {
execsql {
INSERT INTO t5(docid, content, lid) VALUES(
$lid, 'My language is ' || $lid, $lid
)
}
}
}
do_test 5.1.0 {
reset_db
build_multilingual_db_3 db
} {}
do_execsql_test 5.1.1 {
SELECT level FROM t5_segdir;
} [list 0 1024 2048 [expr 1<<40]]
do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0
foreach langid [list 0 1 2 [expr 1<<30]] {
do_execsql_test 5.2.$langid {
SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid
} $langid
}
set lid [expr 1<<30]
do_execsql_test 5.3.1 {
CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid);
INSERT INTO t6 VALUES('I belong to language 0!');
}
do_test 5.3.2 {
for {set i 0} {$i < 20} {incr i} {
execsql {
INSERT INTO t6(content, lid) VALUES(
'I (row '||$i||') belong to langauge N!', $lid
);
}
}
execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' }
} {1}
do_test 5.3.3 {
execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid}
} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {}
do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1}
do_execsql_test 5.3.6 {
SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid
} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
set lid [expr 1<<30]
foreach lid [list 4 [expr 1<<30]] {
do_execsql_test 5.4.$lid.1 {
DELETE FROM t6;
SELECT count(*) FROM t6_segdir;
SELECT count(*) FROM t6_segments;
} {0 0}
do_execsql_test 5.4.$lid.2 {
INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid);
INSERT INTO t6(content, lid) VALUES('zero zero one', $lid);
INSERT INTO t6(content, lid) VALUES('zero one zero', $lid);
INSERT INTO t6(content, lid) VALUES('zero one one', $lid);
INSERT INTO t6(content, lid) VALUES('one zero zero', $lid);
INSERT INTO t6(content, lid) VALUES('one zero one', $lid);
INSERT INTO t6(content, lid) VALUES('one one zero', $lid);
INSERT INTO t6(content, lid) VALUES('one one one', $lid);
SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
} {1 2 5}
do_execsql_test 5.4.$lid.3 {
SELECT count(*) FROM t6_segdir;
SELECT count(*) FROM t6_segments;
} {8 0}
do_execsql_test 5.4.$lid.4 {
INSERT INTO t6(t6) VALUES('optimize');
SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
} {1 2 5}
do_execsql_test 5.4.$lid.5 {
SELECT count(*) FROM t6_segdir;
SELECT count(*) FROM t6_segments;
} {1 0}
}
finish_test