From c8d7b24e939bb0fb0ddedbffb7d867b0d7af444e Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Fri, 16 Mar 2012 16:52:54 +0000
Subject: [PATCH] Cherrypick the [3475092cff] fix for 32-bit overflow with
large language-ids into trunk.
FossilOrigin-Name: 2755edc7f1186ba8955f2c3550a90f40a55ddae2
---
ext/fts3/fts3_write.c | 58 +++++++++++++++++-------
manifest | 14 +++---
manifest.uuid | 2 +-
test/fts4langid.test | 101 ++++++++++++++++++++++++++++++++++++++++++
4 files changed, 152 insertions(+), 23 deletions(-)
diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c
index 8f97c8be98..bc20dfa3f3 100644
--- a/ext/fts3/fts3_write.c
+++ b/ext/fts3/fts3_write.c
@@ -434,16 +434,42 @@ int sqlite3Fts3ReadLock(Fts3Table *p){
return rc;
}
+/*
+** FTS maintains a separate indexes for each language-id (a 32-bit integer).
+** Within each language id, a separate index is maintained to store the
+** document terms, and each configured prefix size (configured the FTS
+** "prefix=" option). And each index consists of multiple levels ("relative
+** levels").
+**
+** All three of these values (the language id, the specific index and the
+** level within the index) are encoded in 64-bit integer values stored
+** in the %_segdir table on disk. This function is used to convert three
+** separate component values into the single 64-bit integer value that
+** can be used to query the %_segdir table.
+**
+** Specifically, each language-id/index combination is allocated 1024
+** 64-bit integer level values ("absolute levels"). The main terms index
+** for language-id 0 is allocate values 0-1023. The first prefix index
+** (if any) for language-id 0 is allocated values 1024-2047. And so on.
+** Language 1 indexes are allocated immediately following language 0.
+**
+** So, for a system with nPrefix prefix indexes configured, the block of
+** absolute levels that corresponds to language-id iLangid and index
+** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024).
+*/
static sqlite3_int64 getAbsoluteLevel(
Fts3Table *p,
int iLangid,
int iIndex,
int iLevel
){
+ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */
assert( iLangid>=0 );
assert( p->nIndex>0 );
assert( iIndex>=0 && iIndex<p->nIndex );
- return (iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL + iLevel;
+
+ iBase = ((sqlite3_int64)iLangid * p->nIndex + iIndex) * FTS3_SEGDIR_MAXLEVEL;
+ return iBase + iLevel;
}
@@ -468,7 +494,7 @@ int sqlite3Fts3AllSegdirs(
Fts3Table *p, /* FTS3 table */
int iLangid, /* Language being queried */
int iIndex, /* Index for p->aIndex[] */
- int iLevel, /* Level to select */
+ int iLevel, /* Level to select (relative level) */
sqlite3_stmt **ppStmt /* OUT: Compiled statement */
){
int rc;
@@ -483,7 +509,7 @@ int sqlite3Fts3AllSegdirs(
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0);
if( rc==SQLITE_OK ){
sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
- sqlite3_bind_int(pStmt, 2,
+ sqlite3_bind_int64(pStmt, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
}
@@ -491,7 +517,7 @@ int sqlite3Fts3AllSegdirs(
/* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */
rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel));
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
}
}
*ppStmt = pStmt;
@@ -1763,7 +1789,7 @@ static int fts3WriteSegment(
*/
static int fts3WriteSegdir(
Fts3Table *p, /* Virtual table handle */
- int iLevel, /* Value for "level" field */
+ sqlite3_int64 iLevel, /* Value for "level" field (absolute level) */
int iIdx, /* Value for "idx" field */
sqlite3_int64 iStartBlock, /* Value for "start_block" field */
sqlite3_int64 iLeafEndBlock, /* Value for "leaves_end_block" field */
@@ -1774,7 +1800,7 @@ static int fts3WriteSegdir(
sqlite3_stmt *pStmt;
int rc = fts3SqlStmt(p, SQL_INSERT_SEGDIR, &pStmt, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pStmt, 1, iLevel);
+ sqlite3_bind_int64(pStmt, 1, iLevel);
sqlite3_bind_int(pStmt, 2, iIdx);
sqlite3_bind_int64(pStmt, 3, iStartBlock);
sqlite3_bind_int64(pStmt, 4, iLeafEndBlock);
@@ -2157,7 +2183,7 @@ static int fts3SegWriterAdd(
static int fts3SegWriterFlush(
Fts3Table *p, /* Virtual table handle */
SegmentWriter *pWriter, /* SegmentWriter to flush to the db */
- int iLevel, /* Value for 'level' column of %_segdir */
+ sqlite3_int64 iLevel, /* Value for 'level' column of %_segdir */
int iIdx /* Value for 'idx' column of %_segdir */
){
int rc; /* Return code */
@@ -2239,7 +2265,7 @@ static int fts3SegmentMaxLevel(
Fts3Table *p,
int iLangid,
int iIndex,
- int *pnMax
+ sqlite3_int64 *pnMax
){
sqlite3_stmt *pStmt;
int rc;
@@ -2253,12 +2279,12 @@ static int fts3SegmentMaxLevel(
*/
rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0);
if( rc!=SQLITE_OK ) return rc;
- sqlite3_bind_int(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
- sqlite3_bind_int(pStmt, 2,
+ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pStmt, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
if( SQLITE_ROW==sqlite3_step(pStmt) ){
- *pnMax = sqlite3_column_int(pStmt, 0);
+ *pnMax = sqlite3_column_int64(pStmt, 0);
}
return sqlite3_reset(pStmt);
}
@@ -2307,15 +2333,17 @@ static int fts3DeleteSegdir(
if( iLevel==FTS3_SEGCURSOR_ALL ){
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
- sqlite3_bind_int(pDelete, 2,
+ sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0));
+ sqlite3_bind_int64(pDelete, 2,
getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1)
);
}
}else{
rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0);
if( rc==SQLITE_OK ){
- sqlite3_bind_int(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel));
+ sqlite3_bind_int64(
+ pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, iLevel)
+ );
}
}
@@ -2792,7 +2820,7 @@ static int fts3SegmentMerge(
){
int rc; /* Return code */
int iIdx = 0; /* Index of new segment */
- int iNewLevel = 0; /* Level/index to create new segment at */
+ sqlite3_int64 iNewLevel = 0; /* Level/index to create new segment at */
SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */
Fts3SegFilter filter; /* Segment term filter condition */
Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */
diff --git a/manifest b/manifest
index 839b111215..4f001824a3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Fix\sharmless\scompiler\swarnings.
-D 2012-03-16T00:28:11.282
+C Cherrypick\sthe\s[3475092cff]\sfix\sfor\s32-bit\soverflow\swith\slarge\slanguage-ids\ninto\strunk.
+D 2012-03-16T16:52:54.032
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -78,7 +78,7 @@ F ext/fts3/fts3_test.c 6b7cc68aef4efb084e1449f7d20c4b20d3bdf6b4
F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
-F ext/fts3/fts3_write.c f87bb2d27d31cb7a7bf306747079095393c9d073
+F ext/fts3/fts3_write.c 8eedfeb2e61114cfd94fd1d917daf3658203332c
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -496,7 +496,7 @@ F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
-F test/fts4langid.test fabdd5a8db0fa00292e0704809f566e3fb6dba3a
+F test/fts4langid.test 2081c357bb6f170f34ef8e08c6abb88002b95c69
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a
@@ -992,7 +992,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings-clang.sh 9f406d66e750e8ac031c63a9ef3248aaa347ef2a
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P fed134a5f395533de0c9d76950697c060ddb126d
-R 9a71f3ab5326ee35b9bbd87536e2a1d5
+P 74eadeec34c4b19cf5f8b7f648db3b7ad601a00e
+R da59c6040c440e1d087c29c77d94171a
U drh
-Z 60cd8a049ba54606f3a162ca33790b57
+Z 86f53efe49c877ef67da98d4e974121b
diff --git a/manifest.uuid b/manifest.uuid
index 9a63bfc3a0..3b6c5ce27f 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-74eadeec34c4b19cf5f8b7f648db3b7ad601a00e
\ No newline at end of file
+2755edc7f1186ba8955f2c3550a90f40a55ddae2
\ No newline at end of file
diff --git a/test/fts4langid.test b/test/fts4langid.test
index 08f1a21c17..8ad06766b7 100644
--- a/test/fts4langid.test
+++ b/test/fts4langid.test
@@ -382,4 +382,105 @@ do_catchsql_test 4.1.5 {
INSERT INTO t4(content, lid) VALUES('hello world', 101)
} {1 {SQL logic error or missing database}}
+#-------------------------------------------------------------------------
+# Test cases 5.*
+#
+# The following test cases are designed to detect a 32-bit overflow bug
+# that existed at one point.
+#
+proc build_multilingual_db_3 {db} {
+ $db eval {
+ CREATE VIRTUAL TABLE t5 USING fts4(languageid=lid);
+ }
+ set languages [list 0 1 2 [expr 1<<30]]
+
+ foreach lid $languages {
+ execsql {
+ INSERT INTO t5(docid, content, lid) VALUES(
+ $lid, 'My language is ' || $lid, $lid
+ )
+ }
+ }
+}
+
+do_test 5.1.0 {
+ reset_db
+ build_multilingual_db_3 db
+} {}
+
+do_execsql_test 5.1.1 {
+ SELECT level FROM t5_segdir;
+} [list 0 1024 2048 [expr 1<<40]]
+
+do_execsql_test 5.1.2 {SELECT docid FROM t5 WHERE t5 MATCH 'language'} 0
+foreach langid [list 0 1 2 [expr 1<<30]] {
+ do_execsql_test 5.2.$langid {
+ SELECT docid FROM t5 WHERE t5 MATCH 'language' AND lid = $langid
+ } $langid
+}
+
+set lid [expr 1<<30]
+do_execsql_test 5.3.1 {
+ CREATE VIRTUAL TABLE t6 USING fts4(languageid=lid);
+ INSERT INTO t6 VALUES('I belong to language 0!');
+}
+do_test 5.3.2 {
+ for {set i 0} {$i < 20} {incr i} {
+ execsql {
+ INSERT INTO t6(content, lid) VALUES(
+ 'I (row '||$i||') belong to langauge N!', $lid
+ );
+ }
+ }
+ execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' }
+} {1}
+
+do_test 5.3.3 {
+ execsql { SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid}
+} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
+
+do_execsql_test 5.3.4 { INSERT INTO t6(t6) VALUES('optimize') } {}
+do_execsql_test 5.3.5 { SELECT docid FROM t6 WHERE t6 MATCH 'belong' } {1}
+do_execsql_test 5.3.6 {
+ SELECT docid FROM t6 WHERE t6 MATCH 'belong' AND lid=$lid
+} {2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21}
+
+
+set lid [expr 1<<30]
+foreach lid [list 4 [expr 1<<30]] {
+ do_execsql_test 5.4.$lid.1 {
+ DELETE FROM t6;
+ SELECT count(*) FROM t6_segdir;
+ SELECT count(*) FROM t6_segments;
+ } {0 0}
+ do_execsql_test 5.4.$lid.2 {
+ INSERT INTO t6(content, lid) VALUES('zero zero zero', $lid);
+ INSERT INTO t6(content, lid) VALUES('zero zero one', $lid);
+ INSERT INTO t6(content, lid) VALUES('zero one zero', $lid);
+ INSERT INTO t6(content, lid) VALUES('zero one one', $lid);
+ INSERT INTO t6(content, lid) VALUES('one zero zero', $lid);
+ INSERT INTO t6(content, lid) VALUES('one zero one', $lid);
+ INSERT INTO t6(content, lid) VALUES('one one zero', $lid);
+ INSERT INTO t6(content, lid) VALUES('one one one', $lid);
+
+ SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
+ } {1 2 5}
+
+ do_execsql_test 5.4.$lid.3 {
+ SELECT count(*) FROM t6_segdir;
+ SELECT count(*) FROM t6_segments;
+ } {8 0}
+
+ do_execsql_test 5.4.$lid.4 {
+ INSERT INTO t6(t6) VALUES('optimize');
+ SELECT docid FROM t6 WHERE t6 MATCH '"zero zero"' AND lid=$lid;
+ } {1 2 5}
+
+ do_execsql_test 5.4.$lid.5 {
+ SELECT count(*) FROM t6_segdir;
+ SELECT count(*) FROM t6_segments;
+ } {1 0}
+}
+
+
finish_test