mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

In fts5 integrity checks, do not compare the contents of the index against an external content table unless specifically requested.

Browse Source 
FossilOrigin-Name: 782163693f37aeb65209bebbaeb6659a36881b8c4b4bec778b366658488bf966
This commit is contained in:
dan 2020-09-21 14:53:21 +00:00
parent 9ca41fbfa9
commit 7548ab20e6
7 changed files with 124 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ext/fts5/fts5Int.h
View File

@ -484,7 +484,7 @@ int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
/* /*
** Functions called by the storage module as part of integrity-check. ** Functions called by the storage module as part of integrity-check.
*/ */
int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum);
/* /*
** Called during virtual module initialization to register UDF ** Called during virtual module initialization to register UDF
@ -639,7 +639,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
int sqlite3Fts5StorageIntegrity(Fts5Storage *p); int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**);
void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*);

4
ext/fts5/fts5_index.c
View File

@ -6062,7 +6062,7 @@ static void fts5IndexIntegrityCheckSegment(
** error, or some other SQLite error code if another error (e.g. OOM) ** error, or some other SQLite error code if another error (e.g. OOM)
** occurs. ** occurs.
*/ */
int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){
int eDetail = p->pConfig->eDetail; int eDetail = p->pConfig->eDetail;
u64 cksum2 = 0; /* Checksum based on contents of indexes */ u64 cksum2 = 0; /* Checksum based on contents of indexes */
Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */
@ -6133,7 +6133,7 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){
fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3);
fts5MultiIterFree(pIter); fts5MultiIterFree(pIter);
if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT;
fts5StructureRelease(pStruct); fts5StructureRelease(pStruct);
#ifdef SQLITE_DEBUG #ifdef SQLITE_DEBUG

3
ext/fts5/fts5_main.c
View File

@ -1508,7 +1508,8 @@ static int fts5SpecialInsert(
int nMerge = sqlite3_value_int(pVal); int nMerge = sqlite3_value_int(pVal);
rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge);
}else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); int iArg = sqlite3_value_int(pVal);
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg);
#ifdef SQLITE_DEBUG #ifdef SQLITE_DEBUG
}else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
pConfig->bPrefixIndex = sqlite3_value_int(pVal); pConfig->bPrefixIndex = sqlite3_value_int(pVal);

12
ext/fts5/fts5_storage.c
View File

@ -881,13 +881,14 @@ static int fts5StorageIntegrityCallback(
** some other SQLite error code if an error occurs while attempting to ** some other SQLite error code if an error occurs while attempting to
** determine this. ** determine this.
*/ */
int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
Fts5Config *pConfig = p->pConfig; Fts5Config *pConfig = p->pConfig;
int rc; /* Return code */ int rc = SQLITE_OK; /* Return code */
int *aColSize; /* Array of size pConfig->nCol */ int *aColSize; /* Array of size pConfig->nCol */
i64 *aTotalSize; /* Array of size pConfig->nCol */ i64 *aTotalSize; /* Array of size pConfig->nCol */
Fts5IntegrityCtx ctx; Fts5IntegrityCtx ctx;
sqlite3_stmt *pScan; sqlite3_stmt *pScan;
int bUseCksum;
memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); memset(&ctx, 0, sizeof(Fts5IntegrityCtx));
ctx.pConfig = p->pConfig; ctx.pConfig = p->pConfig;
@ -896,6 +897,10 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
aColSize = (int*)&aTotalSize[pConfig->nCol]; aColSize = (int*)&aTotalSize[pConfig->nCol];
memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol);
bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL
|| (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg)
);
if( bUseCksum ){
/* Generate the expected index checksum based on the contents of the /* Generate the expected index checksum based on the contents of the
** %_content table. This block stores the checksum in ctx.cksum. */ ** %_content table. This block stores the checksum in ctx.cksum. */
rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
@ -967,12 +972,13 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
rc = fts5StorageCount(p, "docsize", &nRow); rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
} }
}
/* Pass the expected checksum down to the FTS index module. It will /* Pass the expected checksum down to the FTS index module. It will
** verify, amongst other things, that it matches the checksum generated by ** verify, amongst other things, that it matches the checksum generated by
** inspecting the index itself. */ ** inspecting the index itself. */
if( rc==SQLITE_OK ){ if( rc==SQLITE_OK ){
rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum); rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum);
} }
sqlite3_free(aTotalSize); sqlite3_free(aTotalSize);

37
ext/fts5/test/fts5integrity.test
View File

@ -273,7 +273,6 @@ do_execsql_test 9.1.2 {
SELECT quote(x'37'); SELECT quote(x'37');
} {X'37'} } {X'37'}
breakpoint
do_execsql_test 9.2 { do_execsql_test 9.2 {
INSERT INTO vt0 VALUES (SUBSTR(x'37', 0)); INSERT INTO vt0 VALUES (SUBSTR(x'37', 0));
-- INSERT INTO vt0 VALUES (x'37'); -- INSERT INTO vt0 VALUES (x'37');
@ -282,4 +281,40 @@ do_execsql_test 9.3 {
INSERT INTO vt0(vt0) VALUES('integrity-check'); INSERT INTO vt0(vt0) VALUES('integrity-check');
} }
#-------------------------------------------------------------------------
reset_db
do_execsql_test 10.0 {
CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b);
CREATE VIRTUAL TABLE vt0 USING fts5(a, b, content=t1);
INSERT INTO vt0(rowid, a, b) VALUES(1, 'abc', 'def');
}
do_catchsql_test 10.1 {
INSERT INTO vt0(vt0) VALUES('integrity-check');
} {0 {}}
do_catchsql_test 10.2 {
INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 0);
} {0 {}}
do_catchsql_test 10.3 {
INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1);
} {1 {database disk image is malformed}}
do_catchsql_test 10.3 {
INSERT INTO t1 VALUES(1, 'abc', 'def');
INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1);
} {0 {}}
do_execsql_test 10.4 {
CREATE VIRTUAL TABLE vt1 USING fts5(a, b, content=);
INSERT INTO vt1(rowid, a, b) VALUES(1, 'abc', 'def');
}
do_catchsql_test 10.5.1 {
INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 0);
} {0 {}}
do_catchsql_test 10.5.2 {
INSERT INTO vt0(vt0, rank) VALUES('integrity-check', 1);
} {0 {}}
do_catchsql_test 10.5.3 {
INSERT INTO vt0(vt0) VALUES('integrity-check');
} {0 {}}
finish_test finish_test

22
manifest
View File

@ -1,5 +1,5 @@
C Remove\sa\snew\sassert()\sthat\sthe\scompiler\sfinds\sis\salways\strue\sat\scompile-time,\nto\savoid\sa\scompiler\swarning. C In\sfts5\sintegrity\schecks,\sdo\snot\scompare\sthe\scontents\sof\sthe\sindex\sagainst\san\sexternal\scontent\stable\sunless\sspecifically\srequested.
D 2020-09-20T13:33:28.653 D 2020-09-21T14:53:21.394
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -112,15 +112,15 @@ F ext/fts3/unicode/mkunicode.tcl bf7fcaa6d68e6d38223467983785d054f1cff4d9e3905dd
F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0 F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a F ext/fts5/fts5.h c132a9323f22a972c4c93a8d5a3d901113a6e612faf30ca8e695788438c5ca2a
F ext/fts5/fts5Int.h d7cbc214ee167496f70905667e18f73ea0402f7ef09236ce305e117e0efc866a F ext/fts5/fts5Int.h ba835c165bb87650fc806008969799a7a1fbe3e221fd5a850dd044eb6a87b243
F ext/fts5/fts5_aux.c dcc627d8b6e3fc773db528ff67b39955dab7b51628f9dba8e15849e5bedfd7fa F ext/fts5/fts5_aux.c dcc627d8b6e3fc773db528ff67b39955dab7b51628f9dba8e15849e5bedfd7fa
F ext/fts5/fts5_buffer.c 5a5fe0159752c0fb0a5a93c722e9db2662822709490769d482b76a6dc8aaca70 F ext/fts5/fts5_buffer.c 5a5fe0159752c0fb0a5a93c722e9db2662822709490769d482b76a6dc8aaca70
F ext/fts5/fts5_config.c b447948f35ad3354e8fe5e242e0a7e7b5b941555400b9404259944e3aa570037 F ext/fts5/fts5_config.c b447948f35ad3354e8fe5e242e0a7e7b5b941555400b9404259944e3aa570037
F ext/fts5/fts5_expr.c b7b28ed203a3140f2fc503507d2c614a6cf1bd2e8987497f8894abc3f1477ec4 F ext/fts5/fts5_expr.c b7b28ed203a3140f2fc503507d2c614a6cf1bd2e8987497f8894abc3f1477ec4
F ext/fts5/fts5_hash.c 15bffa734fbdca013b2289c6f8827a3b935ef14bd4dde5837d31a75434c00627 F ext/fts5/fts5_hash.c 15bffa734fbdca013b2289c6f8827a3b935ef14bd4dde5837d31a75434c00627
F ext/fts5/fts5_index.c 203a8db4c35923f584e1a79cb43aa389710db6fb3ace72aad476e15b4dbd7b3e F ext/fts5/fts5_index.c 255d3ce3fec28be11c533451e5b23bd79e71a13a1b120f3658b34fff6b097816
F ext/fts5/fts5_main.c e881a2ea0bf01b3a3ff0bc1b31373c58fd54b6c9f3c43ea3d431bea4e5d4025e F ext/fts5/fts5_main.c 30969e4e14e720e9c603e66714bd1905a63defd4492d5a16d2671398a664fcfd
F ext/fts5/fts5_storage.c 34774cac00302fee7c6ffe05e12ec38f90777fb1681245538c97357cf5eb5d0e F ext/fts5/fts5_storage.c 58ba71e6cd3d43a5735815e7956ee167babb4d2cbfe206905174792af4d09d75
F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95 F ext/fts5/fts5_tcl.c 39bcbae507f594aad778172fa914cad0f585bf92fd3b078c686e249282db0d95
F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
F ext/fts5/fts5_test_tok.c f96c6e193c466711d6d7828d5f190407fe7ab897062d371426dd3036f01258e7 F ext/fts5/fts5_test_tok.c f96c6e193c466711d6d7828d5f190407fe7ab897062d371426dd3036f01258e7
@ -184,7 +184,7 @@ F ext/fts5/test/fts5first.test 3fcf2365c00a15fc9704233674789a3b95131d12de18a9b99
F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717 F ext/fts5/test/fts5full.test e1701a112354e0ff9a1fdffb0c940c576530c33732ee20ac5e8361777070d717
F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
F ext/fts5/test/fts5hash.test dc7bc7e0cdeb42cfce31294ad2f8fcf43192bfd0145bb7f3ecc5465d8c72696f F ext/fts5/test/fts5hash.test dc7bc7e0cdeb42cfce31294ad2f8fcf43192bfd0145bb7f3ecc5465d8c72696f
F ext/fts5/test/fts5integrity.test 8ffabcd91b058d812aba3e3e0a06f76ce165ba402a18cca20e34204a7feec92e F ext/fts5/test/fts5integrity.test e387b2bd1c83e50f4a12f58a5fd399111bbab36be2f1c9fd5bb974be08a32de6
F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fdde7ea00e78ea283d227 F ext/fts5/test/fts5interrupt.test 09613247b273a99889808ef852898177e671406fe71fdde7ea00e78ea283d227
F ext/fts5/test/fts5lastrowid.test be98fe3e03235296585b72daad7aed5717ba0062bae5e5c18dd6e04e194c6b28 F ext/fts5/test/fts5lastrowid.test be98fe3e03235296585b72daad7aed5717ba0062bae5e5c18dd6e04e194c6b28
F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad
@ -1880,7 +1880,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P de8ce22a46c90afa5475cd24c28b7a82b26410dc72d662af2f9d9e5e528e0eec P 8bbaee238593741da107b92b254cb5f0cf8b6bcc760bf2b25d20e46c8edee2a9
R 0b8e09249869856bd234668560922f9f R 2dc50a55577b973c60140e20c804f94a
U drh U dan
Z 1341315cc82b02a68b29a94b0a5ab754 Z 711849bf69192b1920219728f40bdd65

2
manifest.uuid
View File

@ -1 +1 @@
8bbaee238593741da107b92b254cb5f0cf8b6bcc760bf2b25d20e46c8edee2a9 782163693f37aeb65209bebbaeb6659a36881b8c4b4bec778b366658488bf966