From 73e7d8b2bbccfc462e57973939c2acd887e1067b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 19:47:19 +0000 Subject: [PATCH 01/36] Fix a compiler warning and an after-OOM memory leak. FossilOrigin-Name: 58f7ca29303c280229f86d01f418e1de5f5aebba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a3dc17de19..e6d729925d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\schanges\sfrom\strunk. -D 2014-03-28T18:35:39.779 +C Fix\sa\scompiler\swarning\sand\san\safter-OOM\smemory\sleak. +D 2014-03-28T19:47:19.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b +F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 81987c8ceb64f051528a6ca42673821d9ab7c0ff 27deb6e49bcc76714dbdc61b34748603155ac770 -R 79fcad4572db3d584834fdf0de4170a6 +P 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e +R 911f6d7d99d9350a186d106d9b5cf527 U drh -Z 871d90dec169e4d494b7ff32475fff97 +Z 55f6b91a865b632e99503eb67df5d777 diff --git a/manifest.uuid b/manifest.uuid index 518ac58c2a..cf96f01707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e \ No newline at end of file +58f7ca29303c280229f86d01f418e1de5f5aebba \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index f229b1f18b..791a2465ca 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -959,7 +959,8 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*) + (aNew + ((u8*)pSorter->pRecord - pSorter->aMemory)); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1052,7 +1053,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree = (int *)&pSorter->aIter[N]; pSorter->nTree = N; - do { + while(1){ int iNew; /* Index of new, merged, PMA */ for(iNew=0; @@ -1105,6 +1106,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( rc==SQLITE_OK ) rc = rc2; } } + if( rc ) break; if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ break; @@ -1117,7 +1119,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->iReadOff = 0; iWrite2 = 0; } - }while( rc==SQLITE_OK ); + } if( pTemp2 ){ sqlite3OsCloseFree(pTemp2); From f690c142806a545dbea5c3ed8cc3680ef2cbd78b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Apr 2014 16:25:25 +0000 Subject: [PATCH 02/36] Improved header comment on the vdbesort.c module. No changes to code. FossilOrigin-Name: bf09ce24d054bc68c226064f5f28d97e0e648a13 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 56 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0187e05510..b7837cbb94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reopen\sthe\sorderby-planning\sbranch\sand\smerge\sin\sthe\slatest\strunk\senhancements\nand\sfixes. -D 2014-04-24T15:06:25.003 +C Improved\sheader\scomment\son\sthe\svdbesort.c\smodule.\s\sNo\schanges\sto\scode. +D 2014-04-24T16:25:25.812 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 +F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 58f7ca29303c280229f86d01f418e1de5f5aebba 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R b46cdf624d79e68eb1974021b4d3f992 +P 6077ddcd93318e24b9756adaaf293ba9fb3cedf7 +R f7111ea680230454a4775d5c337b77f9 U drh -Z d3c484ede684435a1175d26a414ba92a +Z b4087c0ba17f1174753bea8bd453086a diff --git a/manifest.uuid b/manifest.uuid index 4866c44be0..a672982fcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6077ddcd93318e24b9756adaaf293ba9fb3cedf7 \ No newline at end of file +bf09ce24d054bc68c226064f5f28d97e0e648a13 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 791a2465ca..6a34cefa61 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,9 +10,55 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a external merge sort +** algorithm that is efficient even if the aggregate size of +** the elements being sorted exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). +** */ #include "sqliteInt.h" @@ -893,7 +939,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; From aa9933c11529cabf659615d9c94277846dacbd15 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 24 Apr 2014 20:04:49 +0000 Subject: [PATCH 03/36] Changes to the way the planner calculates the costs of various table and index scans. Some test cases still failing. FossilOrigin-Name: c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 --- manifest | 25 +++++----- manifest.uuid | 2 +- src/where.c | 109 ++++++++++++++++++++++--------------------- test/analyze9.test | 8 ++-- test/autoindex1.test | 2 + test/where3.test | 3 +- test/whereG.test | 32 +++++++++++++ 7 files changed, 111 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..2b621bd738 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. +D 2014-04-24T20:04:49.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e072a5172d55afcba98d6ca6a219ce8878c2f5c9 +F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -329,7 +329,7 @@ F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test d4dfe14001dfcb74cfbd7107f45a79fc1ab6183e +F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1079,7 +1079,7 @@ F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db -F test/where3.test d28c51f257e60be30f74308fa385ceeddfb54a6e +F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2533b72ed4a31fd1687230a499b557b911525344 +F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 -U drh -Z cf9f241149456ab1fa24984e95a412d2 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R 6cc54703275bf2ed6708c34ae52cf7ea +T *branch * experimental-costs +T *sym-experimental-costs * +T -sym-trunk * +U dan +Z 868aa60f36dda291ae018583501645e5 diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..4c2b675cff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6a75840525 100644 --- a/src/where.c +++ b/src/where.c @@ -227,7 +227,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; }else{ - pTerm->truthProb = -1; + pTerm->truthProb = 1; } pTerm->pExpr = sqlite3ExprSkipCollate(p); pTerm->wtFlags = wtFlags; @@ -1975,6 +1975,31 @@ static void whereKeyStats( } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +/* +** If it is not NULL, pTerm is a term that provides an upper or lower +** bound on a range scan. Without considering pTerm, it is estimated +** that the scan will visit nNew rows. This function returns the number +** estimated to be visited after taking pTerm into account. +** +** If the user explicitly specified a likelihood() value for this term, +** then the return value is the likelihood multiplied by the number of +** input rows. Otherwise, this function assumes that an "IS NOT NULL" term +** has a likelihood of 0.50, and any other term a likelihood of 0.25. +*/ +static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ + LogEst nRet = nNew; + if( pTerm ){ + if( pTerm->truthProb<=0 ){ + nRet += pTerm->truthProb; + }else if( pTerm->wtFlags & TERM_VNULL ){ + nRet -= 10; assert( 10==sqlite3LogEst(2) ); + }else{ + nRet -= 20; assert( 20==sqlite3LogEst(4) ); + } + } + return nRet; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2127,17 +2152,9 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); - /* TUNING: Each inequality constraint reduces the search space 4-fold. - ** A BETWEEN operator, therefore, reduces the search space 16-fold */ - nNew = nOut; - if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } - if( pUpper ){ - nNew -= 20; assert( 20==sqlite3LogEst(4) ); - nOut--; - } + nNew = whereRangeAdjust(pLower, nOut); + nNew = whereRangeAdjust(pUpper, nNew); + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewnOut = (LogEst)nOut; @@ -3987,7 +4004,9 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX==pTerm ) break; if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } - if( j<0 ) pLoop->nOut += pTerm->truthProb; + if( j<0 ){ + pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + } } } @@ -4081,6 +4100,7 @@ static int whereLoopAddBtreeIndex( pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + LogEst rCostIdx; int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; @@ -4154,7 +4174,8 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm-2] : 0; } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut and rRun for STAT3 range values */ + /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + ** data, using some other estimate. */ assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); } @@ -4181,13 +4202,16 @@ static int whereLoopAddBtreeIndex( } } #endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add + ** it to pNew->rRun, which is currently set to the cost of the index + ** seek only. Then, if this is a non-covering index, add the cost of + ** visiting the rows in the main table. */ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ - /* Each row involves a step of the index, then a binary search of - ** the main table */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun,rLogSize>27 ? rLogSize-17 : 10); + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } - /* Step cost for each output row */ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut); + whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 @@ -4319,6 +4343,7 @@ static int whereLoopAddBtree( sPk.aiRowEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; + sPk.szIdxRow = pTab->szTabRow; aiRowEstPk[0] = pTab->nRowEst; aiRowEstPk[1] = 1; pFirst = pSrc->pTab->pIndex; @@ -4396,10 +4421,8 @@ static int whereLoopAddBtree( /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is 3*(N + log2(N)). - ** + The extra 3 factor is to encourage the use of indexed lookups - ** over full scans. FIXME */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 16; + /* TUNING: Cost of full table scan is (N*3.0). */ + pNew->rRun = rSize + 16; whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4426,35 +4449,16 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: The base cost of an index scan is N + log2(N). - ** The log2(N) is for the initial seek to the beginning and the N - ** is for the scan itself. */ - pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); - if( m==0 ){ - /* TUNING: Cost of a covering index scan is K*(N + log2(N)). - ** + The extra factor K of between 1.1 and 3.0 that depends - ** on the relative sizes of the table and the index. K - ** is smaller for smaller indices, thus favoring them. - ** The upper bound on K (3.0) matches the penalty factor - ** on a full table scan that tries to encourage the use of - ** indexed lookups over full scans. - */ - pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; - }else{ - /* TUNING: The cost of scanning a non-covering index is multiplied - ** by log2(N) to account for the binary search of the main table - ** that must happen for each row of the index. - ** TODO: Should there be a multiplier here, analogous to the 3x - ** multiplier for a fulltable scan or covering index scan, to - ** further discourage the use of an index scan? Or is the log2(N) - ** term sufficient discouragement? - ** TODO: What if some or all of the WHERE clause terms can be - ** computed without reference to the original table. Then the - ** penality should reduce to logK where K is the number of output - ** rows. - */ - pNew->rRun += rLogSize; + + /* The cost of visiting the index rows is N*K, where K is + ** between 1.1 and 3.0, depending on the relative sizes of the + ** index and table rows. If this is a non-covering index scan, + ** also add the cost of visiting table rows (N*3.0). */ + pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; + if( m!=0 ){ + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } + whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4732,8 +4736,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->iSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun + 18; + pNew->rRun = sSum.a[i].rRun; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/analyze9.test b/test/analyze9.test index 820bcdb0e7..125cecf182 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -577,16 +577,16 @@ do_test 13.1 { execsql ANALYZE } {} do_eqp_test 13.2.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<15 AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.2.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'15' AND b<12 } {/SEARCH TABLE t1 USING INDEX i1/} do_eqp_test 13.3.1 { - SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<100 AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} do_eqp_test 13.3.2 { - SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<20 + SELECT * FROM t1 WHERE a='abc' AND rowid<'100' AND b<12 } {/SEARCH TABLE t1 USING INDEX i2/} #------------------------------------------------------------------------- diff --git a/test/autoindex1.test b/test/autoindex1.test index 0e5032b9b9..6cb0ab146a 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -97,6 +97,8 @@ do_test autoindex1-210 { PRAGMA automatic_index=ON; ANALYZE; UPDATE sqlite_stat1 SET stat='10000' WHERE tbl='t1'; + -- Table t2 actually contains 8 rows. + UPDATE sqlite_stat1 SET stat='16' WHERE tbl='t2'; ANALYZE sqlite_master; SELECT b, (SELECT d FROM t2 WHERE c=a) FROM t1; } diff --git a/test/where3.test b/test/where3.test index 8fa9fa7840..c2804b5579 100644 --- a/test/where3.test +++ b/test/where3.test @@ -231,6 +231,7 @@ do_execsql_test where3-3.0 { CREATE TABLE t301(a INTEGER PRIMARY KEY,b,c); CREATE INDEX t301c ON t301(c); INSERT INTO t301 VALUES(1,2,3); + INSERT INTO t301 VALUES(2,2,3); CREATE TABLE t302(x, y); INSERT INTO t302 VALUES(4,5); ANALYZE; @@ -251,7 +252,7 @@ do_execsql_test where3-3.2 { } {} do_execsql_test where3-3.3 { SELECT * FROM t301 WHERE c=3 AND a IS NOT NULL; -} {1 2 3} +} {1 2 3 2 2 3} if 0 { # Query planner no longer does this # Verify that when there are multiple tables in a join which must be diff --git a/test/whereG.test b/test/whereG.test index 17d5653223..6274213491 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix whereG do_execsql_test whereG-1.0 { CREATE TABLE composer( @@ -179,5 +180,36 @@ do_execsql_test whereG-4.0 { ORDER BY x; } {right} +#------------------------------------------------------------------------- +# + +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a, b); +} +do_eqp_test 5.1.2 { + SELECT * FROM t1 WHERE a>? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} +do_eqp_test 5.1.3 { + SELECT * FROM t1 WHERE likelihood(a>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + +do_test 5.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 VALUES('abc', $i, $i); } + } + execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } + execsql { ANALYZE } +} {} + +do_eqp_test 5.2.2 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.01) +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} + +do_eqp_test 5.2.3 { + SELECT * FROM t1 WHERE likelihood(b>?, 0.9) +} {0 0 0 {SCAN TABLE t1}} finish_test + From a01c7c70fd4d36fcbb6b71c783efb877b6cfbd5f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 12:35:31 +0000 Subject: [PATCH 04/36] When VDBE_PROFILE is enabled, ignore negative opcode times, which sometimes occur, perhaps due to context swaps. FossilOrigin-Name: 4e88042f9d3e678914da96c0eb276f3d8fca5a94 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7f4e9189fd..b0a9e4b587 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Comment\stweaks\son\sthe\stest\scase\sfor\sthe\s[b75a9ca6b0]\sbug\sfix. -D 2014-04-21T13:36:54.639 +C When\sVDBE_PROFILE\sis\senabled,\signore\snegative\sopcode\stimes,\swhich\ssometimes\noccur,\sperhaps\sdue\sto\scontext\sswaps. +D 2014-04-25T12:35:31.621 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c bf284edea1cee0508dc93c79a73498e0f317edb7 +F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P de9a490f594183f337a2ec9e0f87792eac83548b -R ce888b84132e0cad3bcca115a32951d3 +P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb +R c6b40d1b54d6305d3c2bd44472b1c76f U drh -Z cf9f241149456ab1fa24984e95a412d2 +Z 6f02403cb59a60204ab5357a363dc55d diff --git a/manifest.uuid b/manifest.uuid index a2761a597b..1b88d91a9c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65d2544af9adc1e2f1d193e57f8be0422fb0d5eb \ No newline at end of file +4e88042f9d3e678914da96c0eb276f3d8fca5a94 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6102cf3c7e..4ef6d0738c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -6323,8 +6323,8 @@ default: { /* This is really OP_Noop and OP_Explain */ #ifdef VDBE_PROFILE { - u64 elapsed = sqlite3Hwtime() - start; - pOp->cycles += elapsed; + u64 endTime = sqlite3Hwtime(); + if( endTime>start ) pOp->cycles += endTime - start; pOp->cnt++; } #endif From 9dfaf621d1e88d5570cc6547a2ab495f66f1a12f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 14:42:17 +0000 Subject: [PATCH 05/36] Make sure ORDER BY clauses on joins that involve virtual tables and that have a DISTINCT clause work correctly. This is a candidate fix for ticket [388d01d4bb8f9]. Test cases for that ticket will be checked in separately. FossilOrigin-Name: 171138122690faafde0dcab0201b90bdf02d3637 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 13 ++++--------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b0a9e4b587..459765f3d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sVDBE_PROFILE\sis\senabled,\signore\snegative\sopcode\stimes,\swhich\ssometimes\noccur,\sperhaps\sdue\sto\scontext\sswaps. -D 2014-04-25T12:35:31.621 +C Make\ssure\sORDER\sBY\sclauses\son\sjoins\sthat\sinvolve\svirtual\stables\sand\sthat\nhave\sa\sDISTINCT\sclause\swork\scorrectly.\s\sThis\sis\sa\scandidate\sfix\sfor\nticket\s[388d01d4bb8f9].\s\sTest\scases\sfor\sthat\sticket\swill\sbe\schecked\sin\nseparately. +D 2014-04-25T14:42:17.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3b127bdc24b7aa84ffa69729170be11555cd7733 +F src/where.c 6ae02f1e8b1b29744d9e8cd9b95eac4c5232736d F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R c6b40d1b54d6305d3c2bd44472b1c76f +P 4e88042f9d3e678914da96c0eb276f3d8fca5a94 +R 1db9e5d4fb5773f03fcd55df7d35ffca U drh -Z 6f02403cb59a60204ab5357a363dc55d +Z 5fd090bfe2e284c7964226b5abb1772a diff --git a/manifest.uuid b/manifest.uuid index 1b88d91a9c..62175a4a6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e88042f9d3e678914da96c0eb276f3d8fca5a94 \ No newline at end of file +171138122690faafde0dcab0201b90bdf02d3637 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9bde27e52f..6342b7b50b 100644 --- a/src/where.c +++ b/src/where.c @@ -4857,14 +4857,6 @@ static i8 wherePathSatisfiesOrderBy( */ assert( pOrderBy!=0 ); - - /* Sortability of virtual tables is determined by the xBestIndex method - ** of the virtual table itself */ - if( pLast->wsFlags & WHERE_VIRTUALTABLE ){ - testcase( nLoop>0 ); /* True when outer loops are one-row and match - ** no ORDER BY terms */ - return pLast->u.vtab.isOrdered; - } if( nLoop && OptimizationDisabled(db, SQLITE_OrderByIdxJoin) ) return 0; nOrderBy = pOrderBy->nExpr; @@ -4877,7 +4869,10 @@ static i8 wherePathSatisfiesOrderBy( for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; pLoop = iLoopaLoop[iLoop] : pLast; - assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ + if( pLoop->u.vtab.isOrdered ) obSat = obDone; + break; + } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; /* Mark off any ORDER BY term X that is a column in the table of From cfc9df76e11b05d275e77d067f359c12044ea943 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 15:01:01 +0000 Subject: [PATCH 06/36] Store values loaded from the stat1 table as logarithmic values in memory. FossilOrigin-Name: 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 --- manifest | 25 +++++++++++-------------- manifest.uuid | 2 +- src/analyze.c | 19 ++++++++++++------- src/build.c | 26 +++++++++++++++++++------- src/pragma.c | 6 ++++-- src/select.c | 10 +++++----- src/sqliteInt.h | 5 ++++- src/where.c | 30 ++++++++++++++++-------------- 8 files changed, 72 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 2b621bd738..0c855c53e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sthe\sway\sthe\splanner\scalculates\sthe\scosts\sof\svarious\stable\sand\sindex\sscans.\sSome\stest\scases\sstill\sfailing. -D 2014-04-24T20:04:49.939 +C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. +D 2014-04-25T15:01:01.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 663e0b291d27eb03c9fd6b421e2d61ba348a2389 +F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5bfeea8f302ec2926c9eea321a61daea92a29fa4 +F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -211,18 +211,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 21ece94d4f3e76e8e150deecafb9c7abd398ec67 +F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c bc7feff0fb4c4a1b9d655b717bef166846b48e33 +F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 03e2f60ccb0745fa2d3a072cb4f75fa29251d2ee +F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c c12bc20cd649bcae39de3e452bfc1a3f164454ee +F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1161,10 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 6cc54703275bf2ed6708c34ae52cf7ea -T *branch * experimental-costs -T *sym-experimental-costs * -T -sym-trunk * +P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 +R 63fad85eb66cf540e6aa11923b167cbf U dan -Z 868aa60f36dda291ae018583501645e5 +Z 04f4e3645ebd9e33526df3bd26c04a76 diff --git a/manifest.uuid b/manifest.uuid index 4c2b675cff..5436721392 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 \ No newline at end of file +1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2a03e292c8..4fbaaa5173 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1371,6 +1371,7 @@ static void decodeIntArray( char *zIntArray, /* String containing int array to decode */ int nOut, /* Number of slots in aOut[] */ tRowcnt *aOut, /* Store integers here */ + LogEst *aLog, /* Or, if aOut==0, here */ Index *pIndex /* Handle extra flags for this index, if not NULL */ ){ char *z = zIntArray; @@ -1389,7 +1390,11 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } - aOut[i] = v; + if( aOut ){ + aOut[i] = v; + }else{ + aLog[i] = sqlite3LogEst(v); + } if( *z==' ' ) z++; } #ifndef SQLITE_ENABLE_STAT3_OR_STAT4 @@ -1445,12 +1450,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ - decodeIntArray((char*)z, pIndex->nKeyCol+1, pIndex->aiRowEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; - decodeIntArray((char*)z, 1, &pTable->nRowEst, &fakeIdx); + decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } @@ -1642,9 +1647,9 @@ static int loadStatTbl( pPrevIdx = pIdx; } pSample = &pIdx->aSample[pIdx->nSample]; - decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); - decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,1),nCol,pSample->anEq,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the diff --git a/src/build.c b/src/build.c index 4d4155ba90..fb025495dc 100644 --- a/src/build.c +++ b/src/build.c @@ -905,7 +905,7 @@ void sqlite3StartTable( pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; pTable->nRef = 1; - pTable->nRowEst = 1048576; + pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; @@ -2730,15 +2730,15 @@ Index *sqlite3AllocateIndexObject( nByte = ROUND8(sizeof(Index)) + /* Index structure */ ROUND8(sizeof(char*)*nCol) + /* Index.azColl */ - ROUND8(sizeof(tRowcnt)*(nCol+1) + /* Index.aiRowEst */ + ROUND8(sizeof(LogEst)*(nCol+1) + /* Index.aiRowLogEst */ sizeof(i16)*nCol + /* Index.aiColumn */ sizeof(u8)*nCol); /* Index.aSortOrder */ p = sqlite3DbMallocZero(db, nByte + nExtra); if( p ){ char *pExtra = ((char*)p)+ROUND8(sizeof(Index)); - p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); - p->aiRowEst = (tRowcnt*)pExtra; pExtra += sizeof(tRowcnt)*(nCol+1); - p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; + p->azColl = (char**)pExtra; pExtra += ROUND8(sizeof(char*)*nCol); + p->aiRowLogEst = (LogEst*)pExtra; pExtra += sizeof(LogEst)*(nCol+1); + p->aiColumn = (i16*)pExtra; pExtra += sizeof(i16)*nCol; p->aSortOrder = (u8*)pExtra; p->nColumn = nCol; p->nKeyCol = nCol - 1; @@ -2968,7 +2968,7 @@ Index *sqlite3CreateIndex( if( db->mallocFailed ){ goto exit_create_index; } - assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowEst) ); + assert( EIGHT_BYTE_ALIGNMENT(pIndex->aiRowLogEst) ); assert( EIGHT_BYTE_ALIGNMENT(pIndex->azColl) ); pIndex->zName = zExtra; zExtra += nName + 1; @@ -3249,7 +3249,7 @@ exit_create_index: ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number -** of rows that match any particular combiniation of the first 2 columns +** of rows that match any particular combination of the first 2 columns ** of the index. And so forth. It must always be the case that * ** aiRowEst[N]<=aiRowEst[N-1] @@ -3260,6 +3260,7 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ +#if 0 tRowcnt *a = pIdx->aiRowEst; int i; tRowcnt n; @@ -3274,6 +3275,17 @@ void sqlite3DefaultRowEst(Index *pIdx){ if( pIdx->onError!=OE_None ){ a[pIdx->nKeyCol] = 1; } +#endif + /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ + LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + LogEst *a = pIdx->aiRowLogEst; + int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + + a[0] = pIdx->pTable->nRowLogEst; + memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); + if( pIdx->onError!=OE_None ){ + a[pIdx->nKeyCol] = 0; + } } /* diff --git a/src/pragma.c b/src/pragma.c index 66d0c3328e..4c69ceb4fd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1488,13 +1488,15 @@ void sqlite3Pragma( sqlite3VdbeAddOp2(v, OP_Null, 0, 2); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pTab->szTabRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pTab->nRowEst, 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pTab->nRowLogEst), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); sqlite3VdbeAddOp2(v, OP_Integer, (int)sqlite3LogEstToInt(pIdx->szIdxRow), 3); - sqlite3VdbeAddOp2(v, OP_Integer, (int)pIdx->aiRowEst[0], 4); + sqlite3VdbeAddOp2(v, OP_Integer, + (int)sqlite3LogEstToInt(pIdx->aiRowLogEst[0]), 4); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); } } diff --git a/src/select.c b/src/select.c index 6efdde4b72..dfca6d3f83 100644 --- a/src/select.c +++ b/src/select.c @@ -1690,7 +1690,7 @@ Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ assert( db->lookaside.bEnabled==0 ); pTab->nRef = 1; pTab->zName = 0; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); selectColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); selectAddColumnTypeAndCollation(pParse, pTab, pSelect); pTab->iPKey = -1; @@ -3829,7 +3829,7 @@ static int withExpand( pTab->nRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); if( db->mallocFailed ) return SQLITE_NOMEM; @@ -4005,7 +4005,7 @@ static int selectExpander(Walker *pWalker, Select *p){ while( pSel->pPrior ){ pSel = pSel->pPrior; } selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; - pTab->nRowEst = 1048576; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral; #endif }else{ @@ -4655,7 +4655,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); pItem->viaCoroutine = 1; pItem->regResult = dest.iSdst; sqlite3VdbeAddOp1(v, OP_EndCoroutine, pItem->regReturn); @@ -4686,7 +4686,7 @@ int sqlite3Select( sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow; + pItem->pTab->nRowLogEst = sqlite3LogEst(pSub->nSelectRow); if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); VdbeComment((v, "end %s", pItem->pTab->zName)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8e81b7a970..3ddcc8610f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1471,7 +1471,7 @@ struct Table { #ifndef SQLITE_OMIT_CHECK ExprList *pCheck; /* All CHECK constraints */ #endif - tRowcnt nRowEst; /* Estimated rows in table - from sqlite_stat1 table */ + LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ int tnum; /* Root BTree node for this table (see note above) */ i16 iPKey; /* If not negative, use aCol[iPKey] as the primary key */ i16 nCol; /* Number of columns in this table */ @@ -1680,7 +1680,10 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ +#if 0 tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ +#endif + LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ Index *pNext; /* The next index associated with the same table */ diff --git a/src/where.c b/src/where.c index 6a75840525..909129f666 100644 --- a/src/where.c +++ b/src/where.c @@ -1956,7 +1956,8 @@ static void whereKeyStats( iLower = 0; iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + i64 nRow0 = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); @@ -2092,7 +2093,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = p->aiRowEst[0]; + iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ @@ -2251,6 +2252,7 @@ static int whereInScanEst( tRowcnt *pnRow /* Write the revised row estimate here */ ){ Index *p = pBuilder->pNew->u.btree.pIndex; + i64 nRow0 = sqlite3LogEstToInt(p->aiRowLogEst[0]); int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ @@ -2259,14 +2261,14 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ - nEst = p->aiRowEst[0]; + nEst = nRow0; rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; pBuilder->nRecValid = nRecValid; } if( rc==SQLITE_OK ){ - if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; + if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); } @@ -4059,8 +4061,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = sqlite3LogEst(pProbe->aiRowEst[pNew->u.btree.nEq+1]); - if( nRowEst==0 && pProbe->onError==OE_None ) nRowEst = 1; + nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; nRowEst = 0; @@ -4074,7 +4075,7 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(sqlite3LogEst(pProbe->aiRowEst[0])); + rLogSize = estLog(pProbe->aiRowLogEst[0]); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4082,10 +4083,11 @@ static int whereLoopAddBtreeIndex( ** number 18 was found by experimentation to be the payoff point where ** skip-scan become faster than a full-scan. */ + assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip && saved_nEq+1nKeyCol - && pProbe->aiRowEst[saved_nEq+1]>=18 /* TUNING: Minimum for skip-scan */ + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ LogEst nIter; @@ -4093,7 +4095,7 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = sqlite3LogEst(pProbe->aiRowEst[0]/pProbe->aiRowEst[saved_nEq+1]); + nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; pNew->rRun = rLogSize + nIter; pNew->nOut += nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); @@ -4305,7 +4307,7 @@ static int whereLoopAddBtree( WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ Index sPk; /* A fake index object for the primary key */ - tRowcnt aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */ + LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ struct SrcList_item *pSrc; /* The FROM clause btree term to add */ @@ -4340,12 +4342,12 @@ static int whereLoopAddBtree( memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; sPk.aiColumn = &aiColumnPk; - sPk.aiRowEst = aiRowEstPk; + sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; sPk.szIdxRow = pTab->szTabRow; - aiRowEstPk[0] = pTab->nRowEst; - aiRowEstPk[1] = 1; + aiRowEstPk[0] = pTab->nRowLogEst; + aiRowEstPk[1] = 0; pFirst = pSrc->pTab->pIndex; if( pSrc->notIndexed==0 ){ /* The real indices of the table are only considered if the @@ -4354,7 +4356,7 @@ static int whereLoopAddBtree( } pProbe = &sPk; } - rSize = sqlite3LogEst(pTab->nRowEst); + rSize = pTab->nRowLogEst; rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX From b79c718f1ab8667bd14040c339a8e87c90bcff82 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Apr 2014 17:37:16 +0000 Subject: [PATCH 07/36] Add test cases to ensure correct operation of joins with a virtual table that include DISTINCT and ORDER BY clauses. Verification for ticket [388d01d4bb8f9]. FossilOrigin-Name: 5ada136f43ce744ae8c349eff39838eb44611b6e --- manifest | 11 ++--- manifest.uuid | 2 +- test/orderby7.test | 106 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 test/orderby7.test diff --git a/manifest b/manifest index 459765f3d0..4bb29e9787 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sORDER\sBY\sclauses\son\sjoins\sthat\sinvolve\svirtual\stables\sand\sthat\nhave\sa\sDISTINCT\sclause\swork\scorrectly.\s\sThis\sis\sa\scandidate\sfix\sfor\nticket\s[388d01d4bb8f9].\s\sTest\scases\sfor\sthat\sticket\swill\sbe\schecked\sin\nseparately. -D 2014-04-25T14:42:17.085 +C Add\stest\scases\sto\sensure\scorrect\soperation\sof\sjoins\swith\sa\svirtual\stable\nthat\sinclude\sDISTINCT\sand\sORDER\sBY\sclauses.\s\sVerification\sfor\sticket\n[388d01d4bb8f9]. +D 2014-04-25T17:37:16.863 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -724,6 +724,7 @@ F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 +F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e88042f9d3e678914da96c0eb276f3d8fca5a94 -R 1db9e5d4fb5773f03fcd55df7d35ffca +P 171138122690faafde0dcab0201b90bdf02d3637 +R a91060e3dc13463e0dda2e961246be75 U drh -Z 5fd090bfe2e284c7964226b5abb1772a +Z 5a55daa8fd800f752600937698ffd8a1 diff --git a/manifest.uuid b/manifest.uuid index 62175a4a6f..f56246ea09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -171138122690faafde0dcab0201b90bdf02d3637 \ No newline at end of file +5ada136f43ce744ae8c349eff39838eb44611b6e \ No newline at end of file diff --git a/test/orderby7.test b/test/orderby7.test new file mode 100644 index 0000000000..8c5fc9a85f --- /dev/null +++ b/test/orderby7.test @@ -0,0 +1,106 @@ +# 2014-04-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing ORDER BY optimizations on joins +# that involve virtual tables. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby7 + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE fts USING fts3(content TEXT); + INSERT INTO fts(rowid,content) + VALUES(1,'this is a test of the fts3 virtual'), + (2,'table used as part of a join together'), + (3,'with the DISTINCT keyword. There was'), + (4,'a bug at one time (2013-06 through 2014-04)'), + (5,'that prevented this from working correctly.'), + (11,'a row that occurs twice'), + (12,'a row that occurs twice'); + + CREATE TABLE t1(x TEXT PRIMARY KEY, y); + INSERT OR IGNORE INTO t1 SELECT content, rowid+100 FROM fts; +} {} +do_execsql_test 1.1 { + SELECT DISTINCT fts.rowid, t1.y + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY y; +} {11 111 12 111} +do_execsql_test 1.2 { + SELECT DISTINCT fts.rowid, t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {11 {a row that occurs twice} 12 {a row that occurs twice}} +do_execsql_test 1.3 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {{a row that occurs twice}} +do_execsql_test 1.4 { + SELECT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x + ORDER BY 1; +} {{a row that occurs twice} {a row that occurs twice}} +do_execsql_test 1.5 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x; +} {{a row that occurs twice}} +do_execsql_test 1.6 { + SELECT t1.x + FROM fts, t1 + WHERE fts MATCH 'that twice' + AND content=x; +} {{a row that occurs twice} {a row that occurs twice}} + +do_execsql_test 2.1 { + SELECT DISTINCT t1.x + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY fts.rowid; +} {{a row that occurs twice}} +do_execsql_test 2.2 { + SELECT DISTINCT t1.* + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY fts.rowid; +} {{a row that occurs twice} 111} +do_execsql_test 2.3 { + SELECT DISTINCT t1.* + FROM fts, t1 + WHERE fts.rowid=11 + AND content=x + ORDER BY t1.y +} {{a row that occurs twice} 111} + + + + +finish_test From 8ad1d8ba38ef4bf965ab1a657892b0e57be00a97 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 25 Apr 2014 20:22:45 +0000 Subject: [PATCH 08/36] If the user provides likelihood() data for a WHERE clause term used as part of an index key, have the planner use it when calculating the expected number of rows visited by the loop. FossilOrigin-Name: c51efaa5d29ee0a91b9e6a83a8dd82530670811a --- manifest | 18 +++--- manifest.uuid | 2 +- src/sqliteInt.h | 3 - src/where.c | 146 ++++++++++++++++++++++++++++------------------- src/whereInt.h | 1 + test/whereG.test | 18 ++++-- 6 files changed, 111 insertions(+), 77 deletions(-) diff --git a/manifest b/manifest index 0c855c53e6..afdc8889fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Store\svalues\sloaded\sfrom\sthe\sstat1\stable\sas\slogarithmic\svalues\sin\smemory. -D 2014-04-25T15:01:01.691 +C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. +D 2014-04-25T20:22:45.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bad694fd6b91b10a7a5aafa16fd05b504bad6b6e +F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 15a5c94c8c93500e141c6cb25af600615dc196d8 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1093,7 +1093,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test fd9120e262f9da3c45940f52aefeef4d15b904e5 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 8189fedf3b98ab581bb70f830175e403a0ef1722 +F test/whereG.test 0ac23e5e8311b69d87245f4a85112de321031658 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c5a6ec0a880652dc8f4593d9f7acd58ddc3dc5f3 -R 63fad85eb66cf540e6aa11923b167cbf +P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 +R 804d3aff8175381c892eb4eebbb86307 U dan -Z 04f4e3645ebd9e33526df3bd26c04a76 +Z 6dd1c2a2c429274ab108441094bd0963 diff --git a/manifest.uuid b/manifest.uuid index 5436721392..71ddf4b2d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 \ No newline at end of file +c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ddcc8610f..b584ff401d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1680,9 +1680,6 @@ struct UnpackedRecord { struct Index { char *zName; /* Name of this index */ i16 *aiColumn; /* Which columns are used by this index. 1st is 0 */ -#if 0 - tRowcnt *aiRowEst; /* From ANALYZE: Est. rows selected by each column */ -#endif LogEst *aiRowLogEst; /* From ANALYZE: Est. rows selected by each column */ Table *pTable; /* The SQL table being indexed */ char *zColAff; /* String defining the affinity of each column */ diff --git a/src/where.c b/src/where.c index 909129f666..73d3a22b7c 100644 --- a/src/where.c +++ b/src/where.c @@ -4040,7 +4040,6 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ - LogEst nRowEst; /* Estimated index selectivity */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4061,10 +4060,8 @@ static int whereLoopAddBtreeIndex( assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); if( pNew->u.btree.nEq < pProbe->nKeyCol ){ iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - nRowEst = pProbe->aiRowLogEst[pNew->u.btree.nEq+1]; }else{ iCol = -1; - nRowEst = 0; } pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); @@ -4095,35 +4092,40 @@ static int whereLoopAddBtreeIndex( pNew->u.btree.nSkip++; pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[0] - pProbe->aiRowLogEst[saved_nEq+1]; - pNew->rRun = rLogSize + nIter; - pNew->nOut += nIter; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter); + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + pNew->nOut -= nIter; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ + u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; + LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; #endif - if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) ){ continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ } if( pTerm->prereqRight & pNew->maskSelf ) continue; - assert( pNew->nOut==saved_nOut ); - pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; - pNew->rRun = rLogSize; /* Baseline cost is log2(N). Adjustments below */ - if( pTerm->eOperator & WO_IN ){ + + assert( nInMul==0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + ); + + if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ @@ -4135,87 +4137,111 @@ static int whereLoopAddBtreeIndex( } assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ - pNew->rRun += nIn; - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_EQ) ){ - assert( - (pNew->wsFlags & (WHERE_COLUMN_NULL|WHERE_COLUMN_IN|WHERE_SKIPSCAN))!=0 - || nInMul==0 - ); + + }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; - if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1)){ - assert( (pNew->wsFlags & WHERE_COLUMN_IN)==0 || iCol<0 ); + if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ if( iCol>=0 && pProbe->onError==OE_None ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; } } - pNew->u.btree.nEq++; - pNew->nOut = nRowEst + nInMul; - }else if( pTerm->eOperator & (WO_ISNULL) ){ - pNew->wsFlags |= WHERE_COLUMN_NULL; - pNew->u.btree.nEq++; - /* TUNING: IS NULL selects 2 rows */ - nIn = 10; assert( 10==sqlite3LogEst(2) ); - pNew->nOut = nRowEst + nInMul + nIn; - }else if( pTerm->eOperator & (WO_GT|WO_GE) ){ - testcase( pTerm->eOperator & WO_GT ); - testcase( pTerm->eOperator & WO_GE ); + + }else if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else{ - assert( pTerm->eOperator & (WO_LT|WO_LE) ); - testcase( pTerm->eOperator & WO_LT ); - testcase( pTerm->eOperator & WO_LE ); + }else if( (eOp & WO_ISNULL)==0 ){ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; pTop = pTerm; pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? pNew->aLTerm[pNew->nLTerm-2] : 0; } + + /* At this point pNew->nOut is set to the number of rows expected to + ** be visited by the index scan before considering term pTerm, or the + ** values of nIn and nInMul. In other words, assuming that all + ** "x IN(...)" terms are replaced with "x = ?". This block updates + ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ + assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 ** data, using some other estimate. */ - assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); - } + }else{ + int nEq = ++pNew->u.btree.nEq; + assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); + + assert( pNew->nOut==saved_nOut ); + if( pTerm->truthProb<=0 ){ + assert( (eOp & WO_IN) || nIn==0 ); + pNew->nOut += pTerm->truthProb; + pNew->nOut -= nIn; + pNew->wsFlags |= WHERE_LIKELIHOOD; + }else{ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) - ){ - Expr *pExpr = pTerm->pExpr; - tRowcnt nOut = 0; - if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ - testcase( pTerm->eOperator & WO_EQ ); - testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); - } - assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ){ - pNew->nOut = sqlite3LogEst(nOut); - if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + tRowcnt nOut = 0; + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && (pNew->wsFlags & WHERE_LIKELIHOOD)==0 + ){ + Expr *pExpr = pTerm->pExpr; + if( (eOp & (WO_EQ|WO_ISNULL))!=0 ){ + testcase( eOp & WO_EQ ); + testcase( eOp & WO_ISNULL ); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + }else{ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); + } + assert( rc!=SQLITE_OK || nOut>0 ); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; + if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ + if( nOut ){ + pNew->nOut = sqlite3LogEst(nOut); + if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; + pNew->nOut -= nIn; + } + } + if( nOut==0 ) +#endif + { + pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); + if( eOp & WO_ISNULL ){ + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows + ** as (col=?). */ + pNew->nOut += 10; + } + } } } -#endif + /* Set rCostIdx to the cost of visiting selected rows in index. Add ** it to pNew->rRun, which is currently set to the cost of the index ** seek only. Then, if this is a non-covering index, add the cost of ** visiting the rows in the main table. */ rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rCostIdx); + pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + nOutUnadjusted = pNew->nOut; + pNew->rRun += nInMul + nIn; + pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); + pNew->nOut = nOutUnadjusted; + if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) ){ diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..010cd6e8ae 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -458,3 +458,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_LIKELIHOOD 0x00020000 /* A likelihood() is affecting nOut */ diff --git a/test/whereG.test b/test/whereG.test index 6274213491..66918a35fa 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -181,9 +181,14 @@ do_execsql_test whereG-4.0 { } {right} #------------------------------------------------------------------------- -# - +# Test that likelihood() specifications on indexed terms are taken into +# account by various forms of loops. +# +# 5.1.*: open ended range scans +# 5.2.*: skip-scans +# reset_db + do_execsql_test 5.1 { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a, b); @@ -202,14 +207,19 @@ do_test 5.2 { execsql { INSERT INTO t1 SELECT 'def', b, c FROM t1; } execsql { ANALYZE } } {} - do_eqp_test 5.2.2 { SELECT * FROM t1 WHERE likelihood(b>?, 0.01) } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)}} - do_eqp_test 5.2.3 { SELECT * FROM t1 WHERE likelihood(b>?, 0.9) } {0 0 0 {SCAN TABLE t1}} +do_eqp_test 5.3.1 { + SELECT * FROM t1 WHERE a=? +} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)}} +do_eqp_test 5.3.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.9) +} {0 0 0 {SCAN TABLE t1}} + finish_test From 9940e2aa45604e9e9bf4c23af16b00bb78a03384 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 14:07:57 +0000 Subject: [PATCH 09/36] Avoid transfering records between tables unless the default values for all columns are the same. Fix for [f67b41381a]. FossilOrigin-Name: f8c4c495e6de1f124d205383d4bafa46accbff5c --- manifest | 15 ++++++------ manifest.uuid | 2 +- src/insert.c | 13 ++++++++--- test/tkt-f67b41381a.test | 50 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 test/tkt-f67b41381a.test diff --git a/manifest b/manifest index 4bb29e9787..498632b350 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\sensure\scorrect\soperation\sof\sjoins\swith\sa\svirtual\stable\nthat\sinclude\sDISTINCT\sand\sORDER\sBY\sclauses.\s\sVerification\sfor\sticket\n[388d01d4bb8f9]. -D 2014-04-25T17:37:16.863 +C Avoid\stransfering\srecords\sbetween\stables\sunless\sthe\sdefault\svalues\sfor\sall\scolumns\sare\sthe\ssame.\sFix\sfor\s[f67b41381a]. +D 2014-04-26T14:07:57.099 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d8bb30535c8c0785876025a4a07f9074640a15d1 +F src/insert.c 08de23111439c650cac52861a5b80d10d9d40560 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -906,6 +906,7 @@ F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 +F test/tkt-f67b41381a.test 18d3477b6fe82e6c5ec5e233d79642d7e6d0de77 F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead @@ -1162,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 171138122690faafde0dcab0201b90bdf02d3637 -R a91060e3dc13463e0dda2e961246be75 -U drh -Z 5a55daa8fd800f752600937698ffd8a1 +P 5ada136f43ce744ae8c349eff39838eb44611b6e +R 72b6179d3d1a0fd8f077c8cde285a253 +U dan +Z 7eecc39cd60ba5323d907c2532e97144 diff --git a/manifest.uuid b/manifest.uuid index f56246ea09..6aa5d7fb27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ada136f43ce744ae8c349eff39838eb44611b6e \ No newline at end of file +f8c4c495e6de1f124d205383d4bafa46accbff5c \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index abdf1ada95..3c77761b32 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1865,15 +1865,22 @@ static int xferOptimization( return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ } for(i=0; inCol; i++){ - if( pDest->aCol[i].affinity!=pSrc->aCol[i].affinity ){ + Column *pDestCol = &pDest->aCol[i]; + Column *pSrcCol = &pSrc->aCol[i]; + if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( !xferCompatibleCollation(pDest->aCol[i].zColl, pSrc->aCol[i].zColl) ){ + if( !xferCompatibleCollation(pDestCol->zColl, pSrcCol->zColl) ){ return 0; /* Collating sequence must be the same on all columns */ } - if( pDest->aCol[i].notNull && !pSrc->aCol[i].notNull ){ + if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } + if( (pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)) + ){ + return 0; /* Default values must be the same for all columns */ + } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ if( pDestIdx->onError!=OE_None ){ diff --git a/test/tkt-f67b41381a.test b/test/tkt-f67b41381a.test new file mode 100644 index 0000000000..d15d985342 --- /dev/null +++ b/test/tkt-f67b41381a.test @@ -0,0 +1,50 @@ +# 2014 April 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Test that ticket f67b41381a has been resolved. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-f67b41381a + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + ALTER TABLE t1 ADD COLUMN b DEFAULT 2; + CREATE TABLE t2(a, b); + INSERT INTO t2 SELECT * FROM t1; + SELECT * FROM t2; +} {1 2} + +db cache size 0 +foreach {tn tbls xfer} { + 1 { CREATE TABLE t1(a, b); CREATE TABLE t2(a, b) } 1 + 2 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b) } 0 + 3 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b DEFAULT 'x') } 1 + 4 { CREATE TABLE t1(a, b DEFAULT NULL); CREATE TABLE t2(a, b) } 0 + 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 0 + 6 { CREATE TABLE t1(a DEFAULT 1, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 +} { + + execsql { DROP TABLE t1; DROP TABLE t2 } + execsql $tbls + + set res 1 + db eval { EXPLAIN INSERT INTO t1 SELECT * FROM t2 } { + if {$opcode == "Column"} { set res 0 } + } + + do_test 2.$tn [list set res] $xfer +} + +finish_test + + From 453e0261dfed94ff6dcccaa6002dbd04499463e4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Apr 2014 17:52:08 +0000 Subject: [PATCH 10/36] Allow the xfer optimization to proceed if the DEFAULT on the very first column of the two tables is different. This is a refinement of the fix for ticket [f67b41381a]. FossilOrigin-Name: 349f483499dd685a8da94923b6bd810a52e5e236 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/insert.c | 6 ++++-- test/tkt-f67b41381a.test | 9 ++++++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 498632b350..3b524ffeaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\stransfering\srecords\sbetween\stables\sunless\sthe\sdefault\svalues\sfor\sall\scolumns\sare\sthe\ssame.\sFix\sfor\s[f67b41381a]. -D 2014-04-26T14:07:57.099 +C Allow\sthe\sxfer\soptimization\sto\sproceed\sif\sthe\sDEFAULT\son\sthe\svery\sfirst\ncolumn\sof\sthe\stwo\stables\sis\sdifferent.\s\sThis\sis\sa\srefinement\sof\sthe\nfix\sfor\sticket\s[f67b41381a]. +D 2014-04-26T17:52:08.640 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 08de23111439c650cac52861a5b80d10d9d40560 +F src/insert.c ab34bea5af4fee9f956a0805a32463fb7f674d00 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -906,7 +906,7 @@ F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 -F test/tkt-f67b41381a.test 18d3477b6fe82e6c5ec5e233d79642d7e6d0de77 +F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5ada136f43ce744ae8c349eff39838eb44611b6e -R 72b6179d3d1a0fd8f077c8cde285a253 -U dan -Z 7eecc39cd60ba5323d907c2532e97144 +P f8c4c495e6de1f124d205383d4bafa46accbff5c +R efe95b9fe528a9f472987911752f5742 +U drh +Z 366467002d461344ec8032510917cbba diff --git a/manifest.uuid b/manifest.uuid index 6aa5d7fb27..26e36016ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8c4c495e6de1f124d205383d4bafa46accbff5c \ No newline at end of file +349f483499dd685a8da94923b6bd810a52e5e236 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index 3c77761b32..909bc27474 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1876,8 +1876,10 @@ static int xferOptimization( if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } - if( (pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) - || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)) + /* Default values for second and subsequent columns need to match. */ + if( i>0 + && ((pDestCol->zDflt==0)!=(pSrcCol->zDflt==0) + || (pDestCol->zDflt && strcmp(pDestCol->zDflt, pSrcCol->zDflt)!=0)) ){ return 0; /* Default values must be the same for all columns */ } diff --git a/test/tkt-f67b41381a.test b/test/tkt-f67b41381a.test index d15d985342..1ddec988cd 100644 --- a/test/tkt-f67b41381a.test +++ b/test/tkt-f67b41381a.test @@ -30,8 +30,13 @@ foreach {tn tbls xfer} { 2 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b) } 0 3 { CREATE TABLE t1(a, b DEFAULT 'x'); CREATE TABLE t2(a, b DEFAULT 'x') } 1 4 { CREATE TABLE t1(a, b DEFAULT NULL); CREATE TABLE t2(a, b) } 0 - 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 0 + 5 { CREATE TABLE t1(a DEFAULT 2, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 6 { CREATE TABLE t1(a DEFAULT 1, b); CREATE TABLE t2(a DEFAULT 1, b) } 1 + 7 { CREATE TABLE t1(a DEFAULT 1, b DEFAULT 1); + CREATE TABLE t2(a DEFAULT 3, b DEFAULT 1) } 1 + 8 { CREATE TABLE t1(a DEFAULT 1, b DEFAULT 1); + CREATE TABLE t2(a DEFAULT 3, b DEFAULT 3) } 0 + } { execsql { DROP TABLE t1; DROP TABLE t2 } @@ -46,5 +51,3 @@ foreach {tn tbls xfer} { } finish_test - - From 1f8bb4b0a84986977b1e4d6712b57cd266c04c95 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Apr 2014 19:23:14 +0000 Subject: [PATCH 11/36] Update requirements marks to fix typos in the requirements text. No changes to code. FossilOrigin-Name: f5a263658187250044afc1a74000e6f6962733ca --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/e_createtable.test | 5 +++-- test/e_fkey.test | 11 ++++++----- 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3b524ffeaf..fea1e4c44c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sxfer\soptimization\sto\sproceed\sif\sthe\sDEFAULT\son\sthe\svery\sfirst\ncolumn\sof\sthe\stwo\stables\sis\sdifferent.\s\sThis\sis\sa\srefinement\sof\sthe\nfix\sfor\sticket\s[f67b41381a]. -D 2014-04-26T17:52:08.640 +C Update\srequirements\smarks\sto\sfix\stypos\sin\sthe\srequirements\stext.\nNo\schanges\sto\scode. +D 2014-04-26T19:23:14.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -433,12 +433,12 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test ee95d48664503d40f6cc9ef4a7d03216188e2ada +F test/e_createtable.test ed82efcedc4b3656b27a5fcd12335cdb7e20eeee F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 F test/e_expr.test 5c71d183fbf519a4769fd2e2124afdc70b5b1f42 -F test/e_fkey.test 630597377549af579d34faaf64c6959a5a68ef76 +F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 1e44f84d2abe44d66e4fbf198be4b20e3cc724a0 F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f8c4c495e6de1f124d205383d4bafa46accbff5c -R efe95b9fe528a9f472987911752f5742 +P 349f483499dd685a8da94923b6bd810a52e5e236 +R 310b081215d2804c69537a1d1f588735 U drh -Z 366467002d461344ec8032510917cbba +Z 3788b514428a54594de7c9ac65d6befd diff --git a/manifest.uuid b/manifest.uuid index 26e36016ef..b7714159cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -349f483499dd685a8da94923b6bd810a52e5e236 \ No newline at end of file +f5a263658187250044afc1a74000e6f6962733ca \ No newline at end of file diff --git a/test/e_createtable.test b/test/e_createtable.test index 2cd63280df..42fc017e2a 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -884,9 +884,10 @@ do_execsql_test e_createtable-3.3.1 { ); } {} -# EVIDENCE-OF: R-10288-43169 For the purposes of the DEFAULT clause, an +# EVIDENCE-OF: R-36381-62919 For the purposes of the DEFAULT clause, an # expression is considered constant provided that it does not contain -# any sub-queries or string constants enclosed in double quotes. +# any sub-queries, column or table references, or string literals +# enclosed in double-quotes instead of single-quotes. # do_createtable_tests 3.4.1 -error { default value of column [x] is not constant diff --git a/test/e_fkey.test b/test/e_fkey.test index 921e967176..09756505c3 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -135,9 +135,9 @@ reset_db # # This also tests that foreign key constraints are disabled by default. # -# EVIDENCE-OF: R-59578-04990 Foreign key constraints are disabled by +# EVIDENCE-OF: R-44261-39702 Foreign key constraints are disabled by # default (for backwards compatibility), so must be enabled separately -# for each database connection separately. +# for each database connection. # drop_all_tables do_test e_fkey-4.1 { @@ -163,9 +163,10 @@ do_test e_fkey-4.2 { } {world} #------------------------------------------------------------------------- -# EVIDENCE-OF: R-15278-54456 The application can can also use a PRAGMA +# EVIDENCE-OF: R-08013-37737 The application can also use a PRAGMA # foreign_keys statement to determine if foreign keys are currently # enabled. + # # This also tests the example code in section 2 of foreignkeys.in. # @@ -2990,8 +2991,8 @@ if {[clang_sanitize_address]==0} { # The setting of the recursive_triggers pragma does not affect foreign # key actions. # -# EVIDENCE-OF: R-51769-32730 The PRAGMA recursive_triggers setting does -# not not affect the operation of foreign key actions. +# EVIDENCE-OF: R-44355-00270 The PRAGMA recursive_triggers setting does +# not affect the operation of foreign key actions. # foreach recursive_triggers_setting [list 0 1 ON OFF] { drop_all_tables From 2dd3cdcfab89b47ee8b47260d0cdf73801094018 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 26 Apr 2014 20:21:14 +0000 Subject: [PATCH 12/36] Fix an sqlite3_stmt_status() problem caused by recent changs on this branch. FossilOrigin-Name: dee204092421a239f9f60ab83c3a5b3e24d1baea --- manifest | 15 +++++----- manifest.uuid | 2 +- src/where.c | 10 ++++--- test/cost.test | 68 ++++++++++++++++++++++++++++++++++++++++++++++ test/orderby5.test | 4 +-- 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 test/cost.test diff --git a/manifest b/manifest index afdc8889fe..c32bc38c65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\suser\sprovides\slikelihood()\sdata\sfor\sa\sWHERE\sclause\sterm\sused\sas\spart\sof\san\sindex\skey,\shave\sthe\splanner\suse\sit\swhen\scalculating\sthe\sexpected\snumber\sof\srows\svisited\sby\sthe\sloop. -D 2014-04-25T20:22:45.291 +C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. +D 2014-04-26T20:21:14.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2b3f47801939c2853c03bba3d5aa83abdd51211e +F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,6 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -722,7 +723,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1bd74c49ddab6f53bb6eaa57907eff44c2580dd6 -R 804d3aff8175381c892eb4eebbb86307 +P c51efaa5d29ee0a91b9e6a83a8dd82530670811a +R 95df5cff60bc1a7189139d237d77a9d7 U dan -Z 6dd1c2a2c429274ab108441094bd0963 +Z c6c47e040169e1387f5404e0d25c212f diff --git a/manifest.uuid b/manifest.uuid index 71ddf4b2d3..22512678d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c51efaa5d29ee0a91b9e6a83a8dd82530670811a \ No newline at end of file +dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 73d3a22b7c..6d924e0782 100644 --- a/src/where.c +++ b/src/where.c @@ -4147,14 +4147,15 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags |= WHERE_ONEROW; } } - + }else if( eOp & WO_ISNULL ){ + pNew->wsFlags |= WHERE_COLUMN_NULL; }else if( eOp & (WO_GT|WO_GE) ){ testcase( eOp & WO_GT ); testcase( eOp & WO_GE ); pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; pBtm = pTerm; pTop = 0; - }else if( (eOp & WO_ISNULL)==0 ){ + }else{ assert( eOp & (WO_LT|WO_LE) ); testcase( eOp & WO_LT ); testcase( eOp & WO_LE ); @@ -5225,9 +5226,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** scans instead? */ LogEst rScale, rSortCost; - assert( nOrderBy>0 ); + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale; + rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; + /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ diff --git a/test/cost.test b/test/cost.test new file mode 100644 index 0000000000..8df32d8ed1 --- /dev/null +++ b/test/cost.test @@ -0,0 +1,68 @@ +# 2014-04-26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix cost + + +do_execsql_test 1.1 { + CREATE TABLE t3(id INTEGER PRIMARY KEY, b NOT NULL); + CREATE TABLE t4(c, d, e); + CREATE UNIQUE INDEX i3 ON t3(b); + CREATE UNIQUE INDEX i4 ON t4(c, d); +} +do_eqp_test 1.2 { + SELECT e FROM t3, t4 WHERE b=c ORDER BY b, d; +} { + 0 0 0 {SCAN TABLE t3 USING COVERING INDEX i3} + 0 1 1 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + + +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); +} + +# It is better to use an index for ORDER BY than sort externally, even +# if the index is a non-covering index. +do_eqp_test 2.2 { + SELECT * FROM t1 ORDER BY a; +} { + 0 0 0 {SCAN TABLE t1 USING INDEX i1} +} + +do_execsql_test 3.1 { + CREATE TABLE t5(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t5b ON t5(b); + CREATE INDEX t5c ON t5(c); + CREATE INDEX t5d ON t5(d); + CREATE INDEX t5e ON t5(e); + CREATE INDEX t5f ON t5(f); + CREATE INDEX t5g ON t5(g); +} + +do_eqp_test 3.2 { + SELECT a FROM t5 + WHERE b IS NULL OR c IS NULL OR d IS NULL + ORDER BY a; +} { + 0 0 0 {SEARCH TABLE t5 USING INDEX t5b (b=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5c (c=?)} + 0 0 0 {SEARCH TABLE t5 USING INDEX t5d (d=?)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + + +finish_test diff --git a/test/orderby5.test b/test/orderby5.test index bccd469f25..c9cce703ba 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -80,12 +80,12 @@ do_execsql_test 2.1a { EXPLAIN QUERY PLAN SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; } {~/B-TREE/} + do_execsql_test 2.1b { EXPLAIN QUERY PLAN - SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; + SELECT * FROM t1 WHERE likelihood(a=0, 0.05) ORDER BY a, b, c; } {/B-TREE/} - do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; From 440e6ff3b023b66ecefa545990d5eeb85bda03ea Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 08:49:54 +0000 Subject: [PATCH 13/36] Fix an error in estimating of the number of rows visited by a range scan. FossilOrigin-Name: d491de62fce69d93e89f65f7713972f7c2c451f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 7 ++++++- test/cost.test | 27 +++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c32bc38c65..2a1d6b7303 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\ssqlite3_stmt_status()\sproblem\scaused\sby\srecent\schangs\son\sthis\sbranch. -D 2014-04-26T20:21:14.140 +C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. +D 2014-04-28T08:49:54.584 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 488cc018a39d33d97555b5a1e079dfafc32d348a +F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 8a2b846e896c744b940c29e8bab46416a1324717 +F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c51efaa5d29ee0a91b9e6a83a8dd82530670811a -R 95df5cff60bc1a7189139d237d77a9d7 +P dee204092421a239f9f60ab83c3a5b3e24d1baea +R 3b87cec4307de74fea71def903d4f518 U dan -Z c6c47e040169e1387f5404e0d25c212f +Z dce0ef4becf30f1f5b6dab0637076369 diff --git a/manifest.uuid b/manifest.uuid index 22512678d7..94e48b42ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dee204092421a239f9f60ab83c3a5b3e24d1baea \ No newline at end of file +d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6d924e0782..dd80ab4ccb 100644 --- a/src/where.c +++ b/src/where.c @@ -4241,7 +4241,12 @@ static int whereLoopAddBtreeIndex( pNew->nOut += nInMul + nIn; whereLoopOutputAdjust(pBuilder->pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); - pNew->nOut = nOutUnadjusted; + + if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ + pNew->nOut = saved_nOut; + }else{ + pNew->nOut = nOutUnadjusted; + } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) diff --git a/test/cost.test b/test/cost.test index 8df32d8ed1..faaa7bdf16 100644 --- a/test/cost.test +++ b/test/cost.test @@ -63,6 +63,33 @@ do_eqp_test 3.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } +#------------------------------------------------------------------------- +# If there is no likelihood() or stat3 data, SQLite assumes that a closed +# range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) +# visits 1/16 of the rows in a table. +# +# Note: 1/17 =~ 0.058 +# Note: 1/15 =~ 0.067 +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); +} +do_eqp_test 4.2 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} +} +do_eqp_test 4.3 { + SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b Date: Mon, 28 Apr 2014 09:35:31 +0000 Subject: [PATCH 14/36] Modify internal function whereLoopAdjustCost() so that it does not prefer a skip-scan over a regular index scan even if the regular scan uses a subset of the WHERE terms used by the skip-scan. FossilOrigin-Name: 88a5758dcce891eb7be15432ebdc9f80071d413b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2a1d6b7303..b301a439d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sestimating\sof\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-04-28T08:49:54.584 +C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. +D 2014-04-28T09:35:31.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4efd09b7b2c3604e48b015199f5393578ed417b +F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dee204092421a239f9f60ab83c3a5b3e24d1baea -R 3b87cec4307de74fea71def903d4f518 +P d491de62fce69d93e89f65f7713972f7c2c451f7 +R a36e9b880c1d76e2d8cd1072dd5871be U dan -Z dce0ef4becf30f1f5b6dab0637076369 +Z 07995bb3734998fef6b8bca04739721a diff --git a/manifest.uuid b/manifest.uuid index 94e48b42ff..8be55d6794 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d491de62fce69d93e89f65f7713972f7c2c451f7 \ No newline at end of file +88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file diff --git a/src/where.c b/src/where.c index dd80ab4ccb..4c1d81e183 100644 --- a/src/where.c +++ b/src/where.c @@ -3779,9 +3779,11 @@ static int whereLoopCheaperProperSubset( */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; + if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; + if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ From b51926e67c4921137cac5e3b66257a445f23ad5f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 10:00:59 +0000 Subject: [PATCH 15/36] Update test script analyze3.test to account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze3.test | 42 ++++++++++++++++++++++++++++++------------ 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b301a439d1..06eee5c7d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sinternal\sfunction\swhereLoopAdjustCost()\sso\sthat\sit\sdoes\snot\sprefer\sa\sskip-scan\sover\sa\sregular\sindex\sscan\seven\sif\sthe\sregular\sscan\suses\sa\ssubset\sof\sthe\sWHERE\sterms\sused\sby\sthe\sskip-scan. -D 2014-04-28T09:35:31.541 +C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T10:00:59.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -305,7 +305,7 @@ F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 -F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d +F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d491de62fce69d93e89f65f7713972f7c2c451f7 -R a36e9b880c1d76e2d8cd1072dd5871be +P 88a5758dcce891eb7be15432ebdc9f80071d413b +R a1e1c4a3f11d62d0bb62f198394034d8 U dan -Z 07995bb3734998fef6b8bca04739721a +Z fd306a356911acfe444509c2efb0dade diff --git a/manifest.uuid b/manifest.uuid index 8be55d6794..5b247aefe3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -88a5758dcce891eb7be15432ebdc9f80071d413b \ No newline at end of file +35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file diff --git a/test/analyze3.test b/test/analyze3.test index fb26303ee3..e7416d5730 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -103,12 +103,21 @@ do_test analyze3-1.1.1 { } } {1} +do_execsql_test analyze3-1.1.x { + SELECT count(*) FROM t1 WHERE x>200 AND x<300; + SELECT count(*) FROM t1 WHERE x>0 AND x<1100; +} {99 1000} + +# The first of the following two SELECT statements visits 99 rows. So +# it is better to use the index. But the second visits every row in +# the table (1000 in total) so it is better to do a full-table scan. +# do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x200 AND x<300 } @@ -125,17 +134,17 @@ do_test analyze3-1.1.6 { } {199 0 14850} do_test analyze3-1.1.7 { sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.1.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} # The following tests are similar to the block above. The difference is @@ -152,12 +161,17 @@ do_test analyze3-1.2.1 { ANALYZE; } } {} +do_execsql_test analyze3-2.1.x { + SELECT count(*) FROM t2 WHERE x>1 AND x<2; + SELECT count(*) FROM t2 WHERE x>0 AND x<99; +} {200 990} do_eqp_test analyze3-1.2.2 { SELECT sum(y) FROM t2 WHERE x>1 AND x<2 } {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x0 AND x<99 -} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x12 AND x<20 } } {161 0 4760} @@ -173,17 +187,17 @@ do_test analyze3-1.2.6 { } {161 0 integer integer 4760} do_test analyze3-1.2.7 { sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 } -} {1981 0 490555} +} {999 999 490555} do_test analyze3-1.2.8 { set l [string range "0" 0 end] set u [string range "99" 0 end] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 text text 490555} +} {999 999 text text 490555} do_test analyze3-1.2.9 { set l [expr int(0)] set u [expr int(99)] sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u} -} {1981 0 integer integer 490555} +} {999 999 integer integer 490555} # Same tests a third time. This time, column x has INTEGER affinity and # is not the leftmost column of the table. This triggered a bug causing @@ -199,12 +213,16 @@ do_test analyze3-1.3.1 { ANALYZE; } } {} +do_execsql_test analyze3-1.3.x { + SELECT count(*) FROM t3 WHERE x>200 AND x<300; + SELECT count(*) FROM t3 WHERE x>0 AND x<1100 +} {99 1000} do_eqp_test analyze3-1.3.2 { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 } {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x0 AND x<1100 -} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x200 AND x<300 } @@ -221,17 +239,17 @@ do_test analyze3-1.3.6 { } {199 0 14850} do_test analyze3-1.3.7 { sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.8 { set l [string range "0" 0 end] set u [string range "1100" 0 end] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} do_test analyze3-1.3.9 { set l [expr int(0)] set u [expr int(1100)] sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u } -} {2000 0 499500} +} {999 999 499500} #------------------------------------------------------------------------- # Test that the values of bound SQL variables may be used for the LIKE From 9881d60d17451055282b2bde12454fa42aeaabc8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 12:08:23 +0000 Subject: [PATCH 16/36] Add an extra column to a table in analyze9.test to give the planner a little more reason to select an index. FossilOrigin-Name: 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 06eee5c7d2..04a8c1374d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\stest\sscript\sanalyze3.test\sto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T10:00:59.075 +C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. +D 2014-04-28T12:08:23.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -311,7 +311,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test e219daa58fd8677c6a43d771798cf37d68f51d3e +F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 88a5758dcce891eb7be15432ebdc9f80071d413b -R a1e1c4a3f11d62d0bb62f198394034d8 +P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 +R 8aefc1341eaf7a6ece9efb278712f5a6 U dan -Z fd306a356911acfe444509c2efb0dade +Z ba7221d22e50ff31dad54b0ec475658f diff --git a/manifest.uuid b/manifest.uuid index 5b247aefe3..7dba22fdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35f46a55d866b9a87c1321aab8e0cfe86ccadb93 \ No newline at end of file +1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 125cecf182..f25e5924e6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -566,7 +566,7 @@ foreach {tn schema} { drop_all_tables do_test 13.1 { execsql { - CREATE TABLE t1(a, b, c); + CREATE TABLE t1(a, b, c, d); CREATE INDEX i1 ON t1(a); CREATE INDEX i2 ON t1(b, c); } From 6b6828625b2843a719fb123dc46feb5666b2cda3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 28 Apr 2014 15:11:25 +0000 Subject: [PATCH 17/36] Update unordered.test to take into account for the fact that SQLite now prefers a full-table scan over a non-covering index scan that visits a large percentage of the table rows. FossilOrigin-Name: 20f468dfbcb247e51446fad411a6e6cc0d130411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/unordered.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 04a8c1374d..05f3d798c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\scolumn\sto\sa\stable\sin\sanalyze9.test\sto\sgive\sthe\splanner\sa\slittle\smore\sreason\sto\sselect\san\sindex. -D 2014-04-28T12:08:23.821 +C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. +D 2014-04-28T15:11:25.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1023,7 +1023,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 -F test/unordered.test ef85ac8f2f3c93ed2b9e811b684de73175fc464c +F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 35f46a55d866b9a87c1321aab8e0cfe86ccadb93 -R 8aefc1341eaf7a6ece9efb278712f5a6 +P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 +R 307957877e44688fb3b83b922c0183e6 U dan -Z ba7221d22e50ff31dad54b0ec475658f +Z 085ae09cb0f2460f340245b5be0529cf diff --git a/manifest.uuid b/manifest.uuid index 7dba22fdc4..8fd0bd92ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 \ No newline at end of file +20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file diff --git a/test/unordered.test b/test/unordered.test index d8f7aa6add..147e91f0d9 100644 --- a/test/unordered.test +++ b/test/unordered.test @@ -42,7 +42,7 @@ foreach idxmode {ordered unordered} { 1 "SELECT * FROM t1 ORDER BY a" {0 0 0 {SCAN TABLE t1 USING INDEX i1}} {0 0 0 {SCAN TABLE t1} 0 0 0 {USE TEMP B-TREE FOR ORDER BY}} - 2 "SELECT * FROM t1 WHERE a >?" + 2 "SELECT * FROM t1 WHERE a > 100" {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a>?)}} {0 0 0 {SCAN TABLE t1}} 3 "SELECT * FROM t1 WHERE a = ? ORDER BY rowid" From 65e6b0dd1266cf5178b1e7b432cdd9165ef2a1f6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Apr 2014 17:56:19 +0000 Subject: [PATCH 18/36] Add the sqlite3_rtree_query_callback() API to the RTree virtual table. (Cherrypick from the sessions branch.) FossilOrigin-Name: af2cbe64adab5f9e3b0f3da00d06428088589d7f --- ext/rtree/rtree.c | 1496 ++++++++++++++++---------------- ext/rtree/rtree1.test | 19 +- ext/rtree/rtree6.test | 18 +- ext/rtree/rtreeB.test | 2 +- ext/rtree/rtreeC.test | 13 +- ext/rtree/rtreeE.test | 129 +++ ext/rtree/sqlite3rtree.h | 67 +- main.mk | 2 +- manifest | 33 +- manifest.uuid | 2 +- src/test_rtree.c | 200 ++++- src/test_vfstrace.c | 2 +- test/show_speedtest1_rtree.tcl | 57 ++ test/speedtest1.c | 186 +++- 14 files changed, 1425 insertions(+), 801 deletions(-) create mode 100644 ext/rtree/rtreeE.test create mode 100644 test/show_speedtest1_rtree.tcl diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index cefb9a8b2a..7b540b4be1 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -54,48 +54,6 @@ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RTREE) -/* -** This file contains an implementation of a couple of different variants -** of the r-tree algorithm. See the README file for further details. The -** same data-structure is used for all, but the algorithms for insert and -** delete operations vary. The variants used are selected at compile time -** by defining the following symbols: -*/ - -/* Either, both or none of the following may be set to activate -** r*tree variant algorithms. -*/ -#define VARIANT_RSTARTREE_CHOOSESUBTREE 0 -#define VARIANT_RSTARTREE_REINSERT 1 - -/* -** Exactly one of the following must be set to 1. -*/ -#define VARIANT_GUTTMAN_QUADRATIC_SPLIT 0 -#define VARIANT_GUTTMAN_LINEAR_SPLIT 0 -#define VARIANT_RSTARTREE_SPLIT 1 - -#define VARIANT_GUTTMAN_SPLIT \ - (VARIANT_GUTTMAN_LINEAR_SPLIT||VARIANT_GUTTMAN_QUADRATIC_SPLIT) - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT - #define PickNext QuadraticPickNext - #define PickSeeds QuadraticPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_GUTTMAN_LINEAR_SPLIT - #define PickNext LinearPickNext - #define PickSeeds LinearPickSeeds - #define AssignCells splitNodeGuttman -#endif -#if VARIANT_RSTARTREE_SPLIT - #define AssignCells splitNodeStartree -#endif - -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) -# define NDEBUG 1 -#endif - #ifndef SQLITE_CORE #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 @@ -105,11 +63,13 @@ #include #include +#include #ifndef SQLITE_AMALGAMATION #include "sqlite3rtree.h" typedef sqlite3_int64 i64; typedef unsigned char u8; +typedef unsigned short u16; typedef unsigned int u32; #endif @@ -127,6 +87,7 @@ typedef struct RtreeConstraint RtreeConstraint; typedef struct RtreeMatchArg RtreeMatchArg; typedef struct RtreeGeomCallback RtreeGeomCallback; typedef union RtreeCoord RtreeCoord; +typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 @@ -135,7 +96,7 @@ typedef union RtreeCoord RtreeCoord; ** ever contain very many entries, so a fixed number of buckets is ** used. */ -#define HASHSIZE 128 +#define HASHSIZE 97 /* The xBestIndex method of this virtual table requires an estimate of ** the number of rows in the virtual table to calculate the costs of @@ -151,15 +112,15 @@ typedef union RtreeCoord RtreeCoord; ** An rtree virtual-table object. */ struct Rtree { - sqlite3_vtab base; + sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ - int nDim; /* Number of dimensions */ - int nBytesPerCell; /* Bytes consumed per cell */ + u8 nDim; /* Number of dimensions */ + u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ + u8 nBytesPerCell; /* Bytes consumed per cell */ int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ char *zName; /* Name of r-tree table */ - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ int nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ @@ -186,10 +147,10 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - int eCoordType; + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; -/* Possible values for eCoordType: */ +/* Possible values for Rtree.eCoordType: */ #define RTREE_COORD_REAL32 0 #define RTREE_COORD_INT32 1 @@ -201,11 +162,30 @@ struct Rtree { #ifdef SQLITE_RTREE_INT_ONLY typedef sqlite3_int64 RtreeDValue; /* High accuracy coordinate */ typedef int RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0 #else typedef double RtreeDValue; /* High accuracy coordinate */ typedef float RtreeValue; /* Low accuracy coordinate */ +# define RTREE_ZERO 0.0 #endif +/* +** When doing a search of an r-tree, instances of the following structure +** record intermediate results from the tree walk. +** +** The id is always a node-id. For iLevel>=1 the id is the node-id of +** the node that the RtreeSearchPoint represents. When iLevel==0, however, +** the id is of the parent node and the cell that RtreeSearchPoint +** represents is the iCell-th entry in the parent node. +*/ +struct RtreeSearchPoint { + RtreeDValue rScore; /* The score for this node. Smallest goes first. */ + sqlite3_int64 id; /* Node ID */ + u8 iLevel; /* 0=entries. 1=leaf node. 2+ for higher */ + u8 eWithin; /* PARTLY_WITHIN or FULLY_WITHIN */ + u8 iCell; /* Cell index within the node */ +}; + /* ** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: @@ -228,21 +208,44 @@ struct Rtree { */ #define RTREE_MAX_DEPTH 40 + +/* +** Number of entries in the cursor RtreeNode cache. The first entry is +** used to cache the RtreeNode for RtreeCursor.sPoint. The remaining +** entries cache the RtreeNode for the first elements of the priority queue. +*/ +#define RTREE_CACHE_SZ 5 + /* ** An rtree cursor object. */ struct RtreeCursor { - sqlite3_vtab_cursor base; - RtreeNode *pNode; /* Node cursor is currently pointing at */ - int iCell; /* Index of current cell in pNode */ + sqlite3_vtab_cursor base; /* Base class. Must be first */ + u8 atEOF; /* True if at end of search */ + u8 bPoint; /* True if sPoint is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ + int nPointAlloc; /* Number of slots allocated for aPoint[] */ + int nPoint; /* Number of slots used in aPoint[] */ + int mxLevel; /* iLevel value for root of the tree */ + RtreeSearchPoint *aPoint; /* Priority queue for search points */ + RtreeSearchPoint sPoint; /* Cached next search point */ + RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ + u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ }; +/* Return the Rtree of a RtreeCursor */ +#define RTREE_OF_CURSOR(X) ((Rtree*)((X)->base.pVtab)) + +/* +** A coordinate can be either a floating point number or a integer. All +** coordinates within a single R-Tree are always of the same time. +*/ union RtreeCoord { - RtreeValue f; - int i; + RtreeValue f; /* Floating point value */ + int i; /* Integer value */ + u32 u; /* Unsigned for byte-order conversions */ }; /* @@ -267,38 +270,67 @@ union RtreeCoord { struct RtreeConstraint { int iCoord; /* Index of constrained coordinate */ int op; /* Constraining operation */ - RtreeDValue rValue; /* Constraint value. */ - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - sqlite3_rtree_geometry *pGeom; /* Constraint callback argument for a MATCH */ + union { + RtreeDValue rValue; /* Constraint value. */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + } u; + sqlite3_rtree_query_info *pInfo; /* xGeom and xQueryFunc argument */ }; /* Possible values for RtreeConstraint.op */ -#define RTREE_EQ 0x41 -#define RTREE_LE 0x42 -#define RTREE_LT 0x43 -#define RTREE_GE 0x44 -#define RTREE_GT 0x45 -#define RTREE_MATCH 0x46 +#define RTREE_EQ 0x41 /* A */ +#define RTREE_LE 0x42 /* B */ +#define RTREE_LT 0x43 /* C */ +#define RTREE_GE 0x44 /* D */ +#define RTREE_GT 0x45 /* E */ +#define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ +#define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ + /* ** An rtree structure node. */ struct RtreeNode { - RtreeNode *pParent; /* Parent node */ - i64 iNode; - int nRef; - int isDirty; - u8 *zData; - RtreeNode *pNext; /* Next node in this hash chain */ + RtreeNode *pParent; /* Parent node */ + i64 iNode; /* The node number */ + int nRef; /* Number of references to this node */ + int isDirty; /* True if the node needs to be written to disk */ + u8 *zData; /* Content of the node, as should be on disk */ + RtreeNode *pNext; /* Next node in this hash collision chain */ }; + +/* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) /* -** Structure to store a deserialized rtree record. +** A single cell from a node, deserialized */ struct RtreeCell { - i64 iRowid; - RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; + i64 iRowid; /* Node or entry ID */ + RtreeCoord aCoord[RTREE_MAX_DIMENSIONS*2]; /* Bounding box coordinates */ +}; + + +/* +** This object becomes the sqlite3_user_data() for the SQL functions +** that are created by sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() and which appear on the right of MATCH +** operators in order to constrain a search. +** +** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc fields is non-NULL, depending on whether the +** SQL function was created using sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback(). +** +** This object is deleted automatically by the destructor mechanism in +** sqlite3_create_function_v2(). +*/ +struct RtreeGeomCallback { + int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); + int (*xQueryFunc)(sqlite3_rtree_query_info*); + void (*xDestructor)(void*); + void *pContext; }; @@ -310,29 +342,16 @@ struct RtreeCell { #define RTREE_GEOMETRY_MAGIC 0x891245AB /* -** An instance of this structure must be supplied as a blob argument to -** the right-hand-side of an SQL MATCH operator used to constrain an -** r-tree query. +** An instance of this structure (in the form of a BLOB) is returned by +** the SQL functions that sqlite3_rtree_geometry_callback() and +** sqlite3_rtree_query_callback() create, and is read as the right-hand +** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue*, int *); - void *pContext; - int nParam; - RtreeDValue aParam[1]; -}; - -/* -** When a geometry callback is created (see sqlite3_rtree_geometry_callback), -** a single instance of the following structure is allocated. It is used -** as the context for the user-function created by by s_r_g_c(). The object -** is eventually deleted by the destructor mechanism provided by -** sqlite3_create_function_v2() (which is called by s_r_g_c() to create -** the geometry callback function). -*/ -struct RtreeGeomCallback { - int (*xGeom)(sqlite3_rtree_geometry*, int, RtreeDValue*, int*); - void *pContext; + u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + RtreeGeomCallback cb; /* Info about the callback functions */ + int nParam; /* Number of parameters to the SQL function */ + RtreeDValue aParam[1]; /* Values for parameters to the SQL function */ }; #ifndef MAX @@ -426,10 +445,7 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** in the Rtree.aHash table. */ static int nodeHash(i64 iNode){ - return ( - (iNode>>56) ^ (iNode>>48) ^ (iNode>>40) ^ (iNode>>32) ^ - (iNode>>24) ^ (iNode>>16) ^ (iNode>> 8) ^ (iNode>> 0) - ) % HASHSIZE; + return iNode % HASHSIZE; } /* @@ -489,8 +505,7 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ /* ** Obtain a reference to an r-tree node. */ -static int -nodeAcquire( +static int nodeAcquire( Rtree *pRtree, /* R-tree structure */ i64 iNode, /* Node number to load */ RtreeNode *pParent, /* Either the parent node or NULL */ @@ -579,10 +594,10 @@ nodeAcquire( ** Overwrite cell iCell of node pNode with the contents of pCell. */ static void nodeOverwriteCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node into which the cell is to be written */ + RtreeCell *pCell, /* The cell to write */ + int iCell /* Index into pNode into which pCell is written */ ){ int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -594,7 +609,7 @@ static void nodeOverwriteCell( } /* -** Remove cell the cell with index iCell from node pNode. +** Remove the cell with index iCell from node pNode. */ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ u8 *pDst = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; @@ -611,11 +626,10 @@ static void nodeDeleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell){ ** ** If there is not enough free space in pNode, return SQLITE_FULL. */ -static int -nodeInsertCell( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell +static int nodeInsertCell( + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* Write new cell into this node */ + RtreeCell *pCell /* The cell to be inserted */ ){ int nCell; /* Current number of cells in pNode */ int nMaxCell; /* Maximum number of cells for pNode */ @@ -636,8 +650,7 @@ nodeInsertCell( /* ** If the node is dirty, write it out to the database. */ -static int -nodeWrite(Rtree *pRtree, RtreeNode *pNode){ +static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode->isDirty ){ sqlite3_stmt *p = pRtree->pWriteNode; @@ -662,8 +675,7 @@ nodeWrite(Rtree *pRtree, RtreeNode *pNode){ ** Release a reference to a node. If the node is dirty and the reference ** count drops to zero, the node data is written to the database. */ -static int -nodeRelease(Rtree *pRtree, RtreeNode *pNode){ +static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); @@ -691,9 +703,9 @@ nodeRelease(Rtree *pRtree, RtreeNode *pNode){ ** an internal node, then the 64-bit integer is a child page number. */ static i64 nodeGetRowid( - Rtree *pRtree, - RtreeNode *pNode, - int iCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract the ID */ + int iCell /* The cell index from which to extract the ID */ ){ assert( iCellzData[4 + pRtree->nBytesPerCell*iCell]); @@ -703,11 +715,11 @@ static i64 nodeGetRowid( ** Return coordinate iCoord from cell iCell in node pNode. */ static void nodeGetCoord( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - int iCoord, - RtreeCoord *pCoord /* Space to write result to */ + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node from which to extract a coordinate */ + int iCell, /* The index of the cell within the node */ + int iCoord, /* Which coordinate to extract */ + RtreeCoord *pCoord /* OUT: Space to write result to */ ){ readCoord(&pNode->zData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -717,15 +729,20 @@ static void nodeGetCoord( ** to by pCell with the results. */ static void nodeGetCell( - Rtree *pRtree, - RtreeNode *pNode, - int iCell, - RtreeCell *pCell + Rtree *pRtree, /* The overall R-Tree */ + RtreeNode *pNode, /* The node containing the cell to be read */ + int iCell, /* Index of the cell within the node */ + RtreeCell *pCell /* OUT: Write the cell contents here */ ){ - int ii; + u8 *pData; + u8 *pEnd; + RtreeCoord *pCoord; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); - for(ii=0; iinDim*2; ii++){ - nodeGetCoord(pRtree, pNode, iCell, ii, &pCell->aCoord[ii]); + pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); + pEnd = pData + pRtree->nDim*8; + pCoord = pCell->aCoord; + for(; pDataaConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ - sqlite3_rtree_geometry *pGeom = pCsr->aConstraint[i].pGeom; - if( pGeom ){ - if( pGeom->xDelUser ) pGeom->xDelUser(pGeom->pUser); - sqlite3_free(pGeom); + sqlite3_rtree_query_info *pInfo = pCsr->aConstraint[i].pInfo; + if( pInfo ){ + if( pInfo->xDelUser ) pInfo->xDelUser(pInfo->pUser); + sqlite3_free(pInfo); } } sqlite3_free(pCsr->aConstraint); @@ -867,12 +884,13 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int rc; + int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; freeCursorConstraints(pCsr); - rc = nodeRelease(pRtree, pCsr->pNode); + sqlite3_free(pCsr->aPoint); + for(ii=0; iiaNode[ii]); sqlite3_free(pCsr); - return rc; + return SQLITE_OK; } /* @@ -883,196 +901,166 @@ static int rtreeClose(sqlite3_vtab_cursor *cur){ */ static int rtreeEof(sqlite3_vtab_cursor *cur){ RtreeCursor *pCsr = (RtreeCursor *)cur; - return (pCsr->pNode==0); + return pCsr->atEOF; } /* -** The r-tree constraint passed as the second argument to this function is -** guaranteed to be a MATCH constraint. -*/ -static int testRtreeGeom( - Rtree *pRtree, /* R-Tree object */ - RtreeConstraint *pConstraint, /* MATCH constraint to test */ - RtreeCell *pCell, /* Cell to test */ - int *pbRes /* OUT: Test result */ -){ - int i; - RtreeDValue aCoord[RTREE_MAX_DIMENSIONS*2]; - int nCoord = pRtree->nDim*2; - - assert( pConstraint->op==RTREE_MATCH ); - assert( pConstraint->pGeom ); - - for(i=0; iaCoord[i]); - } - return pConstraint->xGeom(pConstraint->pGeom, nCoord, aCoord, pbRes); -} - -/* -** Cursor pCursor currently points to a cell in a non-leaf page. -** Set *pbEof to true if the sub-tree headed by the cell is filtered -** (excluded) by the constraints in the pCursor->aConstraint[] -** array, or false otherwise. +** Convert raw bits from the on-disk RTree record into a coordinate value. +** The on-disk format is big-endian and needs to be converted for little- +** endian platforms. The on-disk record stores integer coordinates if +** eInt is true and it stores 32-bit floating point records if eInt is +** false. a[] is the four bytes of the on-disk record to be decoded. +** Store the results in "r". ** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. +** There are three versions of this macro, one each for little-endian and +** big-endian processors and a third generic implementation. The endian- +** specific implementations are much faster and are preferred if the +** processor endianness is known at compile-time. The SQLITE_BYTEORDER +** macro is part of sqliteInt.h and hence the endian-specific +** implementation will only be used if this module is compiled as part +** of the amalgamation. */ -static int testRtreeCell(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - int bRes = 0; - int rc = SQLITE_OK; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; bRes==0 && iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue cell_min = DCOORD(cell.aCoord[(p->iCoord>>1)*2]); - RtreeDValue cell_max = DCOORD(cell.aCoord[(p->iCoord>>1)*2+1]); - - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - - switch( p->op ){ - case RTREE_LE: case RTREE_LT: - bRes = p->rValuerValue>cell_max; - break; - - case RTREE_EQ: - bRes = (p->rValue>cell_max || p->rValueop==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &bRes); - bRes = !bRes; - break; - } - } - } - - *pbEof = bRes; - return rc; +#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + c.u = ((c.u>>24)&0xff)|((c.u>>8)&0xff00)| \ + ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } - -/* -** Test if the cell that cursor pCursor currently points to -** would be filtered (excluded) by the constraints in the -** pCursor->aConstraint[] array. If so, set *pbEof to true before -** returning. If the cell is not filtered (excluded) by the constraints, -** set pbEof to zero. -** -** Return SQLITE_OK if successful or an SQLite error code if an error -** occurs within a geometry callback. -** -** This function assumes that the cell is part of a leaf node. -*/ -static int testRtreeEntry(Rtree *pRtree, RtreeCursor *pCursor, int *pbEof){ - RtreeCell cell; - int ii; - *pbEof = 0; - - nodeGetCell(pRtree, pCursor->pNode, pCursor->iCell, &cell); - for(ii=0; iinConstraint; ii++){ - RtreeConstraint *p = &pCursor->aConstraint[ii]; - RtreeDValue coord = DCOORD(cell.aCoord[p->iCoord]); - int res; - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_MATCH - ); - switch( p->op ){ - case RTREE_LE: res = (coord<=p->rValue); break; - case RTREE_LT: res = (coordrValue); break; - case RTREE_GE: res = (coord>=p->rValue); break; - case RTREE_GT: res = (coord>p->rValue); break; - case RTREE_EQ: res = (coord==p->rValue); break; - default: { - int rc; - assert( p->op==RTREE_MATCH ); - rc = testRtreeGeom(pRtree, p, &cell, &res); - if( rc!=SQLITE_OK ){ - return rc; - } - break; - } - } - - if( !res ){ - *pbEof = 1; - return SQLITE_OK; - } - } - - return SQLITE_OK; +#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + memcpy(&c.u,a,4); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } +#else +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = ((u32)a[0]<<24) + ((u32)a[1]<<16) \ + +((u32)a[2]<<8) + a[3]; \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#endif /* -** Cursor pCursor currently points at a node that heads a sub-tree of -** height iHeight (if iHeight==0, then the node is a leaf). Descend -** to point to the left-most cell of the sub-tree that matches the -** configured constraints. +** Check the RTree node or entry given by pCellData and p against the MATCH +** constraint pConstraint. */ -static int descendToCell( - Rtree *pRtree, - RtreeCursor *pCursor, - int iHeight, - int *pEof /* OUT: Set to true if cannot descend */ +static int rtreeCallbackConstraint( + RtreeConstraint *pConstraint, /* The constraint to test */ + int eInt, /* True if RTree holding integer coordinates */ + u8 *pCellData, /* Raw cell content */ + RtreeSearchPoint *pSearch, /* Container of this cell */ + sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ + int *peWithin /* OUT: visibility of the cell */ ){ - int isEof; - int rc; - int ii; - RtreeNode *pChild; - sqlite3_int64 iRowid; + int i; /* Loop counter */ + sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ + int nCoord = pInfo->nCoord; /* No. of coordinates */ + int rc; /* Callback return code */ + sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ - RtreeNode *pSavedNode = pCursor->pNode; - int iSavedCell = pCursor->iCell; + assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); + assert( nCoord==2 || nCoord==4 || nCoord==6 || nCoord==8 || nCoord==10 ); - assert( iHeight>=0 ); - - if( iHeight==0 ){ - rc = testRtreeEntry(pRtree, pCursor, &isEof); + if( pConstraint->op==RTREE_QUERY && pSearch->iLevel==1 ){ + pInfo->iRowid = readInt64(pCellData); + } + pCellData += 8; + for(i=0; iop==RTREE_MATCH ){ + rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, + nCoord, aCoord, &i); + if( i==0 ) *peWithin = NOT_WITHIN; + *prScore = RTREE_ZERO; }else{ - rc = testRtreeCell(pRtree, pCursor, &isEof); - } - if( rc!=SQLITE_OK || isEof || iHeight==0 ){ - goto descend_to_cell_out; - } - - iRowid = nodeGetRowid(pRtree, pCursor->pNode, pCursor->iCell); - rc = nodeAcquire(pRtree, iRowid, pCursor->pNode, &pChild); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; - } - - nodeRelease(pRtree, pCursor->pNode); - pCursor->pNode = pChild; - isEof = 1; - for(ii=0; isEof && iiiCell = ii; - rc = descendToCell(pRtree, pCursor, iHeight-1, &isEof); - if( rc!=SQLITE_OK ){ - goto descend_to_cell_out; + pInfo->aCoord = aCoord; + pInfo->iLevel = pSearch->iLevel - 1; + pInfo->rScore = pInfo->rParentScore = pSearch->rScore; + pInfo->eWithin = pInfo->eParentWithin = pSearch->eWithin; + rc = pConstraint->u.xQueryFunc(pInfo); + if( pInfo->eWithin<*peWithin ) *peWithin = pInfo->eWithin; + if( pInfo->rScore<*prScore || *prScorerScore; } } - - if( isEof ){ - assert( pCursor->pNode==pChild ); - nodeReference(pSavedNode); - nodeRelease(pRtree, pChild); - pCursor->pNode = pSavedNode; - pCursor->iCell = iSavedCell; - } - -descend_to_cell_out: - *pEof = isEof; return rc; } +/* +** Check the internal RTree node given by pCellData against constraint p. +** If this constraint cannot be satisfied by any child within the node, +** set *peWithin to NOT_WITHIN. +*/ +static void rtreeNonleafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + sqlite3_rtree_dbl val; /* Coordinate value convert to a double */ + + /* p->iCoord might point to either a lower or upper bound coordinate + ** in a coordinate pair. But make pCellData point to the lower bound. + */ + pCellData += 8 + 4*(p->iCoord&0xfe); + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + switch( p->op ){ + case RTREE_LE: + case RTREE_LT: + case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ) return; + if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ + /* Fall through for the RTREE_EQ case */ + + default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + *peWithin = NOT_WITHIN; +} + +/* +** Check the leaf RTree cell given by pCellData against constraint p. +** If this constraint is not satisfied, set *peWithin to NOT_WITHIN. +** If the constraint is satisfied, leave *peWithin unchanged. +** +** The constraint is of the form: xN op $val +** +** The op is given by p->op. The xN is p->iCoord-th coordinate in +** pCellData. $val is given by p->u.rValue. +*/ +static void rtreeLeafConstraint( + RtreeConstraint *p, /* The constraint to test */ + int eInt, /* True if RTree holds integer coordinates */ + u8 *pCellData, /* Raw cell content as appears on disk */ + int *peWithin /* Adjust downward, as appropriate */ +){ + RtreeDValue xN; /* Coordinate value converted to a double */ + + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ ); + pCellData += 8 + p->iCoord*4; + RTREE_DECODE_COORD(eInt, pCellData, xN); + switch( p->op ){ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; + } + *peWithin = NOT_WITHIN; +} + /* ** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. @@ -1085,6 +1073,7 @@ static int nodeRowidIndex( ){ int ii; int nCell = NCELL(pNode); + assert( nCell<200 ); for(ii=0; iirScorerScore ) return -1; + if( pA->rScore>pB->rScore ) return +1; + if( pA->iLeveliLevel ) return -1; + if( pA->iLevel>pB->iLevel ) return +1; + return 0; +} + +/* +** Interchange to search points in a cursor. +*/ +static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ + RtreeSearchPoint t = p->aPoint[i]; + assert( iaPoint[i] = p->aPoint[j]; + p->aPoint[j] = t; + i++; j++; + if( i=RTREE_CACHE_SZ ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + }else{ + RtreeNode *pTemp = p->aNode[i]; + p->aNode[i] = p->aNode[j]; + p->aNode[j] = pTemp; + } + } +} + +/* +** Return the search point with the lowest current score. +*/ +static RtreeSearchPoint *rtreeSearchPointFirst(RtreeCursor *pCur){ + return pCur->bPoint ? &pCur->sPoint : pCur->nPoint ? pCur->aPoint : 0; +} + +/* +** Get the RtreeNode for the search point with the lowest score. +*/ +static RtreeNode *rtreeNodeOfFirstSearchPoint(RtreeCursor *pCur, int *pRC){ + sqlite3_int64 id; + int ii = 1 - pCur->bPoint; + assert( ii==0 || ii==1 ); + assert( pCur->bPoint || pCur->nPoint ); + if( pCur->aNode[ii]==0 ){ + assert( pRC!=0 ); + id = ii ? pCur->aPoint[0].id : pCur->sPoint.id; + *pRC = nodeAcquire(RTREE_OF_CURSOR(pCur), id, 0, &pCur->aNode[ii]); + } + return pCur->aNode[ii]; +} + +/* +** Push a new element onto the priority queue +*/ +static RtreeSearchPoint *rtreeEnqueue( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + int i, j; + RtreeSearchPoint *pNew; + if( pCur->nPoint>=pCur->nPointAlloc ){ + int nNew = pCur->nPointAlloc*2 + 8; + pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + if( pNew==0 ) return 0; + pCur->aPoint = pNew; + pCur->nPointAlloc = nNew; + } + i = pCur->nPoint++; + pNew = pCur->aPoint + i; + pNew->rScore = rScore; + pNew->iLevel = iLevel; + assert( iLevel>=0 && iLevel<=RTREE_MAX_DEPTH ); + while( i>0 ){ + RtreeSearchPoint *pParent; + j = (i-1)/2; + pParent = pCur->aPoint + j; + if( rtreeSearchPointCompare(pNew, pParent)>=0 ) break; + rtreeSearchPointSwap(pCur, j, i); + i = j; + pNew = pParent; + } + return pNew; +} + +/* +** Allocate a new RtreeSearchPoint and return a pointer to it. Return +** NULL if malloc fails. +*/ +static RtreeSearchPoint *rtreeSearchPointNew( + RtreeCursor *pCur, /* The cursor */ + RtreeDValue rScore, /* Score for the new search point */ + u8 iLevel /* Level for the new search point */ +){ + RtreeSearchPoint *pNew, *pFirst; + pFirst = rtreeSearchPointFirst(pCur); + pCur->anQueue[iLevel]++; + if( pFirst==0 + || pFirst->rScore>rScore + || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) + ){ + if( pCur->bPoint ){ + int ii; + pNew = rtreeEnqueue(pCur, rScore, iLevel); + if( pNew==0 ) return 0; + ii = (int)(pNew - pCur->aPoint) + 1; + if( iiaNode[ii]==0 ); + pCur->aNode[ii] = pCur->aNode[0]; + }else{ + nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); + } + pCur->aNode[0] = 0; + *pNew = pCur->sPoint; + } + pCur->sPoint.rScore = rScore; + pCur->sPoint.iLevel = iLevel; + pCur->bPoint = 1; + return &pCur->sPoint; + }else{ + return rtreeEnqueue(pCur, rScore, iLevel); + } +} + +#if 0 +/* Tracing routines for the RtreeSearchPoint queue */ +static void tracePoint(RtreeSearchPoint *p, int idx, RtreeCursor *pCur){ + if( idx<0 ){ printf(" s"); }else{ printf("%2d", idx); } + printf(" %d.%05lld.%02d %g %d", + p->iLevel, p->id, p->iCell, p->rScore, p->eWithin + ); + idx++; + if( idxaNode[idx]); + }else{ + printf("\n"); + } +} +static void traceQueue(RtreeCursor *pCur, const char *zPrefix){ + int ii; + printf("=== %9s ", zPrefix); + if( pCur->bPoint ){ + tracePoint(&pCur->sPoint, -1, pCur); + } + for(ii=0; iinPoint; ii++){ + if( ii>0 || pCur->bPoint ) printf(" "); + tracePoint(&pCur->aPoint[ii], ii, pCur); + } +} +# define RTREE_QUEUE_TRACE(A,B) traceQueue(A,B) +#else +# define RTREE_QUEUE_TRACE(A,B) /* no-op */ +#endif + +/* Remove the search point with the lowest current score. +*/ +static void rtreeSearchPointPop(RtreeCursor *p){ + int i, j, k, n; + i = 1 - p->bPoint; + assert( i==0 || i==1 ); + if( p->aNode[i] ){ + nodeRelease(RTREE_OF_CURSOR(p), p->aNode[i]); + p->aNode[i] = 0; + } + if( p->bPoint ){ + p->anQueue[p->sPoint.iLevel]--; + p->bPoint = 0; + }else if( p->nPoint ){ + p->anQueue[p->aPoint[0].iLevel]--; + n = --p->nPoint; + p->aPoint[0] = p->aPoint[n]; + if( naNode[1] = p->aNode[n+1]; + p->aNode[n+1] = 0; + } + i = 0; + while( (j = i*2+1)aPoint[k], &p->aPoint[j])<0 ){ + if( rtreeSearchPointCompare(&p->aPoint[k], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, k); + i = k; + }else{ + break; + } + }else{ + if( rtreeSearchPointCompare(&p->aPoint[j], &p->aPoint[i])<0 ){ + rtreeSearchPointSwap(p, i, j); + i = j; + }else{ + break; + } + } + } + } +} + + +/* +** Continue the search on cursor pCur until the front of the queue +** contains an entry suitable for returning as a result-set row, +** or until the RtreeSearchPoint queue is empty, indicating that the +** query has completed. +*/ +static int rtreeStepToLeaf(RtreeCursor *pCur){ + RtreeSearchPoint *p; + Rtree *pRtree = RTREE_OF_CURSOR(pCur); + RtreeNode *pNode; + int eWithin; + int rc = SQLITE_OK; + int nCell; + int nConstraint = pCur->nConstraint; + int ii; + int eInt; + RtreeSearchPoint x; + + eInt = pRtree->eCoordType==RTREE_COORD_INT32; + while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); + if( rc ) return rc; + nCell = NCELL(pNode); + assert( nCell<200 ); + while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); + eWithin = FULLY_WITHIN; + for(ii=0; iiaConstraint + ii; + if( pConstraint->op>=RTREE_MATCH ){ + rc = rtreeCallbackConstraint(pConstraint, eInt, pCellData, p, + &rScore, &eWithin); + if( rc ) return rc; + }else if( p->iLevel==1 ){ + rtreeLeafConstraint(pConstraint, eInt, pCellData, &eWithin); + }else{ + rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); + } + if( eWithin==NOT_WITHIN ) break; + } + p->iCell++; + if( eWithin==NOT_WITHIN ) continue; + x.iLevel = p->iLevel - 1; + if( x.iLevel ){ + x.id = readInt64(pCellData); + x.iCell = 0; + }else{ + x.id = p->id; + x.iCell = p->iCell - 1; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-S:"); + rtreeSearchPointPop(pCur); + } + if( rScoreeWithin = eWithin; + p->id = x.id; + p->iCell = x.iCell; + RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); + break; + } + if( p->iCell>=nCell ){ + RTREE_QUEUE_TRACE(pCur, "POP-Se:"); + rtreeSearchPointPop(pCur); + } + } + pCur->atEOF = p==0; + return SQLITE_OK; +} + /* ** Rtree virtual table module xNext method. */ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ - Rtree *pRtree = (Rtree *)(pVtabCursor->pVtab); RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; int rc = SQLITE_OK; - /* RtreeCursor.pNode must not be NULL. If is is NULL, then this cursor is - ** already at EOF. It is against the rules to call the xNext() method of - ** a cursor that has already reached EOF. - */ - assert( pCsr->pNode ); - - if( pCsr->iStrategy==1 ){ - /* This "scan" is a direct lookup by rowid. There is no next entry. */ - nodeRelease(pRtree, pCsr->pNode); - pCsr->pNode = 0; - }else{ - /* Move to the next entry that matches the configured constraints. */ - int iHeight = 0; - while( pCsr->pNode ){ - RtreeNode *pNode = pCsr->pNode; - int nCell = NCELL(pNode); - for(pCsr->iCell++; pCsr->iCelliCell++){ - int isEof; - rc = descendToCell(pRtree, pCsr, iHeight, &isEof); - if( rc!=SQLITE_OK || !isEof ){ - return rc; - } - } - pCsr->pNode = pNode->pParent; - rc = nodeParentIndex(pRtree, pNode, &pCsr->iCell); - if( rc!=SQLITE_OK ){ - return rc; - } - nodeReference(pCsr->pNode); - nodeRelease(pRtree, pNode); - iHeight++; - } - } - + /* Move to the next entry that matches the configured constraints. */ + RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + rtreeSearchPointPop(pCsr); + rc = rtreeStepToLeaf(pCsr); return rc; } @@ -1156,13 +1399,14 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ - Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - - assert(pCsr->pNode); - *pRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - - return SQLITE_OK; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc==SQLITE_OK && p ){ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } + return rc; } /* @@ -1171,13 +1415,18 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ Rtree *pRtree = (Rtree *)cur->pVtab; RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + RtreeCoord c; + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; if( i==0 ){ - i64 iRowid = nodeGetRowid(pRtree, pCsr->pNode, pCsr->iCell); - sqlite3_result_int64(ctx, iRowid); + sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); }else{ - RtreeCoord c; - nodeGetCoord(pRtree, pCsr->pNode, pCsr->iCell, i-1, &c); + if( rc ) return rc; + nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ sqlite3_result_double(ctx, c.f); @@ -1188,7 +1437,6 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ sqlite3_result_int(ctx, c.i); } } - return SQLITE_OK; } @@ -1199,12 +1447,18 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf ** to zero and return an SQLite error code. */ -static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ +static int findLeafNode( + Rtree *pRtree, /* RTree to search */ + i64 iRowid, /* The rowid searching for */ + RtreeNode **ppLeaf, /* Write the node here */ + sqlite3_int64 *piNode /* Write the node-id here */ +){ int rc; *ppLeaf = 0; sqlite3_bind_int64(pRtree->pReadRowid, 1, iRowid); if( sqlite3_step(pRtree->pReadRowid)==SQLITE_ROW ){ i64 iNode = sqlite3_column_int64(pRtree->pReadRowid, 0); + if( piNode ) *piNode = iNode; rc = nodeAcquire(pRtree, iNode, 0, ppLeaf); sqlite3_reset(pRtree->pReadRowid); }else{ @@ -1220,9 +1474,10 @@ static int findLeafNode(Rtree *pRtree, i64 iRowid, RtreeNode **ppLeaf){ ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *p; - sqlite3_rtree_geometry *pGeom; - int nBlob; + RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + sqlite3_rtree_query_info *pInfo; /* Callback information */ + int nBlob; /* Size of the geometry function blob */ + int nExpected; /* Expected size of the BLOB */ /* Check that value is actually a blob. */ if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; @@ -1235,27 +1490,29 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_ERROR; } - pGeom = (sqlite3_rtree_geometry *)sqlite3_malloc( - sizeof(sqlite3_rtree_geometry) + nBlob - ); - if( !pGeom ) return SQLITE_NOMEM; - memset(pGeom, 0, sizeof(sqlite3_rtree_geometry)); - p = (RtreeMatchArg *)&pGeom[1]; + pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + if( !pInfo ) return SQLITE_NOMEM; + memset(pInfo, 0, sizeof(*pInfo)); + pBlob = (RtreeMatchArg*)&pInfo[1]; - memcpy(p, sqlite3_value_blob(pValue), nBlob); - if( p->magic!=RTREE_GEOMETRY_MAGIC - || nBlob!=(int)(sizeof(RtreeMatchArg) + (p->nParam-1)*sizeof(RtreeDValue)) - ){ - sqlite3_free(pGeom); + memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); + nExpected = (int)(sizeof(RtreeMatchArg) + + (pBlob->nParam-1)*sizeof(RtreeDValue)); + if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ + sqlite3_free(pInfo); return SQLITE_ERROR; } + pInfo->pContext = pBlob->cb.pContext; + pInfo->nParam = pBlob->nParam; + pInfo->aParam = pBlob->aParam; - pGeom->pContext = p->pContext; - pGeom->nParam = p->nParam; - pGeom->aParam = p->aParam; - - pCons->xGeom = p->xGeom; - pCons->pGeom = pGeom; + if( pBlob->cb.xGeom ){ + pCons->u.xGeom = pBlob->cb.xGeom; + }else{ + pCons->op = RTREE_QUERY; + pCons->u.xQueryFunc = pBlob->cb.xQueryFunc; + } + pCons->pInfo = pInfo; return SQLITE_OK; } @@ -1269,10 +1526,10 @@ static int rtreeFilter( ){ Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; - RtreeNode *pRoot = 0; int ii; int rc = SQLITE_OK; + int iCell = 0; rtreeReference(pRtree); @@ -1282,31 +1539,42 @@ static int rtreeFilter( if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); - rc = findLeafNode(pRtree, iRowid, &pLeaf); - pCsr->pNode = pLeaf; - if( pLeaf ){ - assert( rc==SQLITE_OK ); - rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &pCsr->iCell); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; } }else{ /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array ** with the configured constraints. */ - if( argc>0 ){ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && argc>0 ){ pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; }else{ memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*argc); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); assert( (idxStr==0 && argc==0) || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; p->op = idxStr[ii*2]; - p->iCoord = idxStr[ii*2+1]-'a'; - if( p->op==RTREE_MATCH ){ + p->iCoord = idxStr[ii*2+1]-'0'; + if( p->op>=RTREE_MATCH ){ /* A MATCH operator. The right-hand-side must be a blob that ** can be cast into an RtreeMatchArg object. One created using ** an sqlite3_rtree_geometry_callback() SQL user function. @@ -1315,41 +1583,35 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } + p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->anQueue = pCsr->anQueue; + p->pInfo->mxLevel = pRtree->iDepth + 1; }else{ #ifdef SQLITE_RTREE_INT_ONLY - p->rValue = sqlite3_value_int64(argv[ii]); + p->u.rValue = sqlite3_value_int64(argv[ii]); #else - p->rValue = sqlite3_value_double(argv[ii]); + p->u.rValue = sqlite3_value_double(argv[ii]); #endif } } } } - if( rc==SQLITE_OK ){ - pCsr->pNode = 0; - rc = nodeAcquire(pRtree, 1, 0, &pRoot); - } - if( rc==SQLITE_OK ){ - int isEof = 1; - int nCell = NCELL(pRoot); - pCsr->pNode = pRoot; - for(pCsr->iCell=0; rc==SQLITE_OK && pCsr->iCelliCell++){ - assert( pCsr->pNode==pRoot ); - rc = descendToCell(pRtree, pCsr, pRtree->iDepth, &isEof); - if( !isEof ){ - break; - } - } - if( rc==SQLITE_OK && isEof ){ - assert( pCsr->pNode==pRoot ); - nodeRelease(pRtree, pRoot); - pCsr->pNode = 0; - } - assert( rc!=SQLITE_OK || !pCsr->pNode || pCsr->iCellpNode) ); + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); + if( pNew==0 ) return SQLITE_NOMEM; + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); } } + nodeRelease(pRtree, pRoot); rtreeRelease(pRtree); return rc; } @@ -1451,7 +1713,7 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ break; } zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + 'a'; + zIdxStr[iIdx++] = p->iColumn - 1 + '0'; pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); pIdxInfo->aConstraintUsage[ii].omit = 1; } @@ -1544,62 +1806,32 @@ static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ return (cellArea(pRtree, &cell)-area); } -#if VARIANT_RSTARTREE_CHOOSESUBTREE || VARIANT_RSTARTREE_SPLIT static RtreeDValue cellOverlap( Rtree *pRtree, RtreeCell *p, RtreeCell *aCell, - int nCell, - int iExclude + int nCell ){ int ii; - RtreeDValue overlap = 0.0; + RtreeDValue overlap = RTREE_ZERO; for(ii=0; iinDim*2); jj+=2){ - RtreeDValue x1, x2; - - x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); - x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); - - if( x2nDim*2); jj+=2){ + RtreeDValue x1, x2; + x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); + x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); + if( x2iDepth-1) ){ - int jj; - aCell = sqlite3_malloc(sizeof(RtreeCell)*nCell); - if( !aCell ){ - rc = SQLITE_NOMEM; - nodeRelease(pRtree, pNode); - pNode = 0; - continue; - } - for(jj=0; jjiDepth-1) ){ - overlap = cellOverlapEnlargement(pRtree,&cell,pCell,aCell,nCell,iCell); - }else{ - overlap = 0.0; - } - if( (iCell==0) - || (overlapnDim; i++){ - RtreeDValue x1 = DCOORD(aCell[0].aCoord[i*2]); - RtreeDValue x2 = DCOORD(aCell[0].aCoord[i*2+1]); - RtreeDValue x3 = x1; - RtreeDValue x4 = x2; - int jj; - - int iCellLeft = 0; - int iCellRight = 0; - - for(jj=1; jjx4 ) x4 = right; - if( left>x3 ){ - x3 = left; - iCellRight = jj; - } - if( rightmaxNormalInnerWidth ){ - iLeftSeed = iCellLeft; - iRightSeed = iCellRight; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_LINEAR_SPLIT */ - -#if VARIANT_GUTTMAN_QUADRATIC_SPLIT -/* -** Implementation of the quadratic variant of the PickNext() function from -** Guttman[84]. -*/ -static RtreeCell *QuadraticPickNext( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - RtreeCell *pLeftBox, - RtreeCell *pRightBox, - int *aiUsed -){ - #define FABS(a) ((a)<0.0?-1.0*(a):(a)) - - int iSelect = -1; - RtreeDValue fDiff; - int ii; - for(ii=0; iifDiff ){ - fDiff = diff; - iSelect = ii; - } - } - } - aiUsed[iSelect] = 1; - return &aCell[iSelect]; -} - -/* -** Implementation of the quadratic variant of the PickSeeds() function from -** Guttman[84]. -*/ -static void QuadraticPickSeeds( - Rtree *pRtree, - RtreeCell *aCell, - int nCell, - int *piLeftSeed, - int *piRightSeed -){ - int ii; - int jj; - - int iLeftSeed = 0; - int iRightSeed = 1; - RtreeDValue fWaste = 0.0; - - for(ii=0; iifWaste ){ - iLeftSeed = ii; - iRightSeed = jj; - fWaste = waste; - } - } - } - - *piLeftSeed = iLeftSeed; - *piRightSeed = iRightSeed; -} -#endif /* VARIANT_GUTTMAN_QUADRATIC_SPLIT */ /* ** Arguments aIdx, aDistance and aSpare all point to arrays of size @@ -2040,7 +2086,6 @@ static void SortByDimension( } } -#if VARIANT_RSTARTREE_SPLIT /* ** Implementation of the R*-tree variant of SplitNode from Beckman[1990]. */ @@ -2059,7 +2104,7 @@ static int splitNodeStartree( int iBestDim = 0; int iBestSplit = 0; - RtreeDValue fBestMargin = 0.0; + RtreeDValue fBestMargin = RTREE_ZERO; int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); @@ -2080,9 +2125,9 @@ static int splitNodeStartree( } for(ii=0; iinDim; ii++){ - RtreeDValue margin = 0.0; - RtreeDValue fBestOverlap = 0.0; - RtreeDValue fBestArea = 0.0; + RtreeDValue margin = RTREE_ZERO; + RtreeDValue fBestOverlap = RTREE_ZERO; + RtreeDValue fBestArea = RTREE_ZERO; int iBestLeft = 0; int nLeft; @@ -2108,7 +2153,7 @@ static int splitNodeStartree( } margin += cellMargin(pRtree, &left); margin += cellMargin(pRtree, &right); - overlap = cellOverlap(pRtree, &left, &right, 1, -1); + overlap = cellOverlap(pRtree, &left, &right, 1); area = cellArea(pRtree, &left) + cellArea(pRtree, &right); if( (nLeft==RTREE_MINCELLS(pRtree)) || (overlap0; i--){ - RtreeCell *pNext; - pNext = PickNext(pRtree, aCell, nCell, pBboxLeft, pBboxRight, aiUsed); - RtreeDValue diff = - cellGrowth(pRtree, pBboxLeft, pNext) - - cellGrowth(pRtree, pBboxRight, pNext) - ; - if( (RTREE_MINCELLS(pRtree)-NCELL(pRight)==i) - || (diff>0.0 && (RTREE_MINCELLS(pRtree)-NCELL(pLeft)!=i)) - ){ - nodeInsertCell(pRtree, pRight, pNext); - cellUnion(pRtree, pBboxRight, pNext); - }else{ - nodeInsertCell(pRtree, pLeft, pNext); - cellUnion(pRtree, pBboxLeft, pNext); - } - } - - sqlite3_free(aiUsed); - return SQLITE_OK; -} -#endif static int updateMapping( Rtree *pRtree, @@ -2274,7 +2263,8 @@ static int SplitNode( memset(pLeft->zData, 0, pRtree->iNodeSize); memset(pRight->zData, 0, pRtree->iNodeSize); - rc = AssignCells(pRtree, aCell, nCell, pLeft, pRight, &leftbbox, &rightbbox); + rc = splitNodeStartree(pRtree, aCell, nCell, pLeft, pRight, + &leftbbox, &rightbbox); if( rc!=SQLITE_OK ){ goto splitnode_out; } @@ -2557,7 +2547,7 @@ static int Reinsert( } for(ii=0; iinDim; iDim++){ RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - DCOORD(aCell[ii].aCoord[iDim*2])); @@ -2623,16 +2613,12 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ -#if VARIANT_RSTARTREE_REINSERT if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ pRtree->iReinsertHeight = iHeight; rc = Reinsert(pRtree, pNode, pCell, iHeight); } -#else - rc = SplitNode(pRtree, pNode, pCell, iHeight); -#endif }else{ rc = AdjustTree(pRtree, pNode, pCell); if( rc==SQLITE_OK ){ @@ -2702,7 +2688,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ ** about to be deleted. */ if( rc==SQLITE_OK ){ - rc = findLeafNode(pRtree, iDelete, &pLeaf); + rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } /* Delete the cell in question from the leaf node. */ @@ -3039,7 +3025,8 @@ static int rtreeSqlInit( char *zCreate = sqlite3_mprintf( "CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY, parentnode INTEGER);" +"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," + " parentnode INTEGER);" "INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize ); @@ -3253,10 +3240,10 @@ static int rtreeInit( ** Implementation of a scalar function that decodes r-tree nodes to ** human readable strings. This can be used for debugging and analysis. ** -** The scalar function takes two arguments, a blob of data containing -** an r-tree node, and the number of dimensions the r-tree indexes. -** For a two-dimensional r-tree structure called "rt", to deserialize -** all nodes, a statement like: +** The scalar function takes two arguments: (1) the number of dimensions +** to the rtree (between 1 and 5, inclusive) and (2) a blob of data containing +** an r-tree node. For a two-dimensional r-tree structure called "rt", to +** deserialize all nodes, a statement like: ** ** SELECT rtreenode(2, data) FROM rt_node; ** @@ -3289,7 +3276,7 @@ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ nCell = (int)strlen(zCell); for(jj=0; jjxDestructor ) pInfo->xDestructor(pInfo->pContext); sqlite3_free(p); } /* -** Each call to sqlite3_rtree_geometry_callback() creates an ordinary SQLite -** scalar user function. This C function is the callback used for all such -** registered SQL functions. +** Each call to sqlite3_rtree_geometry_callback() or +** sqlite3_rtree_query_callback() creates an ordinary SQLite +** scalar function that is implemented by this routine. ** -** The scalar user functions return a blob that is interpreted by r-tree -** table MATCH operators. +** All this function does is construct an RtreeMatchArg object that +** contains the geometry-checking callback routines and a list of +** parameters to this function, then return that RtreeMatchArg object +** as a BLOB. +** +** The R-Tree MATCH operator will read the returned BLOB, deserialize +** the RtreeMatchArg object, and use the RtreeMatchArg object to figure +** out which elements of the R-Tree should be returned by the query. */ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ RtreeGeomCallback *pGeomCtx = (RtreeGeomCallback *)sqlite3_user_data(ctx); @@ -3381,8 +3386,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ }else{ int i; pBlob->magic = RTREE_GEOMETRY_MAGIC; - pBlob->xGeom = pGeomCtx->xGeom; - pBlob->pContext = pGeomCtx->pContext; + pBlob->cb = pGeomCtx[0]; pBlob->nParam = nArg; for(i=0; iaParam[i] = sqlite3_value_double(aArg[i]); #endif } - sqlite3_result_blob(ctx, pBlob, nBlob, doSqlite3Free); + sqlite3_result_blob(ctx, pBlob, nBlob, sqlite3_free); } } @@ -3399,10 +3403,10 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ ** Register a new geometry function for use with the r-tree MATCH operator. */ int sqlite3_rtree_geometry_callback( - sqlite3 *db, - const char *zGeom, - int (*xGeom)(sqlite3_rtree_geometry *, int, RtreeDValue *, int *), - void *pContext + sqlite3 *db, /* Register SQL function on this connection */ + const char *zGeom, /* Name of the new SQL function */ + int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ + void *pContext /* Extra data associated with the callback */ ){ RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ @@ -3410,12 +3414,36 @@ int sqlite3_rtree_geometry_callback( pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); if( !pGeomCtx ) return SQLITE_NOMEM; pGeomCtx->xGeom = xGeom; + pGeomCtx->xQueryFunc = 0; + pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - - /* Create the new user-function. Register a destructor function to delete - ** the context object when it is no longer required. */ return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, - (void *)pGeomCtx, geomCallback, 0, 0, doSqlite3Free + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback + ); +} + +/* +** Register a new 2nd-generation geometry function for use with the +** r-tree MATCH operator. +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, /* Register SQL function on this connection */ + const char *zQueryFunc, /* Name of new SQL function */ + int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ + void *pContext, /* Extra data passed into the callback */ + void (*xDestructor)(void*) /* Destructor for the extra data */ +){ + RtreeGeomCallback *pGeomCtx; /* Context object for new user-function */ + + /* Allocate and populate the context object. */ + pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); + if( !pGeomCtx ) return SQLITE_NOMEM; + pGeomCtx->xGeom = 0; + pGeomCtx->xQueryFunc = xQueryFunc; + pGeomCtx->xDestructor = xDestructor; + pGeomCtx->pContext = pContext; + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 275b13264f..9de5362781 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -120,12 +120,13 @@ proc execsql_intout {sql} { # Test that it is possible to open an existing database that contains # r-tree tables. # -do_test rtree-1.4.1 { - execsql { - CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); - INSERT INTO t1 VALUES(1, 5.0, 10.0); - INSERT INTO t1 VALUES(2, 15.0, 20.0); - } +do_execsql_test rtree-1.4.1a { + CREATE VIRTUAL TABLE t1 USING rtree(ii, x1, x2); + INSERT INTO t1 VALUES(1, 5.0, 10.0); + SELECT substr(hex(data),1,40) FROM t1_node; +} {00000001000000000000000140A0000041200000} +do_execsql_test rtree-1.4.1b { + INSERT INTO t1 VALUES(2, 15.0, 20.0); } {} do_test rtree-1.4.2 { db close @@ -435,16 +436,18 @@ do_test rtree-11.2 { # Test on-conflict clause handling. # db_delete_and_reopen -do_execsql_test 12.0 { +do_execsql_test 12.0.1 { CREATE VIRTUAL TABLE t1 USING rtree_i32(idx, x1, x2, y1, y2); INSERT INTO t1 VALUES(1, 1, 2, 3, 4); + SELECT substr(hex(data),1,56) FROM t1_node; +} {00000001000000000000000100000001000000020000000300000004} +do_execsql_test 12.0.2 { INSERT INTO t1 VALUES(2, 2, 3, 4, 5); INSERT INTO t1 VALUES(3, 3, 4, 5, 6); CREATE TABLE source(idx, x1, x2, y1, y2); INSERT INTO source VALUES(5, 8, 8, 8, 8); INSERT INTO source VALUES(2, 7, 7, 7, 7); - } db_save_and_close foreach {tn sql_template testdata} { diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index bdc9bb4146..c5a78bb3e3 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -57,31 +57,31 @@ do_test rtree6-1.1 { do_test rtree6-1.2 { rtree_strategy {SELECT * FROM t1 WHERE x1>10} -} {Ea} +} {E0} do_test rtree6-1.3 { rtree_strategy {SELECT * FROM t1 WHERE x1<10} -} {Ca} +} {C0} do_test rtree6-1.4 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=ii AND x1<10} -} {Ca} +} {C0} do_test rtree6-1.5 { rtree_strategy {SELECT * FROM t1,t2 WHERE k=+ii AND x1<10} -} {Ca} +} {C0} do_eqp_test rtree6.2.1 { SELECT * FROM t1,t2 WHERE k=+ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } do_eqp_test rtree6.2.2 { SELECT * FROM t1,t2 WHERE k=ii AND x1<10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:Ca} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0} 0 1 1 {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid=?)} } @@ -95,7 +95,7 @@ do_eqp_test rtree6.2.3 { do_eqp_test rtree6.2.4 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { - 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:CaEb} + 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} } @@ -126,7 +126,7 @@ do_test rtree6.3.2 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_test rtree6.3.3 { rtree_strategy { SELECT * FROM t3 WHERE @@ -137,7 +137,7 @@ do_test rtree6.3.3 { x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 AND x1>0.5 } -} {EaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEaEa} +} {E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0E0} do_execsql_test rtree6-3.4 { SELECT * FROM t3 WHERE x1>0.5 AND x1>0.8 AND x1>1.1 diff --git a/ext/rtree/rtreeB.test b/ext/rtree/rtreeB.test index 7cb445cc4f..aeb308eca7 100644 --- a/ext/rtree/rtreeB.test +++ b/ext/rtree/rtreeB.test @@ -41,7 +41,7 @@ ifcapable rtree_int_only { INSERT INTO t1 VALUES(9223372036854775807, 150, 150, 400, 400); SELECT rtreenode(2, data) FROM t1_node; } - } {{{1073741824 0.000000 0.000000 100.000000 100.000000} {2147483646 0.000000 0.000000 200.000000 200.000000} {4294967296 0.000000 0.000000 300.000000 300.000000} {8589934592 20.000000 20.000000 150.000000 150.000000} {9223372036854775807 150.000000 150.000000 400.000000 400.000000}}} + } {{{1073741824 0 0 100 100} {2147483646 0 0 200 200} {4294967296 0 0 300 300} {8589934592 20 20 150 150} {9223372036854775807 150 150 400 400}}} } finish_test diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index 23dc607841..94db05a4d1 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -29,7 +29,7 @@ do_eqp_test 1.1 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 1 {SCAN TABLE t} - 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.2 { @@ -37,7 +37,7 @@ do_eqp_test 1.2 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.3 { @@ -45,7 +45,7 @@ do_eqp_test 1.3 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 1.5 { @@ -82,7 +82,7 @@ do_eqp_test 2.1 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 1 {SCAN TABLE t} - 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 0 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.2 { @@ -90,7 +90,7 @@ do_eqp_test 2.2 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND t.x<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.3 { @@ -98,7 +98,7 @@ do_eqp_test 2.3 { WHERE t.x>=min_x AND t.x<=max_x AND t.y>=min_y AND ?<=max_y } { 0 0 0 {SCAN TABLE t} - 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:DdBcDbBa} + 0 1 1 {SCAN TABLE r_tree VIRTUAL TABLE INDEX 2:D3B2D1B0} } do_eqp_test 2.5 { @@ -271,4 +271,3 @@ ifcapable rtree { finish_test - diff --git a/ext/rtree/rtreeE.test b/ext/rtree/rtreeE.test new file mode 100644 index 0000000000..c450623790 --- /dev/null +++ b/ext/rtree/rtreeE.test @@ -0,0 +1,129 @@ +# 2010 August 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. Specifically, it tests +# that new-style custom r-tree queries (geometry callbacks) work. +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } +ifcapable rtree_int_only { finish_test; return } + + +#------------------------------------------------------------------------- +# Test the example 2d "circle" geometry callback. +# +register_circle_geom db + +do_execsql_test rtreeE-1.1 { + PRAGMA page_size=512; + CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1); + + /* A tight pattern of small boxes near 0,0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT x+5*y, x, x+2, y, y+2 FROM x, y; + + /* A looser pattern of small boxes near 100, 0 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 100+x+5*y, x*3+100, x*3+102, y*3, y*3+2 FROM x, y; + + /* A looser pattern of larger boxes near 0, 200 */ + WITH RECURSIVE + x(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM x WHERE x<4), + y(y) AS (VALUES(0) UNION ALL SELECT y+1 FROM y WHERE y<4) + INSERT INTO rt1 SELECT 200+x+5*y, x*7, x*7+15, y*7+200, y*7+215 FROM x, y; +} {} + +# Queries against each of the three clusters */ +do_execsql_test rtreeE-1.1 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 0.0, 50.0, 3) ORDER BY id; +} {0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24} +do_execsql_test rtreeE-1.2 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(100.0, 0.0, 50.0, 3) ORDER BY id; +} {100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124} +do_execsql_test rtreeE-1.3 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0.0, 200.0, 50.0, 3) ORDER BY id; +} {200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224} + +# The Qcircle geometry function gives a lower score to larger leaf-nodes. +# This causes the 200s to sort before the 100s and the 0s to sort before +# last. +# +do_execsql_test rtreeE-1.4 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,3) AND id%100==0 +} {200 100 0} + +# Exclude odd rowids on a depth-first search +do_execsql_test rtreeE-1.5 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,4) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Exclude odd rowids on a breadth-first search. +do_execsql_test rtreeE-1.6 { + SELECT id FROM rt1 WHERE id MATCH Qcircle(0,0,1000,5) ORDER BY +id +} {0 2 4 6 8 10 12 14 16 18 20 22 24 100 102 104 106 108 110 112 114 116 118 120 122 124 200 202 204 206 208 210 212 214 216 218 220 222 224} + +# Construct a large 2-D RTree with thousands of random entries. +# +do_test rtreeE-2.1 { + db eval { + CREATE TABLE t2(id,x0,x1,y0,y1); + CREATE VIRTUAL TABLE rt2 USING rtree(id,x0,x1,y0,y1); + BEGIN; + } + expr srand(0) + for {set i 1} {$i<=10000} {incr i} { + set dx [expr {int(rand()*40)+1}] + set dy [expr {int(rand()*40)+1}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set id [expr {$i+10000}] + db eval {INSERT INTO t2 VALUES($id,$x0,$x1,$y0,$y1)} + } + db eval { + INSERT INTO rt2 SELECT * FROM t2; + COMMIT; + } +} {} + +for {set i 1} {$i<=200} {incr i} { + set dx [expr {int(rand()*100)}] + set dy [expr {int(rand()*100)}] + set x0 [expr {int(rand()*(10000 - $dx))}] + set x1 [expr {$x0+$dx}] + set y0 [expr {int(rand()*(10000 - $dy))}] + set y1 [expr {$y0+$dy}] + set ans [db eval {SELECT id FROM t2 WHERE x1>=$x0 AND x0<=$x1 AND y1>=$y0 AND y0<=$y1 ORDER BY id}] + do_execsql_test rtreeE-2.2.$i { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch($x0,$x1,$y0,$y1) ORDER BY id + } $ans +} + +# Run query that have very deep priority queues +# +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=5000 AND y1>=0 AND y0<=5000 ORDER BY id}] +do_execsql_test rtreeE-2.3 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,5000,0,5000) ORDER BY id +} $ans +set ans [db eval {SELECT id FROM t2 WHERE x1>=0 AND x0<=10000 AND y1>=0 AND y0<=10000 ORDER BY id}] +do_execsql_test rtreeE-2.4 { + SELECT id FROM rt2 WHERE id MATCH breadthfirstsearch(0,10000,0,10000) ORDER BY id +} $ans + +finish_test diff --git a/ext/rtree/sqlite3rtree.h b/ext/rtree/sqlite3rtree.h index c849091f29..5de0508d00 100644 --- a/ext/rtree/sqlite3rtree.h +++ b/ext/rtree/sqlite3rtree.h @@ -21,6 +21,16 @@ extern "C" { #endif typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif /* ** Register a geometry callback named zGeom that can be used as part of an @@ -31,11 +41,7 @@ typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, -#ifdef SQLITE_RTREE_INT_ONLY - int (*xGeom)(sqlite3_rtree_geometry*, int n, sqlite3_int64 *a, int *pRes), -#else - int (*xGeom)(sqlite3_rtree_geometry*, int n, double *a, int *pRes), -#endif + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), void *pContext ); @@ -47,11 +53,60 @@ int sqlite3_rtree_geometry_callback( struct sqlite3_rtree_geometry { void *pContext; /* Copy of pContext passed to s_r_g_c() */ int nParam; /* Size of array aParam[] */ - double *aParam; /* Parameters passed to SQL geom function */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ void *pUser; /* Callback implementation user data */ void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ }; +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visiblity */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + #ifdef __cplusplus } /* end of the 'extern "C"' block */ diff --git a/main.mk b/main.mk index ebd508a279..fcc5c03db7 100644 --- a/main.mk +++ b/main.mk @@ -476,7 +476,7 @@ parse.c: $(TOP)/src/parse.y lemon $(TOP)/addopcodes.awk mv parse.h parse.h.temp $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h -sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION +sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION $(TOP)/ext/rtree/sqlite3rtree.h tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c diff --git a/manifest b/manifest index fea1e4c44c..41c2bd4ae0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\srequirements\smarks\sto\sfix\stypos\sin\sthe\srequirements\stext.\nNo\schanges\sto\scode. -D 2014-04-26T19:23:14.120 +C Add\sthe\ssqlite3_rtree_query_callback()\sAPI\sto\sthe\sRTree\svirtual\stable.\n(Cherrypick\sfrom\sthe\ssessions\sbranch.) +D 2014-04-28T17:56:19.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,30 +120,31 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de +F ext/rtree/rtree.c 6f70db93e0e42c369325c5cddcf2024c5a87ca43 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 +F ext/rtree/rtree1.test e2da4aaa426918d27122d1a1066c6ecf8409a514 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test fe0bd377a21c68ce2826129d14354c884cb1f354 +F ext/rtree/rtree6.test 756585abc51727fec97c77852476445c10c0ee95 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf -F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test 16d7aa86ecb6a876d2a38cf590a1471a41b3a46d +F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e +F ext/rtree/rtreeC.test df158dcc81f1a43ce7eef361af03c48ec91f1e06 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca +F ext/rtree/rtreeE.test 388c1c8602c3ce55c15f03b509e9cf545fb7c41f F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea -F ext/rtree/sqlite3rtree.h c34c1e41d1ab80bb8ad09aae402c9c956871a765 +F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 3ae543fa446525c1dec55f58de67f41b78651812 +F main.mk 9546867b42992c554e7af8672549ba13afaadade F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -260,7 +261,7 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 30c64f0ef84734f2231a686df41ed882b0c59bc0 F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_rtree.c f3d1d12538dccb75fd916e3fa58f250edbdd3b47 +F src/test_rtree.c fdd8d29ca5165c7857987a2ba263fac5c69e231f F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e @@ -270,7 +271,7 @@ F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 -F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 +F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb @@ -811,6 +812,7 @@ F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 F test/shell5.test bb755ea9144b8078a752fc56223582627070b5f1 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 +F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f @@ -826,7 +828,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 +F test/speedtest1.c d29c8048beb7ea9254191f3fde9414709166a920 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1163,7 +1165,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 349f483499dd685a8da94923b6bd810a52e5e236 -R 310b081215d2804c69537a1d1f588735 +P f5a263658187250044afc1a74000e6f6962733ca +Q +3dca2809352c6c6d56db74447a814f77011c6220 +R 70a04a84bf76743284b12f147604df6e U drh -Z 3788b514428a54594de7c9ac65d6befd +Z 8441eee0dd2b010346629b18a46aad71 diff --git a/manifest.uuid b/manifest.uuid index b7714159cf..fbb0a3e2f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5a263658187250044afc1a74000e6f6962733ca \ No newline at end of file +af2cbe64adab5f9e3b0f3da00d06428088589d7f \ No newline at end of file diff --git a/src/test_rtree.c b/src/test_rtree.c index e1966c2437..9d19fa0e2c 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -35,6 +35,8 @@ struct Circle { double centerx; double centery; double radius; + double mxArea; + int eScoreType; }; /* @@ -50,11 +52,7 @@ static void circle_del(void *p){ static int circle_geom( sqlite3_rtree_geometry *p, int nCoord, -#ifdef SQLITE_RTREE_INT_ONLY - sqlite3_int64 *aCoord, -#else - double *aCoord, -#endif + sqlite3_rtree_dbl *aCoord, int *pRes ){ int i; /* Iterator variable */ @@ -62,7 +60,12 @@ static int circle_geom( double xmin, xmax; /* X dimensions of box being tested */ double ymin, ymax; /* X dimensions of box being tested */ - if( p->pUser==0 ){ + xmin = aCoord[0]; + xmax = aCoord[1]; + ymin = aCoord[2]; + ymax = aCoord[3]; + pCircle = (Circle *)p->pUser; + if( pCircle==0 ){ /* If pUser is still 0, then the parameter values have not been tested ** for correctness or stored into a Circle structure yet. Do this now. */ @@ -108,14 +111,9 @@ static int circle_geom( pCircle->aBox[1].xmax = pCircle->centerx - pCircle->radius; pCircle->aBox[1].ymin = pCircle->centery; pCircle->aBox[1].ymax = pCircle->centery; + pCircle->mxArea = (xmax - xmin)*(ymax - ymin) + 1.0; } - pCircle = (Circle *)p->pUser; - xmin = aCoord[0]; - xmax = aCoord[1]; - ymin = aCoord[2]; - ymax = aCoord[3]; - /* Check if any of the 4 corners of the bounding-box being tested lie ** inside the circular region. If they do, then the bounding-box does ** intersect the region of interest. Set the output variable to true and @@ -154,6 +152,170 @@ static int circle_geom( return SQLITE_OK; } +/* +** Implementation of "circle" r-tree geometry callback using the +** 2nd-generation interface that allows scoring. +*/ +static int circle_query_func(sqlite3_rtree_query_info *p){ + int i; /* Iterator variable */ + Circle *pCircle; /* Structure defining circular region */ + double xmin, xmax; /* X dimensions of box being tested */ + double ymin, ymax; /* X dimensions of box being tested */ + int nWithin = 0; /* Number of corners inside the circle */ + + xmin = p->aCoord[0]; + xmax = p->aCoord[1]; + ymin = p->aCoord[2]; + ymax = p->aCoord[3]; + pCircle = (Circle *)p->pUser; + if( pCircle==0 ){ + /* If pUser is still 0, then the parameter values have not been tested + ** for correctness or stored into a Circle structure yet. Do this now. */ + + /* This geometry callback is for use with a 2-dimensional r-tree table. + ** Return an error if the table does not have exactly 2 dimensions. */ + if( p->nCoord!=4 ) return SQLITE_ERROR; + + /* Test that the correct number of parameters (4) have been supplied, + ** and that the parameters are in range (that the radius of the circle + ** radius is greater than zero). */ + if( p->nParam!=4 || p->aParam[2]<0.0 ) return SQLITE_ERROR; + + /* Allocate a structure to cache parameter data in. Return SQLITE_NOMEM + ** if the allocation fails. */ + pCircle = (Circle *)(p->pUser = sqlite3_malloc(sizeof(Circle))); + if( !pCircle ) return SQLITE_NOMEM; + p->xDelUser = circle_del; + + /* Record the center and radius of the circular region. One way that + ** tested bounding boxes that intersect the circular region are detected + ** is by testing if each corner of the bounding box lies within radius + ** units of the center of the circle. */ + pCircle->centerx = p->aParam[0]; + pCircle->centery = p->aParam[1]; + pCircle->radius = p->aParam[2]; + pCircle->eScoreType = (int)p->aParam[3]; + + /* Define two bounding box regions. The first, aBox[0], extends to + ** infinity in the X dimension. It covers the same range of the Y dimension + ** as the circular region. The second, aBox[1], extends to infinity in + ** the Y dimension and is constrained to the range of the circle in the + ** X dimension. + ** + ** Then imagine each box is split in half along its short axis by a line + ** that intersects the center of the circular region. A bounding box + ** being tested can be said to intersect the circular region if it contains + ** points from each half of either of the two infinite bounding boxes. + */ + pCircle->aBox[0].xmin = pCircle->centerx; + pCircle->aBox[0].xmax = pCircle->centerx; + pCircle->aBox[0].ymin = pCircle->centery + pCircle->radius; + pCircle->aBox[0].ymax = pCircle->centery - pCircle->radius; + pCircle->aBox[1].xmin = pCircle->centerx + pCircle->radius; + pCircle->aBox[1].xmax = pCircle->centerx - pCircle->radius; + pCircle->aBox[1].ymin = pCircle->centery; + pCircle->aBox[1].ymax = pCircle->centery; + pCircle->mxArea = 200.0*200.0; + } + + /* Check if any of the 4 corners of the bounding-box being tested lie + ** inside the circular region. If they do, then the bounding-box does + ** intersect the region of interest. Set the output variable to true and + ** return SQLITE_OK in this case. */ + for(i=0; i<4; i++){ + double x = (i&0x01) ? xmax : xmin; + double y = (i&0x02) ? ymax : ymin; + double d2; + + d2 = (x-pCircle->centerx)*(x-pCircle->centerx); + d2 += (y-pCircle->centery)*(y-pCircle->centery); + if( d2<(pCircle->radius*pCircle->radius) ) nWithin++; + } + + /* Check if the bounding box covers any other part of the circular region. + ** See comments above for a description of how this test works. If it does + ** cover part of the circular region, set the output variable to true + ** and return SQLITE_OK. */ + if( nWithin==0 ){ + for(i=0; i<2; i++){ + if( xmin<=pCircle->aBox[i].xmin + && xmax>=pCircle->aBox[i].xmax + && ymin<=pCircle->aBox[i].ymin + && ymax>=pCircle->aBox[i].ymax + ){ + nWithin = 1; + break; + } + } + } + + if( pCircle->eScoreType==1 ){ + /* Depth first search */ + p->rScore = p->iLevel; + }else if( pCircle->eScoreType==2 ){ + /* Breadth first search */ + p->rScore = 100 - p->iLevel; + }else if( pCircle->eScoreType==3 ){ + /* Depth-first search, except sort the leaf nodes by area with + ** the largest area first */ + if( p->iLevel==1 ){ + p->rScore = 1.0 - (xmax-xmin)*(ymax-ymin)/pCircle->mxArea; + if( p->rScore<0.01 ) p->rScore = 0.01; + }else{ + p->rScore = 0.0; + } + }else if( pCircle->eScoreType==4 ){ + /* Depth-first search, except exclude odd rowids */ + p->rScore = p->iLevel; + if( p->iRowid&1 ) nWithin = 0; + }else{ + /* Breadth-first search, except exclude odd rowids */ + p->rScore = 100 - p->iLevel; + if( p->iRowid&1 ) nWithin = 0; + } + if( nWithin==0 ){ + p->eWithin = NOT_WITHIN; + }else if( nWithin>=4 ){ + p->eWithin = FULLY_WITHIN; + }else{ + p->eWithin = PARTLY_WITHIN; + } + return SQLITE_OK; +} +/* +** Implementation of "breadthfirstsearch" r-tree geometry callback using the +** 2nd-generation interface that allows scoring. +** +** ... WHERE id MATCH breadthfirstsearch($x0,$x1,$y0,$y1) ... +** +** It returns all entries whose bounding boxes overlap with $x0,$x1,$y0,$y1. +*/ +static int bfs_query_func(sqlite3_rtree_query_info *p){ + double x0,x1,y0,y1; /* Dimensions of box being tested */ + double bx0,bx1,by0,by1; /* Boundary of the query function */ + + if( p->nParam!=4 ) return SQLITE_ERROR; + x0 = p->aCoord[0]; + x1 = p->aCoord[1]; + y0 = p->aCoord[2]; + y1 = p->aCoord[3]; + bx0 = p->aParam[0]; + bx1 = p->aParam[1]; + by0 = p->aParam[2]; + by1 = p->aParam[3]; + p->rScore = 100 - p->iLevel; + if( p->eParentWithin==FULLY_WITHIN ){ + p->eWithin = FULLY_WITHIN; + }else if( x0>=bx0 && x1<=bx1 && y0>=by0 && y1<=by1 ){ + p->eWithin = FULLY_WITHIN; + }else if( x1>=bx0 && x0<=bx1 && y1>=by0 && y0<=by1 ){ + p->eWithin = PARTLY_WITHIN; + }else{ + p->eWithin = NOT_WITHIN; + } + return SQLITE_OK; +} + /* END of implementation of "circle" geometry callback. ************************************************************************** *************************************************************************/ @@ -194,11 +356,7 @@ static int gHere = 42; static int cube_geom( sqlite3_rtree_geometry *p, int nCoord, -#ifdef SQLITE_RTREE_INT_ONLY - sqlite3_int64 *aCoord, -#else - double *aCoord, -#endif + sqlite3_rtree_dbl *aCoord, int *piRes ){ Cube *pCube = (Cube *)p->pUser; @@ -293,6 +451,14 @@ static int register_circle_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "circle", circle_geom, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_rtree_query_callback(db, "Qcircle", + circle_query_func, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_rtree_query_callback(db, "breadthfirstsearch", + bfs_query_func, 0, 0); + } Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; diff --git a/src/test_vfstrace.c b/src/test_vfstrace.c index c1792f116e..d0bc29f0c3 100644 --- a/src/test_vfstrace.c +++ b/src/test_vfstrace.c @@ -678,7 +678,7 @@ static int vfstraceAccess( vfstrace_info *pInfo = (vfstrace_info*)pVfs->pAppData; sqlite3_vfs *pRoot = pInfo->pRootVfs; int rc; - vfstrace_printf(pInfo, "%s.xDelete(\"%s\",%d)", + vfstrace_printf(pInfo, "%s.xAccess(\"%s\",%d)", pInfo->zVfsName, zPath, flags); rc = pRoot->xAccess(pRoot, zPath, flags, pResOut); vfstrace_print_errcode(pInfo, " -> %s", rc); diff --git a/test/show_speedtest1_rtree.tcl b/test/show_speedtest1_rtree.tcl new file mode 100644 index 0000000000..3faa168146 --- /dev/null +++ b/test/show_speedtest1_rtree.tcl @@ -0,0 +1,57 @@ +#!/usr/bin/tclsh +# +# This script displays the field of rectangles used by --testset rtree +# of speedtest1. Run this script as follows: +# +# rm test.db +# ./speedtest1 --testset rtree --size 25 test.db +# sqlite3 --separator ' ' test.db 'SELECT * FROM rt1' >data.txt +# wish show_speedtest1_rtree.tcl +# +# The filename "data.txt" is hard coded into this script and so that name +# must be used on lines 3 and 4 above. Elsewhere, different filenames can +# be used. The --size N parameter can be adjusted as desired. +# +package require Tk +set f [open data.txt rb] +set data [read $f] +close $f +canvas .c +frame .b +button .b.b1 -text X-Y -command refill-xy +button .b.b2 -text X-Z -command refill-xz +button .b.b3 -text Y-Z -command refill-yz +pack .b.b1 .b.b2 .b.b3 -side left +pack .c -side top -fill both -expand 1 +pack .b -side top +proc resize_canvas_to_fit {} { + foreach {x0 y0 x1 y1} [.c bbox all] break + set w [expr {$x1-$x0}] + set h [expr {$y1-$y0}] + .c config -width $w -height $h +} +proc refill-xy {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $x0 $y0 $x1 $y1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +proc refill-xz {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $x0 $z0 $x1 $z1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +proc refill-yz {} { + .c delete all + foreach {id x0 x1 y0 y1 z0 z1} $::data { + .c create rectangle $y0 $z0 $y1 $z1 + } + .c scale all 0 0 0.05 0.05 + resize_canvas_to_fit +} +refill-xy diff --git a/test/speedtest1.c b/test/speedtest1.c index 28c24e0885..383f5809a9 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -29,6 +29,7 @@ static const char zHelp[] = " --trace Turn on SQL tracing\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" + " --verify Run additional verification steps.\n" " --without-rowid Use WITHOUT ROWID where appropriate\n" ; @@ -51,6 +52,7 @@ static struct Global { int bReprepare; /* True to reprepare the SQL on each rerun */ int bSqlOnly; /* True to print the SQL once only */ int bExplain; /* Print SQL with EXPLAIN prefix */ + int bVerify; /* Try to verify that results are correct */ int szTest; /* Scale factor for test iterations */ const char *zWR; /* Might be WITHOUT ROWID */ const char *zNN; /* Might be NOT NULL */ @@ -931,6 +933,183 @@ void testset_cte(void){ } +/* Generate two numbers between 1 and mx. The first number is less than +** the second. Usually the numbers are near each other but can sometimes +** be far apart. +*/ +static void twoCoords( + int p1, int p2, /* Parameters adjusting sizes */ + unsigned mx, /* Range of 1..mx */ + unsigned *pX0, unsigned *pX1 /* OUT: write results here */ +){ + unsigned d, x0, x1, span; + + span = mx/100 + 1; + if( speedtest1_random()%3==0 ) span *= p1; + if( speedtest1_random()%p2==0 ) span = mx/2; + d = speedtest1_random()%span + 1; + x0 = speedtest1_random()%(mx-d) + 1; + x1 = x0 + d; + *pX0 = x0; + *pX1 = x1; +} + +/* The following routine is an R-Tree geometry callback. It returns +** true if the object overlaps a slice on the Y coordinate between the +** two values given as arguments. In other words +** +** SELECT count(*) FROM rt1 WHERE id MATCH xslice(10,20); +** +** Is the same as saying: +** +** SELECT count(*) FROM rt1 WHERE y1>=10 AND y0<=20; +*/ +static int xsliceGeometryCallback( + sqlite3_rtree_geometry *p, + int nCoord, + double *aCoord, + int *pRes +){ + *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1]; + return SQLITE_OK; +} + +/* +** A testset for the R-Tree virtual table +*/ +void testset_rtree(int p1, int p2){ + unsigned i, n; + unsigned mxCoord; + unsigned x0, x1, y0, y1, z0, z1; + unsigned iStep; + int *aCheck = sqlite3_malloc( sizeof(int)*g.szTest*100 ); + + mxCoord = 15000; + n = g.szTest*100; + speedtest1_begin_test(100, "%d INSERTs into an r-tree", n); + speedtest1_exec("BEGIN"); + speedtest1_exec("CREATE VIRTUAL TABLE rt1 USING rtree(id,x0,x1,y0,y1,z0,z1)"); + speedtest1_prepare("INSERT INTO rt1(id,x0,x1,y0,y1,z0,z1)" + "VALUES(?1,?2,?3,?4,?5,?6,?7)"); + for(i=1; i<=n; i++){ + twoCoords(p1, p2, mxCoord, &x0, &x1); + twoCoords(p1, p2, mxCoord, &y0, &y1); + twoCoords(p1, p2, mxCoord, &z0, &z1); + sqlite3_bind_int(g.pStmt, 1, i); + sqlite3_bind_int(g.pStmt, 2, x0); + sqlite3_bind_int(g.pStmt, 3, x1); + sqlite3_bind_int(g.pStmt, 4, y0); + sqlite3_bind_int(g.pStmt, 5, y1); + sqlite3_bind_int(g.pStmt, 6, z0); + sqlite3_bind_int(g.pStmt, 7, z1); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + speedtest1_begin_test(101, "Copy from rtree to a regular table"); + speedtest1_exec("CREATE TABLE t1(id INTEGER PRIMARY KEY,x0,x1,y0,y1,z0,z1)"); + speedtest1_exec("INSERT INTO t1 SELECT * FROM rt1"); + speedtest1_end_test(); + + n = g.szTest*20; + speedtest1_begin_test(110, "%d one-dimensional intersect slice queries", n); + speedtest1_prepare("SELECT count(*) FROM rt1 WHERE x0>=?1 AND x1<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND x1<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND y0<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND y0<=?2"); + iStep = mxCoord/n; + for(i=0; i=?1 AND x0<=?2" + " AND y1>=?1 AND y0<=?2 AND z1>=?1 AND z0<=?2"); + iStep = mxCoord/n; + for(i=0; i Date: Mon, 28 Apr 2014 19:34:06 +0000 Subject: [PATCH 19/36] The trunk assumes that an open range constraint on an indexed term (col>?) term matches 1/4 of the indexed rows, and that a closed constraint (col BETWEEN ? AND ?) matches 1/64. Change this branch to do the same. FossilOrigin-Name: 4047ac75e2a8f0b330255501c42e4f04e5ab500d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 05f3d798c0..b037731f09 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sunordered.test\sto\stake\sinto\saccount\sfor\sthe\sfact\sthat\sSQLite\snow\sprefers\sa\sfull-table\sscan\sover\sa\snon-covering\sindex\sscan\sthat\svisits\sa\slarge\spercentage\sof\sthe\stable\srows. -D 2014-04-28T15:11:25.118 +C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. +D 2014-04-28T19:34:06.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 263f8940aa21adabbc397590046f040c54aca5f4 +F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1b95544f84bf83c28cc15f6d0690fdf8a6bb3941 -R 307957877e44688fb3b83b922c0183e6 +P 20f468dfbcb247e51446fad411a6e6cc0d130411 +R eabc843f8d46cb0d3b0b7dd8faf4ef00 U dan -Z 085ae09cb0f2460f340245b5be0529cf +Z 37bdf577e72e4cd110b1edb4853d30e0 diff --git a/manifest.uuid b/manifest.uuid index 8fd0bd92ad..405d43fc27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20f468dfbcb247e51446fad411a6e6cc0d130411 \ No newline at end of file +4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4c1d81e183..4b2eef9c52 100644 --- a/src/where.c +++ b/src/where.c @@ -2155,6 +2155,12 @@ static int whereRangeScanEst( assert( pLower || pUpper ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is + ** reduced by an additional 75%. This means that, by default, an open-ended + ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the + ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to + ** match 1/64 of the index. */ + if( pLower && pUpper ) nNew -= 20; nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNew Date: Mon, 28 Apr 2014 20:11:20 +0000 Subject: [PATCH 20/36] Do not reduce the number of rows scanned at all for "IS NOT NULL" constraints. Fix a bug in calculating the number of rows visited by scans of partial indicies. FossilOrigin-Name: a8ae93f0cffa116df0ba34d46a53f49d42dace41 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/where.c | 8 ++-- test/cost.test | 95 ++++++++++++++++++++++++++++++++++++++++++++++++ test/index6.test | 5 ++- 5 files changed, 112 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b037731f09..823653ac7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\strunk\sassumes\sthat\san\sopen\srange\sconstraint\son\san\sindexed\sterm\s(col>?)\sterm\smatches\s1/4\sof\sthe\sindexed\srows,\sand\sthat\sa\sclosed\sconstraint\s(col\sBETWEEN\s?\sAND\s?)\smatches\s1/64.\sChange\sthis\sbranch\sto\sdo\sthe\ssame. -D 2014-04-28T19:34:06.281 +C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. +D 2014-04-28T20:11:20.674 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0a518940065c10ad8dedf8f2d6e73f1e14eb8472 +F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 39d014a90b67169f8482ede8713bff694761d879 +F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -611,7 +611,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test a0a2d286ffa6d35813f5003fdb7be124825b4422 +F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 20f468dfbcb247e51446fad411a6e6cc0d130411 -R eabc843f8d46cb0d3b0b7dd8faf4ef00 +P 4047ac75e2a8f0b330255501c42e4f04e5ab500d +R fc72ae401d0e8a316c9dd5e0ec98069f U dan -Z 37bdf577e72e4cd110b1edb4853d30e0 +Z f6f096240da2e8d5cf26af9c9310dce5 diff --git a/manifest.uuid b/manifest.uuid index 405d43fc27..99d11f9b9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4047ac75e2a8f0b330255501c42e4f04e5ab500d \ No newline at end of file +a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4b2eef9c52..8abe6bd1a1 100644 --- a/src/where.c +++ b/src/where.c @@ -1992,9 +1992,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ if( pTerm ){ if( pTerm->truthProb<=0 ){ nRet += pTerm->truthProb; - }else if( pTerm->wtFlags & TERM_VNULL ){ - nRet -= 10; assert( 10==sqlite3LogEst(2) ); - }else{ + }else if( (pTerm->wtFlags & TERM_VNULL)==0 ){ nRet -= 20; assert( 20==sqlite3LogEst(4) ); } } @@ -2153,14 +2151,17 @@ static int whereRangeScanEst( UNUSED_PARAMETER(pBuilder); #endif assert( pLower || pUpper ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); + /* TUNING: If there is both an upper and lower limit, assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ if( pLower && pUpper ) nNew -= 20; + nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; if( nNewiTab, pWC, pProbe->pPartIdxWhere) ){ continue; /* Partial index inappropriate for this query */ } + rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; pNew->u.btree.nSkip = 0; pNew->nLTerm = 0; diff --git a/test/cost.test b/test/cost.test index faaa7bdf16..0f9314fc8e 100644 --- a/test/cost.test +++ b/test/cost.test @@ -89,6 +89,101 @@ do_eqp_test 4.3 { } +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t2(x, y); + CREATE INDEX t2i1 ON t2(x); +} + +do_eqp_test 5.2 { + SELECT * FROM t2 ORDER BY x, y; +} {} +#exit + +# TODO: Check this one out!! +# set sqlite_where_trace 0xfff +do_eqp_test 5.3 { + SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; +} {} +#exit + +# where7.test, where8.test: +# +do_execsql_test 6.1 { + CREATE TABLE t3(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t3i1 ON t3(b); + CREATE INDEX t3i2 ON t3(c); +} + +#set sqlite_where_trace 0xfff +# eqp.test +do_eqp_test 6.2 { + SELECT a FROM t3 WHERE (b BETWEEN 2 AND 4) OR c=100 ORDER BY a +} { +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY,b,c,d,e,f,g); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + CREATE INDEX t1d ON t1(d); + CREATE INDEX t1e ON t1(e); + CREATE INDEX t1f ON t1(f); + CREATE INDEX t1g ON t1(g); +} + +do_eqp_test 7.2 { + SELECT a FROM t1 + WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) + ORDER BY a +} { +} + +#set sqlite_where_trace 0xfff +do_eqp_test 7.3 { + SELECT rowid FROM t1 + WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) + OR (b NOT NULL AND c IS NULL AND d NOT NULL) + OR (b NOT NULL AND c NOT NULL AND d IS NULL) +} {} +#exit + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE composer( + cid INTEGER PRIMARY KEY, + cname TEXT + ); + CREATE TABLE album( + aid INTEGER PRIMARY KEY, + aname TEXT + ); + CREATE TABLE track( + tid INTEGER PRIMARY KEY, + cid INTEGER REFERENCES composer, + aid INTEGER REFERENCES album, + title TEXT + ); + CREATE INDEX track_i1 ON track(cid); + CREATE INDEX track_i2 ON track(aid); +} + +do_eqp_test 8.2 { + SELECT DISTINCT aname + FROM album, composer, track + WHERE cname LIKE '%bach%' + AND unlikely(composer.cid=track.cid) + AND unlikely(album.aid=track.aid); +} { +} + finish_test diff --git a/test/index6.test b/test/index6.test index 3451e5c1df..68bdd06c14 100644 --- a/test/index6.test +++ b/test/index6.test @@ -145,11 +145,11 @@ do_test index6-2.1 { execsql { CREATE TABLE t2(a,b); INSERT INTO t2(a,b) SELECT value, value FROM nums WHERE value<1000; - UPDATE t2 SET a=NULL WHERE b%5==0; + UPDATE t2 SET a=NULL WHERE b%2==0; CREATE INDEX t2a1 ON t2(a) WHERE a IS NOT NULL; SELECT count(*) FROM t2 WHERE a IS NOT NULL; } -} {800} +} {500} do_test index6-2.2 { execsql { EXPLAIN QUERY PLAN @@ -157,6 +157,7 @@ do_test index6-2.2 { } } {/.* TABLE t2 USING INDEX t2a1 .*/} ifcapable stat4||stat3 { + execsql ANALYZE do_test index6-2.3stat4 { execsql { EXPLAIN QUERY PLAN From 67a03cfd6e5213f1b7997356d5db620bd21e5329 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 12:01:35 +0000 Subject: [PATCH 21/36] Fix a test case so that it updates sqlite_stat1 consistently. FossilOrigin-Name: 2dc5a0b55567f13f0528ed17242e680cde2f2a29 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/skipscan2.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 823653ac7d..a7e8fbdb50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreduce\sthe\snumber\sof\srows\sscanned\sat\sall\sfor\s"IS\sNOT\sNULL"\sconstraints.\sFix\sa\sbug\sin\scalculating\sthe\snumber\sof\srows\svisited\sby\sscans\sof\spartial\sindicies. -D 2014-04-28T20:11:20.674 +C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. +D 2014-04-29T12:01:35.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -814,7 +814,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f -F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d +F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4047ac75e2a8f0b330255501c42e4f04e5ab500d -R fc72ae401d0e8a316c9dd5e0ec98069f +P a8ae93f0cffa116df0ba34d46a53f49d42dace41 +R 176e01464114f39a9f52a0420836d3f7 U dan -Z f6f096240da2e8d5cf26af9c9310dce5 +Z 9c22069b17d97b138b62636ee980813e diff --git a/manifest.uuid b/manifest.uuid index 99d11f9b9b..5debc1c5f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8ae93f0cffa116df0ba34d46a53f49d42dace41 \ No newline at end of file +2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file diff --git a/test/skipscan2.test b/test/skipscan2.test index e39b16ed27..27d193e94d 100644 --- a/test/skipscan2.test +++ b/test/skipscan2.test @@ -74,6 +74,7 @@ do_execsql_test skipscan2-1.4 { -- of a skip-scan. So make a manual adjustment to the stat1 table -- to make it seem like there are many more. UPDATE sqlite_stat1 SET stat='10000 5000 20' WHERE idx='people_idx1'; + UPDATE sqlite_stat1 SET stat='10000 1' WHERE idx='sqlite_autoindex_people_1'; ANALYZE sqlite_master; } db cache flush From 09e1df6c009bc90558ec0250f005bb59fe8f808e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Apr 2014 16:10:22 +0000 Subject: [PATCH 22/36] Ignore likelihood() values on indexed IPK lookups. FossilOrigin-Name: 5bb7757a7b32a74482d3e93e9c9eea02273fe981 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- test/cost.test | 36 ++++++++++++++++++++++-------------- 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a7e8fbdb50..cf16f91a6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stest\scase\sso\sthat\sit\supdates\ssqlite_stat1\sconsistently. -D 2014-04-29T12:01:35.119 +C Ignore\slikelihood()\svalues\son\sindexed\sIPK\slookups. +D 2014-04-29T16:10:22.104 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da9c2c1d0c2ecf51ea3b7a27c826999972f24086 +F src/where.c 397bd5d4f402238c396d73b3c98b9fa2c312eea4 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -406,7 +406,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 84473f27749e0f3b6837a8e1403967010c347ca5 +F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1162,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a8ae93f0cffa116df0ba34d46a53f49d42dace41 -R 176e01464114f39a9f52a0420836d3f7 +P 2dc5a0b55567f13f0528ed17242e680cde2f2a29 +R b5feca3cdcf90dbcdc4d81625fcb13e8 U dan -Z 9c22069b17d97b138b62636ee980813e +Z d7294cbb283905147264033ecb5a94d9 diff --git a/manifest.uuid b/manifest.uuid index 5debc1c5f8..0097b17163 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2dc5a0b55567f13f0528ed17242e680cde2f2a29 \ No newline at end of file +5bb7757a7b32a74482d3e93e9c9eea02273fe981 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8abe6bd1a1..566c907e55 100644 --- a/src/where.c +++ b/src/where.c @@ -4189,7 +4189,7 @@ static int whereLoopAddBtreeIndex( assert( eOp & (WO_ISNULL|WO_EQ|WO_IN) ); assert( pNew->nOut==saved_nOut ); - if( pTerm->truthProb<=0 ){ + if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; diff --git a/test/cost.test b/test/cost.test index 0f9314fc8e..979815f9f7 100644 --- a/test/cost.test +++ b/test/cost.test @@ -66,10 +66,10 @@ do_eqp_test 3.2 { #------------------------------------------------------------------------- # If there is no likelihood() or stat3 data, SQLite assumes that a closed # range scan (e.g. one constrained by "col BETWEEN ? AND ?" constraint) -# visits 1/16 of the rows in a table. +# visits 1/64 of the rows in a table. # -# Note: 1/17 =~ 0.058 -# Note: 1/15 =~ 0.067 +# Note: 1/63 =~ 0.016 +# Note: 1/65 =~ 0.015 # reset_db do_execsql_test 4.1 { @@ -78,12 +78,12 @@ do_execsql_test 4.1 { CREATE INDEX i2 ON t1(b); } do_eqp_test 4.2 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.058) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.014) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (a=?)} } do_eqp_test 4.3 { - SELECT * FROM t1 WHERE likelihood(a=?, 0.067) AND b BETWEEN ? AND ?; + SELECT * FROM t1 WHERE likelihood(a=?, 0.016) AND b BETWEEN ? AND ?; } { 0 0 0 {SEARCH TABLE t1 USING INDEX i2 (b>? AND b? AND x? AND b=950 AND b<=1010) OR (b IS NULL AND c NOT NULL) ORDER BY a } { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b Date: Tue, 29 Apr 2014 19:01:57 +0000 Subject: [PATCH 23/36] Test that the default values used when sqlite_stat1 data is not available are calculated correctly. Fixes for the same. FossilOrigin-Name: e2d42f909de85a0586389f2dc0e654f7af2e351a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 35 +++++++++++++++-------------------- test/cost.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 5dc1bf6a38..f4b5efc137 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-04-29T16:46:24.176 +C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. +D 2014-04-29T19:01:57.481 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 6c9b51abd404ce5b78b173b6f2248e8cb824758c F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 9f7b2ed2af66dd2d186c0835d1c2a672d1f768e0 +F src/build.c 02665ca158431a0926b10cbd7d8178a4c9fc4a22 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 62386ccac862ea9a808a0954037f430690974b0f +F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5bb7757a7b32a74482d3e93e9c9eea02273fe981 af2cbe64adab5f9e3b0f3da00d06428088589d7f -R 9c74729ef8e10b97c890b0be6d308bd3 +P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e +R e73cccb2b7e246d7f5476a97d396afde U dan -Z e5760c6e9001de9044df542c0c7c165c +Z 19f8efc43d85a395f7ff59d4035b6f55 diff --git a/manifest.uuid b/manifest.uuid index e9d3392789..92aa1b2d7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e \ No newline at end of file +e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file diff --git a/src/build.c b/src/build.c index fb025495dc..a79abed3b6 100644 --- a/src/build.c +++ b/src/build.c @@ -3260,32 +3260,27 @@ exit_create_index: ** are based on typical values found in actual indices. */ void sqlite3DefaultRowEst(Index *pIdx){ -#if 0 - tRowcnt *a = pIdx->aiRowEst; - int i; - tRowcnt n; - assert( a!=0 ); - a[0] = pIdx->pTable->nRowEst; - if( a[0]<10 ) a[0] = 10; - n = 10; - for(i=1; i<=pIdx->nKeyCol; i++){ - a[i] = n; - if( n>5 ) n--; - } - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 1; - } -#endif - /* 1000000, 10, 9, 8, 7, 6, 5, 4, 3, 2 */ - LogEst aVal[] = { 33, 32, 30, 28, 26, 23, 20, 16, 10 }; + /* 10, 9, 8, 7, 6 */ + LogEst aVal[] = { 33, 32, 30, 28, 26 }; LogEst *a = pIdx->aiRowLogEst; int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); + int i; + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table. Or 10, if the estimated number of rows + ** in the table is less than that. */ a[0] = pIdx->pTable->nRowLogEst; + if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + + /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is + ** 6 and each subsequent value (if any) is 5. */ memcpy(&a[1], aVal, nCopy*sizeof(LogEst)); - if( pIdx->onError!=OE_None ){ - a[pIdx->nKeyCol] = 0; + for(i=nCopy+1; i<=pIdx->nKeyCol; i++){ + a[i] = 23; assert( 23==sqlite3LogEst(5) ); } + + assert( 0==sqlite3LogEst(1) ); + if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; } /* diff --git a/test/cost.test b/test/cost.test index 979815f9f7..e04dc9392a 100644 --- a/test/cost.test +++ b/test/cost.test @@ -192,6 +192,51 @@ do_eqp_test 8.2 { 0 0 0 {USE TEMP B-TREE FOR DISTINCT} } +#------------------------------------------------------------------------- +# +do_execsql_test 9.1 { + CREATE TABLE t1( + a,b,c,d,e, f,g,h,i,j, + k,l,m,n,o, p,q,r,s,t + ); + CREATE INDEX i1 ON t1(k,l,m,n,o,p,q,r,s,t); +} +do_test 9.2 { + for {set i 0} {$i < 100} {incr i} { + execsql { INSERT INTO t1 DEFAULT VALUES } + } + execsql { + ANALYZE; + CREATE INDEX i2 ON t1(a,b,c,d,e,f,g,h,i,j); + } +} {} + +set L [list a=? b=? c=? d=? e=? f=? g=? h=? i=? j=?] +foreach {tn nTerm nRow} { + 1 1 10 + 2 2 9 + 3 3 8 + 4 4 7 + 5 5 6 + 6 6 5 + 7 7 5 + 8 8 5 + 9 9 5 + 10 10 5 +} { + set w [join [lrange $L 0 [expr $nTerm-1]] " AND "] + set p1 [expr ($nRow-1) / 100.0] + set p2 [expr ($nRow+1) / 100.0] + + set sql1 "SELECT * FROM t1 WHERE likelihood(k=?, $p1) AND $w" + set sql2 "SELECT * FROM t1 WHERE likelihood(k=?, $p2) AND $w" + + do_eqp_test 9.3.$tn.1 $sql1 {/INDEX i1/} + do_eqp_test 9.3.$tn.2 $sql2 {/INDEX i2/} +} + + + finish_test From 224155dd1c165a133279ebaa52f80f771844cedc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 13:19:09 +0000 Subject: [PATCH 24/36] Fix long-standing typos in comments. FossilOrigin-Name: b9f91317c34d07769a95dc2f905a6ccabceb64a3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 4 ++-- src/util.c | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f4b5efc137..44e14c4938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\sthat\sthe\sdefault\svalues\sused\swhen\ssqlite_stat1\sdata\sis\snot\savailable\sare\scalculated\scorrectly.\sFixes\sfor\sthe\ssame. -D 2014-04-29T19:01:57.481 +C Fix\slong-standing\stypos\sin\scomments. +D 2014-04-30T13:19:09.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 63656cfa5a8221c3eb1a182e97d61b1fe2dfd7da +F src/sqliteInt.h b2947801eccefd7ba3e5f14e1353289351a83cf3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c -F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf +F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d74299f037f3a6a4b3bce8b4d1c76c407c1f3b3e -R e73cccb2b7e246d7f5476a97d396afde -U dan -Z 19f8efc43d85a395f7ff59d4035b6f55 +P e2d42f909de85a0586389f2dc0e654f7af2e351a +R cdeda4c4d8ae1ea7276fce5a8779e48d +U drh +Z 6915f97ffb226384426800091a7da39d diff --git a/manifest.uuid b/manifest.uuid index 92aa1b2d7c..42cd607e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2d42f909de85a0586389f2dc0e654f7af2e351a \ No newline at end of file +b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b584ff401d..4bd65a690f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -525,10 +525,10 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantatites are suppose to be estimates, +** of 40. However, since LogEst quantaties are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** -** "LogEst" is short for "Logarithimic Estimate". +** "LogEst" is short for "Logarithmic Estimate". ** ** Examples: ** 1 -> 0 20 -> 43 10000 -> 132 diff --git a/src/util.c b/src/util.c index d88c17b759..577d552ac3 100644 --- a/src/util.c +++ b/src/util.c @@ -1246,8 +1246,8 @@ LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ } /* -** Convert an integer into a LogEst. In other words, compute a -** good approximatation for 10*log2(x). +** Convert an integer into a LogEst. In other words, compute an +** approximation for 10*log2(x). */ LogEst sqlite3LogEst(u64 x){ static LogEst a[] = { 0, 2, 3, 5, 6, 7, 8, 9 }; From 253666e5203974b28d990e321b243153f3a432dd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Apr 2014 14:22:38 +0000 Subject: [PATCH 25/36] Improved rendering of LogEst values corresponding to real values near 0.0 in the tool/logest.c utility program. FossilOrigin-Name: 32910c8c595858245bb7ecfe3aa0f90eeae641af --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/logest.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 44e14c4938..29bd3a7832 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\slong-standing\stypos\sin\scomments. -D 2014-04-30T13:19:09.070 +C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. +D 2014-04-30T14:22:38.579 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1128,7 +1128,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 +F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2d42f909de85a0586389f2dc0e654f7af2e351a -R cdeda4c4d8ae1ea7276fce5a8779e48d +P b9f91317c34d07769a95dc2f905a6ccabceb64a3 +R 0e5a317f465f102c0002288cd477a07d U drh -Z 6915f97ffb226384426800091a7da39d +Z 88c640e8cc6f42fa637c9ebb251cd8ac diff --git a/manifest.uuid b/manifest.uuid index 42cd607e10..7421cca676 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9f91317c34d07769a95dc2f905a6ccabceb64a3 \ No newline at end of file +32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 1ac337d36c..347fa68a4f 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -83,7 +83,8 @@ static LogEst logEstFromDouble(double x){ LogEst e; assert( sizeof(x)==8 && sizeof(a)==8 ); if( x<=0.0 ) return -32768; - if( x<1.0 ) return -logEstFromDouble(1/x); + if( x<0.01 ) return -logEstFromDouble(1.0/x); + if( x<1.0 ) return logEstFromDouble(100.0*x) - 66; if( x<1024.0 ) return logEstFromInteger((sqlite3_uint64)(1024.0*x)) - 100; if( x<=2000000000.0 ) return logEstFromInteger((sqlite3_uint64)x); memcpy(&a, &x, 8); @@ -156,8 +157,10 @@ int main(int argc, char **argv){ } } for(i=n-1; i>=0; i--){ - if( a[i]<0 ){ + if( a[i]<-40 ){ printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + }else if( a[i]<10 ){ + printf("%5d (%f)\n", a[i], logEstToInt(a[i]+100)/1024.0); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); From 4a6b8a05cd203606f227263f9146aa7dff36c888 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:47:01 +0000 Subject: [PATCH 26/36] Fix a couple of out-of-date comments in where.c. FossilOrigin-Name: eefeda32d54efbbdf7d20b719299eda48b891fae --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 47 ++++++++++++++++++++++++++++++----------------- 3 files changed, 38 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29bd3a7832..8167b75420 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\srendering\sof\sLogEst\svalues\scorresponding\sto\sreal\svalues\snear\s0.0\nin\sthe\stool/logest.c\sutility\sprogram. -D 2014-04-30T14:22:38.579 +C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. +D 2014-04-30T14:47:01.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9651faf05f900d9235bf3166fa54a9e5f12a40f2 +F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b9f91317c34d07769a95dc2f905a6ccabceb64a3 -R 0e5a317f465f102c0002288cd477a07d -U drh -Z 88c640e8cc6f42fa637c9ebb251cd8ac +P 32910c8c595858245bb7ecfe3aa0f90eeae641af +R 26fff8d56fc56eff56bc1e57cba958ab +U dan +Z b9aa63fbb519903c6b0c5b0a6a8e051f diff --git a/manifest.uuid b/manifest.uuid index 7421cca676..6c9c8ac5b7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32910c8c595858245bb7ecfe3aa0f90eeae641af \ No newline at end of file +eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file diff --git a/src/where.c b/src/where.c index aea0be3042..ec7a908b1a 100644 --- a/src/where.c +++ b/src/where.c @@ -4022,8 +4022,13 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ } /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the index pIndex. -** Try to match one more. +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** index pIndex. Try to match one more. +** +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this +** function returns. ** ** If pProbe->tnum==0, that means pIndex is a fake index used for the ** INTEGER PRIMARY KEY. @@ -4085,10 +4090,14 @@ static int whereLoopAddBtreeIndex( /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. The magic - ** number 18 was found by experimentation to be the payoff point where - ** skip-scan become faster than a full-scan. - */ + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ assert( 42==sqlite3LogEst(18) ); if( pTerm==0 && saved_nEq==saved_nSkip @@ -5226,23 +5235,27 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** also N*log(N) but it has a larger constant of proportionality. - ** Multiply by 3.0. */ + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ rSortCost += 16; } From 75525cbe2423852e551e1843e5fad91acf803024 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 14:53:21 +0000 Subject: [PATCH 27/36] Update a couple of test cases to account for the fact that this branch prefers an index scan and partial sort over a full-table scan and full external sort. FossilOrigin-Name: 9b975bf33cd8fc28c64183a9642bf9fb436a4746 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/cost.test | 5 ++++- test/eqp.test | 12 ++++++------ 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8167b75420..9f27ec6f9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sout-of-date\scomments\sin\swhere.c. -D 2014-04-30T14:47:01.628 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. +D 2014-04-30T14:53:21.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 41b350fcc811a4fcccffbab06a3a79f33b8ea0de +F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -454,7 +454,7 @@ F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 -F test/eqp.test 57c6c604c2807fb5531731c5323133453c24afac +F test/eqp.test 90b56d03a93a2e7bb90f88be6083a8ea53f11a0e F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32910c8c595858245bb7ecfe3aa0f90eeae641af -R 26fff8d56fc56eff56bc1e57cba958ab +P eefeda32d54efbbdf7d20b719299eda48b891fae +R 50e654a1c0b26308b8a927bcd26c6421 U dan -Z b9aa63fbb519903c6b0c5b0a6a8e051f +Z f4d62e136a5d6e05753f5050c6fb0c4a diff --git a/manifest.uuid b/manifest.uuid index 6c9c8ac5b7..00e612df5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eefeda32d54efbbdf7d20b719299eda48b891fae \ No newline at end of file +9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file diff --git a/test/cost.test b/test/cost.test index e04dc9392a..045a4c08d6 100644 --- a/test/cost.test +++ b/test/cost.test @@ -99,7 +99,10 @@ do_execsql_test 5.1 { do_eqp_test 5.2 { SELECT * FROM t2 ORDER BY x, y; -} {} +} { + 0 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 0 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} +} do_eqp_test 5.3 { SELECT * FROM t2 WHERE x BETWEEN ? AND ? ORDER BY rowid; diff --git a/test/eqp.test b/test/eqp.test index 610ebdc5ca..9e2e9505cc 100644 --- a/test/eqp.test +++ b/test/eqp.test @@ -312,8 +312,8 @@ do_eqp_test 4.2.3 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION)} } do_eqp_test 4.2.4 { @@ -321,8 +321,8 @@ do_eqp_test 4.2.4 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (INTERSECT)} } do_eqp_test 4.2.5 { @@ -330,8 +330,8 @@ do_eqp_test 4.2.5 { } { 1 0 0 {SCAN TABLE t1} 1 0 0 {USE TEMP B-TREE FOR ORDER BY} - 2 0 0 {SCAN TABLE t2} - 2 0 0 {USE TEMP B-TREE FOR ORDER BY} + 2 0 0 {SCAN TABLE t2 USING INDEX t2i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (EXCEPT)} } From 8164722c5806410eb5443a2498f619475fee3fff Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 15:00:16 +0000 Subject: [PATCH 28/36] Add text to the header comment of whereLoopAddBtree() describing how the costs of various b-tree loops are estimated. FossilOrigin-Name: 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 9f27ec6f9b..35f78b46c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sthis\sbranch\sprefers\san\sindex\sscan\sand\spartial\ssort\sover\sa\sfull-table\sscan\sand\sfull\sexternal\ssort. -D 2014-04-30T14:53:21.345 +C Add\stext\sto\sthe\sheader\scomment\sof\swhereLoopAddBtree()\sdescribing\show\sthe\scosts\sof\svarious\sb-tree\sloops\sare\sestimated. +D 2014-04-30T15:00:16.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e91b92908c973b2b6b03145dcd131a8ca8b1bf64 +F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eefeda32d54efbbdf7d20b719299eda48b891fae -R 50e654a1c0b26308b8a927bcd26c6421 +P 9b975bf33cd8fc28c64183a9642bf9fb436a4746 +R 20e49e26218874ddfe5fb9438df8f580 U dan -Z f4d62e136a5d6e05753f5050c6fb0c4a +Z c1e32968c738a6da213a45755f91539c diff --git a/manifest.uuid b/manifest.uuid index 00e612df5f..6ee1deefef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b975bf33cd8fc28c64183a9642bf9fb436a4746 \ No newline at end of file +05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ec7a908b1a..7d01db61ee 100644 --- a/src/where.c +++ b/src/where.c @@ -4349,6 +4349,29 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Add all WhereLoop objects for a single table of the join where the table ** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. +** +** The costs (WhereLoop.rRun) of the b-tree loops added by this function +** are calculated as follows: +** +** For a full scan, assuming the table (or index) contains nRow rows: +** +** cost = nRow * 3.0 // full-table scan +** cost = nRow * K // scan of covering index +** cost = nRow * (K+3.0) // scan of non-covering index +** +** where K is a value between 1.1 and 3.0 set based on the relative +** estimated average size of the index and table records. +** +** For an index scan, where nVisit is the number of index rows visited +** by the scan, and nSeek is the number of seek operations required on +** the index b-tree: +** +** cost = nSeek * (log(nRow) + K * nVisit) // covering index +** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index +** +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ From 5da73e1a09b89f3605af1e438ac5e269969015f5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 30 Apr 2014 18:11:55 +0000 Subject: [PATCH 29/36] Fix a problem in calculating the costs of "OR" scans. FossilOrigin-Name: 9bbca48b42e4fe16f2188e18dc736da30a96435c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 17 +++++++++++++++-- test/cost.test | 7 ++++++- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c2ff19fbfe..88f6a39dac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sway\sthe\scosts\sof\svarious\squery\splans\sare\sestimated.\sIf\sthe\suser\ssupplies\sa\slikelihood()\svalue\s(or\sequivalent)\son\san\sindexed\sWHERE\sconstraint,\suse\sit\sto\sestimate\sthe\snumber\sof\sindex\srows\svisited. -D 2014-04-30T15:22:25.359 +C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. +D 2014-04-30T18:11:55.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4aeb1caa0a16c76e0c0566af4c64ba003836a0aa +F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -407,7 +407,7 @@ F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 -F test/cost.test 3f7904d623ef8dc6e55f2206db5ce0549077b438 +F test/cost.test 04842adec34311d70c3f3c5fd698b22be54138fd F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P af2cbe64adab5f9e3b0f3da00d06428088589d7f 05e6e16cb28c9ffb4596bd2ef81f687c5403ecbb -R 20e49e26218874ddfe5fb9438df8f580 +P 90e36676476e8db00658772e6c938242f766d306 +R cf0f5a4a25fa17ee63b35d342904cb95 U dan -Z 75da520dd73c7e9f7e5ee96ec52c5c59 +Z 15897e765882c5d27e5f655ffd8d3c94 diff --git a/manifest.uuid b/manifest.uuid index 437f1a2753..274547b53f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90e36676476e8db00658772e6c938242f766d306 \ No newline at end of file +9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7d01db61ee..3b96e5fbea 100644 --- a/src/where.c +++ b/src/where.c @@ -4738,7 +4738,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ int iCur; WhereClause tempWC; WhereLoopBuilder sSubBuild; - WhereOrSet sSum, sCur, sPrev; + WhereOrSet sSum, sCur; struct SrcList_item *pItem; pWC = pBuilder->pWC; @@ -4794,6 +4794,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ whereOrMove(&sSum, &sCur); once = 0; }else{ + WhereOrSet sPrev; whereOrMove(&sPrev, &sSum); sSum.n = 0; for(i=0; iiSortIdx = 0; memset(&pNew->u, 0, sizeof(pNew->u)); for(i=0; rc==SQLITE_OK && irRun = sSum.a[i].rRun; + /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs + ** of all sub-scans required by the OR-scan. However, due to rounding + ** errors, it may be that the cost of the OR-scan is equal to its + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that + ** this does not happen. Otherwise, for WHERE clauses such as the + ** following where there is an index on "y": + ** + ** WHERE likelihood(x=?, 0.99) OR y=? + ** + ** the planner may elect to "OR" together a full-table scan and an + ** index lookup. And other similarly odd results. */ + pNew->rRun = sSum.a[i].rRun + 1; pNew->nOut = sSum.a[i].nOut; pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); diff --git a/test/cost.test b/test/cost.test index 045a4c08d6..c413c3cdfa 100644 --- a/test/cost.test +++ b/test/cost.test @@ -150,7 +150,6 @@ do_eqp_test 7.2 { 0 0 0 {USE TEMP B-TREE FOR ORDER BY} } -#set sqlite_where_trace 0xfff do_eqp_test 7.3 { SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL AND d NOT NULL) @@ -160,6 +159,12 @@ do_eqp_test 7.3 { 0 0 0 {SCAN TABLE t1} } +do_eqp_test 7.4 { + SELECT rowid FROM t1 WHERE (+b IS NULL AND c NOT NULL) OR c IS NULL +} { + 0 0 0 {SCAN TABLE t1} +} + #------------------------------------------------------------------------- # reset_db From e03d76254c850555579aa203af78d3e75671383b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 10:19:16 +0000 Subject: [PATCH 30/36] Update a test case in wal2.test that explicitly corrupts a checksum in the wal file to account for the fact that the first byte of said checksum may initially be 0xFF. FossilOrigin-Name: 2b935bdea1452505f36dc8c7aad49e6c42f4eceb --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal2.test | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 88f6a39dac..c24d03cdc7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\scalculating\sthe\scosts\sof\s"OR"\sscans. -D 2014-04-30T18:11:55.566 +C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. +D 2014-05-01T10:19:16.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1057,7 +1057,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc -F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af +F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 90e36676476e8db00658772e6c938242f766d306 -R cf0f5a4a25fa17ee63b35d342904cb95 +P 9bbca48b42e4fe16f2188e18dc736da30a96435c +R 82092de2428e80a3b03a937e23116b0e U dan -Z 15897e765882c5d27e5f655ffd8d3c94 +Z f6ffad5e4f0572fc8e2909e59c29de84 diff --git a/manifest.uuid b/manifest.uuid index 274547b53f..98d646217d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bbca48b42e4fe16f2188e18dc736da30a96435c \ No newline at end of file +2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file diff --git a/test/wal2.test b/test/wal2.test index b331d5ed10..9d45444d6a 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -811,7 +811,13 @@ do_test wal2-7.1.1 { do_test wal2-7.1.2 { forcecopy test.db test2.db forcecopy test.db-wal test2.db-wal - hexio_write test2.db-wal 48 FF + # The first 32 bytes of the WAL file contain the WAL header. Offset 48 + # is the first byte of the checksum for the first frame in the WAL. + # The following three lines replaces the contents of that byte with + # a different value. + set newval FF + if {$newval == [hexio_read test2.db-wal 48 1]} { set newval 00 } + hexio_write test2.db-wal 48 $newval } {1} do_test wal2-7.1.3 { sqlite3 db2 test2.db From c5f246eb3747ce3ab28aff821774493560e84611 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 May 2014 20:24:21 +0000 Subject: [PATCH 31/36] Add #ifdefs for test coverage. Add a testcase(). FossilOrigin-Name: be2702ce35c713b33c9b7689643b45fb0de6af2a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 8 +++++++- src/where.c | 1 + 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c24d03cdc7..a856daec13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\stest\scase\sin\swal2.test\sthat\sexplicitly\scorrupts\sa\schecksum\sin\sthe\swal\sfile\sto\saccount\sfor\sthe\sfact\sthat\sthe\sfirst\sbyte\sof\ssaid\schecksum\smay\sinitially\sbe\s0xFF. -D 2014-05-01T10:19:16.340 +C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). +D 2014-05-01T20:24:21.190 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -159,7 +159,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 92f1495304dd33b4f9e0b0e5aa030b068ada504d +F src/analyze.c 3596f863bb80126fe56ba217df5932749271efc8 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3eaf3d241d86452c0f21aa3fe2b5df25d8c99a24 +F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9bbca48b42e4fe16f2188e18dc736da30a96435c -R 82092de2428e80a3b03a937e23116b0e -U dan -Z f6ffad5e4f0572fc8e2909e59c29de84 +P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb +R a1dff26029b41608141f15cdd82c4ee9 +U drh +Z 39b43641ac7d3a2ec2ad0946638e023c diff --git a/manifest.uuid b/manifest.uuid index 98d646217d..c97cdf989b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b935bdea1452505f36dc8c7aad49e6c42f4eceb \ No newline at end of file +be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4fbaaa5173..2952b364c3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1390,9 +1390,15 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( aOut ){ aOut[i] = v; - }else{ + }else +#else + assert( aOut==0 ); + UNUSED_PARAMETER(aOut); +#endif + { aLog[i] = sqlite3LogEst(v); } if( *z==' ' ) z++; diff --git a/src/where.c b/src/where.c index 3b96e5fbea..238d1ef42a 100644 --- a/src/where.c +++ b/src/where.c @@ -4200,6 +4200,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->nOut==saved_nOut ); if( pTerm->truthProb<=0 && iCol>=0 ){ assert( (eOp & WO_IN) || nIn==0 ); + testcase( eOp & WO_IN ); pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; pNew->wsFlags |= WHERE_LIKELIHOOD; From 38524132466e77081fc2262122d86d655eb36be3 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 1 May 2014 20:26:48 +0000 Subject: [PATCH 32/36] Fix an obscure problem to do with temp register allocation that could occur if more than one simple SELECT within a compound SELECT uses a partial sort. FossilOrigin-Name: 427409ae106cdab7892a6b50fe30c5f52de5addc --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/select.c | 12 ++++----- src/vdbe.c | 1 + test/selectA.test | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index a856daec13..c7b6bc3a26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s#ifdefs\sfor\stest\scoverage.\s\sAdd\sa\stestcase(). -D 2014-05-01T20:24:21.190 +C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. +D 2014-05-01T20:26:48.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c ed459f7f478a1e533d19c4b953693b3ffa2efd15 +F src/select.c 089c4d46f067a5cccae93524c6377f981ba99bd9 F src/shell.c 2afe7a7154e97be0c74c5feacf09626bda8493be F src/sqlite.h.in bde98816e1ba0c9ffef50afe7b32f4e5a8f54fe0 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -279,7 +279,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c 2b5fb283a190aacdb286f7835a447c45b345b83c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 699693bea6710ed436392c928b02cb4e91944137 +F src/vdbe.c 7f359193bf2366cc914a9ece093ebf284e56acdc F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -789,7 +789,7 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 -F test/selectA.test 77adaffe9704cb80e301ebaeff4b107b58d435c5 +F test/selectA.test 64b88a80271c1710966e50e633380696b60a12a4 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b935bdea1452505f36dc8c7aad49e6c42f4eceb -R a1dff26029b41608141f15cdd82c4ee9 -U drh -Z 39b43641ac7d3a2ec2ad0946638e023c +P be2702ce35c713b33c9b7689643b45fb0de6af2a +R 31df819a20c9dcbfb809510504bd5463 +U dan +Z aa2318c066a07655ea4b90fb2a4008ea diff --git a/manifest.uuid b/manifest.uuid index c97cdf989b..76ebe330ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be2702ce35c713b33c9b7689643b45fb0de6af2a \ No newline at end of file +427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file diff --git a/src/select.c b/src/select.c index dfca6d3f83..5fff010f34 100644 --- a/src/select.c +++ b/src/select.c @@ -466,15 +466,17 @@ static void pushOntoSorter( ){ Vdbe *v = pParse->pVdbe; int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); + int regRecord = ++pParse->nMem; + int regBase = pParse->nMem+1; int nOBSat = pSort->nOBSat; int op; + + pParse->nMem += nExpr+2; /* nExpr+2 registers allocated at regBase */ sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat,regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -511,10 +513,6 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( nOBSat==0 ){ - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); - } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; diff --git a/src/vdbe.c b/src/vdbe.c index 4ef6d0738c..a64e5bf5e1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4232,6 +4232,7 @@ case OP_SorterData: { pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); + assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); break; } diff --git a/test/selectA.test b/test/selectA.test index ca2ec38da4..6e593e8e22 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -21,6 +21,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix selectA ifcapable !compound { finish_test @@ -1310,4 +1311,68 @@ do_execsql_test selectA-3.98 { SELECT n FROM xyz ORDER BY +n; } {MAD MAD+ MAD++} +#------------------------------------------------------------------------- +# At one point the following code exposed a temp register reuse problem. +# +proc f {args} { return 1 } +db func f f + +do_execsql_test 4.1.1 { + CREATE TABLE t4(a, b); + CREATE TABLE t5(c, d); + + INSERT INTO t5 VALUES(1, 'x'); + INSERT INTO t5 VALUES(2, 'x'); + INSERT INTO t4 VALUES(3, 'x'); + INSERT INTO t4 VALUES(4, 'x'); + + CREATE INDEX i1 ON t4(a); + CREATE INDEX i2 ON t5(c); +} + +do_eqp_test 4.1.2 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 0 0 {SCAN TABLE t5 USING INDEX i2} + 1 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 2 0 0 {SCAN TABLE t4 USING INDEX i1} + 2 0 0 {USE TEMP B-TREE FOR RIGHT PART OF ORDER BY} + 0 0 0 {COMPOUND SUBQUERIES 1 AND 2 (UNION ALL)} +} + +do_execsql_test 4.1.3 { + SELECT c, d FROM t5 + UNION ALL + SELECT a, b FROM t4 WHERE f()==f() + ORDER BY 1,2 +} { + 1 x 2 x 3 x 4 x +} + +do_execsql_test 4.2.1 { + CREATE TABLE t6(a, b); + CREATE TABLE t7(c, d); + + INSERT INTO t7 VALUES(2, 9); + INSERT INTO t6 VALUES(3, 0); + INSERT INTO t6 VALUES(4, 1); + INSERT INTO t7 VALUES(5, 6); + INSERT INTO t6 VALUES(6, 0); + INSERT INTO t7 VALUES(7, 6); + + CREATE INDEX i6 ON t6(a); + CREATE INDEX i7 ON t7(c); +} + +do_execsql_test 4.2.2 { + SELECT c, f(d,c,d,c,d) FROM t7 + UNION ALL + SELECT a, b FROM t6 + ORDER BY 1,2 +} {/2 . 3 . 4 . 5 . 6 . 7 ./} + + finish_test From e724d3d26a6db490377d85b6015d6f85768b26dc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 00:09:40 +0000 Subject: [PATCH 33/36] Add a comment explaining why WhereLoop cost adjustments are omitted for skip-scan loops. FossilOrigin-Name: 3bc43594aaeee9225c0590677fcce480bedcb37b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c7b6bc3a26..af2b3b36ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sproblem\sto\sdo\swith\stemp\sregister\sallocation\sthat\scould\soccur\sif\smore\sthan\sone\ssimple\sSELECT\swithin\sa\scompound\sSELECT\suses\sa\spartial\ssort. -D 2014-05-01T20:26:48.793 +C Add\sa\scomment\sexplaining\swhy\sWhereLoop\scost\sadjustments\sare\somitted\sfor\nskip-scan\sloops. +D 2014-05-02T00:09:40.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9e67d6c48006445d964bc835ea82500c190c3a6f +F src/where.c 91dfd382273c3f67fcdc0ac4728f9140f91c6ab3 F src/whereInt.h 6804c2e5010378568c2bb1350477537755296a46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P be2702ce35c713b33c9b7689643b45fb0de6af2a -R 31df819a20c9dcbfb809510504bd5463 -U dan -Z aa2318c066a07655ea4b90fb2a4008ea +P 427409ae106cdab7892a6b50fe30c5f52de5addc +R ee65c8e54ebb299c9693aee246a7e200 +U drh +Z 5a37e5aacc1e047352493d523b6fb849 diff --git a/manifest.uuid b/manifest.uuid index 76ebe330ab..6c6f1f805d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -427409ae106cdab7892a6b50fe30c5f52de5addc \ No newline at end of file +3bc43594aaeee9225c0590677fcce480bedcb37b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 238d1ef42a..c5892590bc 100644 --- a/src/where.c +++ b/src/where.c @@ -3783,6 +3783,17 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. +** +** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the +** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE +** clause terms covered, since some of the first nLTerm entries in aLTerm[] +** will be NULL (because they are skipped). That makes it more difficult +** to compare the loops. We could add extra code to do the comparison, and +** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this +** adjustment is sufficient minor, that it is very difficult to construct +** a test case where the extra code would improve the query plan. Better +** to avoid the added complexity and just omit cost adjustments to SKIPSCAN +** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; From 7a0fd192ccf4efba952b38a0d5a7b38d18ba19fe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 15:25:24 +0000 Subject: [PATCH 34/36] Fix a faulty assert() statement. FossilOrigin-Name: 9196ce407379ca3b151b601b98848771e5cb4e8f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6166275bb9..789773a941 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\senhancements\sand\sfixes\sinto\sthe\sorderby-planning\sbranch. -D 2014-05-02T13:09:06.754 +C Fix\sa\sfaulty\sassert()\sstatement. +D 2014-05-02T15:25:24.157 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 +F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bf09ce24d054bc68c226064f5f28d97e0e648a13 3bc43594aaeee9225c0590677fcce480bedcb37b -R 54b60e57b7c4903b7427b2471b549ff9 +P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d +R f6e2655ab9521eb2aa3af53a52824666 U drh -Z 3f220c7986d95ea2653ad2dab9b5d42e +Z 7c7e4b02db28ce1795a4d7aa65a0e17d diff --git a/manifest.uuid b/manifest.uuid index 2809ea21e5..a5c49533a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84862d3a095629d20c8e7b8a16f4dc26cd41ab6d \ No newline at end of file +9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6a34cefa61..70fbbcf425 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -733,10 +733,10 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ while( p ){ SorterRecord *pNext; if( pSorter->aMemory ){ - assert( p->u.iNextnMemory ); if( (u8*)p==pSorter->aMemory ){ pNext = 0; }else{ + assert( p->u.iNextnMemory ); pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; } }else{ From c7f6c148d9b4dcf7c19b3edd19c82d8738d4048e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 16:22:55 +0000 Subject: [PATCH 35/36] Failure to extend a temp file for use with mmap() in vdbesort.c is benign. FossilOrigin-Name: d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 789773a941..d6c9c71387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sstatement. -D 2014-05-02T15:25:24.157 +C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. +D 2014-05-02T16:22:55.791 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 +F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d -R f6e2655ab9521eb2aa3af53a52824666 +P 9196ce407379ca3b151b601b98848771e5cb4e8f +R 51e21c516ddbd5830d43ea54f991a56e U drh -Z 7c7e4b02db28ce1795a4d7aa65a0e17d +Z 19bdd7c6b9f54ab4b2fbcff19d13a623 diff --git a/manifest.uuid b/manifest.uuid index a5c49533a7..e8d80914a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file +d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 70fbbcf425..86fef69256 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -857,17 +857,16 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ +static void vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } - return rc; } #else -# define vdbeSorterExtendFile(x,y) SQLITE_OK +# define vdbeSorterExtendFile(x,y) #endif /* @@ -907,8 +906,8 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile( - pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 ); } @@ -1132,7 +1131,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); } } From 79211e194d465fba319faae7fbf70774a12415ae Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 17:33:16 +0000 Subject: [PATCH 36/36] Simplify assert() statements used to verify correct operation of record comparison routines. FossilOrigin-Name: 3300d62dcbe74842cf86ca436959fe4e77a89f84 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 40 +++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index d6c9c71387..8fd72b4b86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. -D 2014-05-02T16:22:55.791 +C Simplify\sassert()\sstatements\sused\sto\sverify\scorrect\soperation\sof\s\nrecord\scomparison\sroutines. +D 2014-05-02T17:33:16.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbe.c b3510cc71f706beffc66e2aa4bbda54bcd5e9668 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 +F src/vdbeaux.c c9a8c917776c941af99285594d5c30d99e21c99a F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9196ce407379ca3b151b601b98848771e5cb4e8f -R 51e21c516ddbd5830d43ea54f991a56e +P d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 +R defe6572bf21263fd4b2b33706bcd8e7 U drh -Z 19bdd7c6b9f54ab4b2fbcff19d13a623 +Z 26e6dec065eafceab8263c10d377378f diff --git a/manifest.uuid b/manifest.uuid index e8d80914a0..ea44721a8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file +3300d62dcbe74842cf86ca436959fe4e77a89f84 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a6b536720..172d12042b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3137,10 +3137,14 @@ void sqlite3VdbeRecordUnpack( ** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used ** in assert() statements to ensure that the optimized code in ** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -3202,7 +3206,7 @@ static int vdbeRecordCompareDebug( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } - return rc; + goto debugCompareEnd; } i++; }while( idx1nField ); @@ -3216,7 +3220,15 @@ static int vdbeRecordCompareDebug( /* rc==0 here means that one of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - return pPKey2->default_rc; + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; } #endif @@ -3564,11 +3576,7 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB - || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || pKeyInfo->db->mallocFailed - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.zMalloc==0 ); /* See comment below */ return rc; } @@ -3587,9 +3595,7 @@ int sqlite3VdbeRecordCompare( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( CORRUPT_DB - || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) ); return pPKey2->default_rc; } @@ -3686,11 +3692,7 @@ static int vdbeRecordCompareInt( res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } @@ -3750,11 +3752,7 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; }