From 87cd93215e5f877005ae7a545f7a156286faa176 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Aug 2013 15:52:41 +0000 Subject: [PATCH] When estimating the number of rows scanned using data from the sqlite_stat4 table, avoid allocating UnpackedRecord and KeyInfo structures until they are definitely required. FossilOrigin-Name: 353950a5269fa439cc3e57b62e16558a84ea2557 --- manifest | 16 ++-- manifest.uuid | 2 +- src/sqliteInt.h | 5 +- src/vdbemem.c | 230 ++++++++++++++++++++++++++++++++++-------------- src/where.c | 27 ++---- 5 files changed, 183 insertions(+), 97 deletions(-) diff --git a/manifest b/manifest index 539c78a499..4bba71c21f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sbuilds\swithout\sSQLITE_ENABLE_STAT4. -D 2013-08-06T20:15:06.692 +C When\sestimating\sthe\snumber\sof\srows\sscanned\susing\sdata\sfrom\sthe\ssqlite_stat4\stable,\savoid\sallocating\sUnpackedRecord\sand\sKeyInfo\sstructures\suntil\sthey\sare\sdefinitely\srequired. +D 2013-08-07T15:52:41.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 31057687e0ebc9e56a025bcebb252bbd84e5db13 +F src/sqliteInt.h f56ec862ae37cf1b6e699ab5a2b53d40c8ec5068 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c 4389b3692969b4415fcfd00de36818a02f84df28 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 83f9b6e68a421cfcdde125a66b5ebe8220c12de2 +F src/vdbemem.c f0512045147702adec3ca6388663e243c17d2ea4 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 142525786b7855a0997e113f3932833bf5c1f63c +F src/where.c 4e188dc4a1f668d761750eb27e603616179806d0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed -R 3fc350a341f72f5a912cf9e1ccf56bc1 +P 84999e27cc0d14b89d9fe024e29d287c69285369 +R b05f226dd58169f35c18db270697f169 U dan -Z 97c03093fe0c6a197e3d6120177c182a +Z 7ab1efe6195a9307bc618357bb65cf2b diff --git a/manifest.uuid b/manifest.uuid index 3d5a8cb3bf..48fb3c47e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84999e27cc0d14b89d9fe024e29d287c69285369 \ No newline at end of file +353950a5269fa439cc3e57b62e16558a84ea2557 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 63c77085af..d3825a597c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1545,7 +1545,7 @@ struct Index { unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -3101,9 +3101,8 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); void sqlite3BackupRestart(sqlite3_backup *); void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); -int sqlite3Stat4ProbeSetValue(Parse*, UnpackedRecord*, Expr*, u8, int, int*); +int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); void sqlite3Stat4ProbeFree(UnpackedRecord*); -int sqlite3Stat4ProbeNew(Parse*, Index*, UnpackedRecord**); /* ** The interface to the LEMON-generated parser diff --git a/src/vdbemem.c b/src/vdbemem.c index 8627360113..9c7f579b1f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1005,27 +1005,30 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){ return p; } -static sqlite3_value *valueNew(sqlite3 *db, sqlite3_value *pOld){ - if( pOld ) return pOld; - return sqlite3ValueNew(db); +/* +** Argument pCtx is actually a pointer to a database handle. Allocate and +** return an sqlite3_value object associated with this database handle. +** +** This function used as the xAlloc callback for valueFromExpr() when +** it is called by sqlite3ValueFromExpr(). +*/ +static sqlite3_value *valueNew(void *pCtx){ + return sqlite3ValueNew((sqlite3*)pCtx); } /* -** Create a new sqlite3_value object, containing the value of pExpr. -** -** This only works for very simple expressions that consist of one constant -** token (i.e. "5", "5.1", "'a string'"). If the expression can -** be converted directly into a value, then the value is allocated and -** a pointer written to *ppVal. The caller is responsible for deallocating -** the value by passing it to sqlite3ValueFree() later on. If the expression -** cannot be converted to a value, then *ppVal is set to NULL. +** This function is the same as sqlite3ValueFromExpr(), except that instead +** of allocating any required sqlite3_value object by calling +** sqlite3ValueNew(), it does so by calling the supplied xAlloc hook. */ -int sqlite3ValueFromExpr( - sqlite3 *db, /* The database connection */ - Expr *pExpr, /* The expression to evaluate */ - u8 enc, /* Encoding to use */ - u8 affinity, /* Affinity to use */ - sqlite3_value **ppVal /* Write the new value here */ +int valueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + sqlite3_value *(*xAlloc)(void*), /* Used to allocate new sqlite3_value */ + void *pAlloc /* Argument passed to xAlloc */ ){ int op; char *zVal = 0; @@ -1061,7 +1064,7 @@ int sqlite3ValueFromExpr( } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); @@ -1095,7 +1098,7 @@ int sqlite3ValueFromExpr( sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( pVal==0 ) goto no_mem; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -1103,7 +1106,7 @@ int sqlite3ValueFromExpr( int nVal; assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite3Strlen30(zVal)-1; @@ -1127,51 +1130,178 @@ no_mem: return SQLITE_NOMEM; } +/* +** Create a new sqlite3_value object, containing the value of pExpr. +** +** This only works for very simple expressions that consist of one constant +** token (i.e. "5", "5.1", "'a string'"). If the expression can +** be converted directly into a value, then the value is allocated and +** a pointer written to *ppVal. The caller is responsible for deallocating +** the value by passing it to sqlite3ValueFree() later on. If the expression +** cannot be converted to a value, then *ppVal is set to NULL. +*/ +int sqlite3ValueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* Write the new value here */ +){ + return valueFromExpr(db, pExpr, enc, affinity, ppVal, valueNew, (void*)db); +} + #ifdef SQLITE_ENABLE_STAT4 +/* +** A pointer to an instance of this object is passed as the context +** pointer to valueNewStat4() (see below. +*/ +struct ValueNewStat4Ctx { + Parse *pParse; + Index *pIdx; + UnpackedRecord **ppRec; + int iVal; +}; + +/* +** This function is used as the xAlloc function with valueFromExpr() when +** it is called by sqlite3Stat4ProbeSetValue(). The argument points to +** an object of type ValueNewStat4Ctx (see above). +** +** If it has not already been allocated, this function allocates an +** UnpackedRecord structure and space for up to N values, where N is the +** number of columns in the index being probed. +*/ +static sqlite3_value *valueNewStat4(void *pCtx){ + struct ValueNewStat4Ctx *p = (struct ValueNewStat4Ctx*)pCtx; + UnpackedRecord *pRec = p->ppRec[0]; + + if( pRec==0 ){ + sqlite3 *db = p->pParse->db; /* Database handle */ + Index *pIdx = p->pIdx; /* Index being probed */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + + nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); + pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( pRec ){ + pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx); + if( pRec->pKeyInfo ){ + pRec->pKeyInfo->enc = ENC(db); + pRec->flags = UNPACKED_PREFIX_MATCH; + pRec->aMem = (Mem *)&pRec[1]; + for(i=0; inColumn; i++){ + pRec->aMem[i].flags = MEM_Null; + pRec->aMem[i].type = SQLITE_NULL; + pRec->aMem[i].db = db; + } + }else{ + sqlite3DbFree(db, pRec); + pRec = 0; + } + } + if( pRec==0 ) return 0; + p->ppRec[0] = pRec; + } + + pRec->nField = p->iVal+1; + return &pRec->aMem[p->iVal]; +} + +/* +** This function is used to allocate and populate UnpackedRecord +** structures intended to be compared against sample index keys stored +** in the sqlite_stat4 table. +** +** A single call to this function attempts to populates field iVal (leftmost +** is 0 etc.) of the unpacked record with a value extracted from expression +** pExpr. Extraction of values is possible if: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The sqlite3ValueFromExpr() function is able to extract a value +** from the expression (i.e. the expression is a literal value). +** +** If a value can be extracted, the affinity passed as the 5th argument +** is applied to it before it is copied into the UnpackedRecord. Output +** parameter *pbOk is set to true if a value is extracted, or false +** otherwise. +** +** When this function is called, *ppRec must either point to an object +** allocated by an earlier call to this function, or must be NULL. If it +** is NULL and a value can be successfully extracted, a new UnpackedRecord +** is allocated (and *ppRec set to point to it) before returning. +** +** Unless an error is encountered, SQLITE_OK is returned. It is not an +** error if a value cannot be extracted from pExpr. If an error does +** occur, an SQLite error code is returned. +*/ int sqlite3Stat4ProbeSetValue( Parse *pParse, /* Parse context */ - UnpackedRecord *pRec, /* Set field in this probe */ + Index *pIdx, /* Index being probed */ + UnpackedRecord **ppRec, /* IN/OUT: Probe record */ Expr *pExpr, /* The expression to extract a value from */ u8 affinity, /* Affinity to use */ int iVal, /* Array element to populate */ int *pbOk /* OUT: True if value was extracted */ ){ int rc = SQLITE_OK; - sqlite3_value *pVal = &pRec->aMem[iVal]; + sqlite3_value *pVal = 0; + + struct ValueNewStat4Ctx alloc; + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + alloc.iVal = iVal; #if 0 if( iVal>0 ){ *pbOk = 0; return SQLITE_OK; } #endif if( !pExpr ){ - sqlite3VdbeMemSetNull((Mem*)pVal); - *pbOk = 1; + pVal = valueNewStat4((void*)&alloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + *pbOk = 1; + } }else if( pExpr->op==TK_VARIABLE || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) ){ Vdbe *v; int iVar = pExpr->iColumn; sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); - if( v = pParse->pReprepare ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + if( (v = pParse->pReprepare) ){ + pVal = valueNewStat4((void*)&alloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } + pVal->db = pParse->db; + *pbOk = 1; + sqlite3VdbeMemStoreType((Mem*)pVal); } - pVal->db = pParse->db; - *pbOk = 1; - sqlite3VdbeMemStoreType((Mem*)pVal); }else{ *pbOk = 0; } }else{ - sqlite3 *db = pRec->aMem[0].db; - rc = sqlite3ValueFromExpr(db, pExpr, ENC(db), affinity, &pVal); + sqlite3 *db = pParse->db; + rc = valueFromExpr( + db, pExpr, ENC(db), affinity, &pVal, valueNewStat4, (void*)&alloc + ); *pbOk = (pVal!=0); } + assert( pVal==0 || pVal->db==pParse->db ); return rc; } +/* +** Unless it is NULL, the argument must be an UnpackedRecord object returned +** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes +** the object. +*/ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ if( pRec ){ int i; @@ -1184,40 +1314,6 @@ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ sqlite3DbFree(db, pRec); } } - -int sqlite3Stat4ProbeNew( - Parse *pParse, /* Parse context */ - Index *pIdx, /* Allocate record for this index */ - UnpackedRecord **ppRec /* OUT: Allocated record */ -){ - sqlite3 *db = pParse->db; /* Database handle */ - UnpackedRecord *pRec; /* Return value */ - int nByte; /* Bytes of space to allocate */ - int i; /* Counter variable */ - - assert( *ppRec==0 ); - if( pIdx->nSample==0 ) return SQLITE_OK; - - nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); - *ppRec = pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); - if( !pRec ) return SQLITE_NOMEM; - pRec->pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx); - if( !pRec->pKeyInfo ){ - sqlite3DbFree(db, pRec); - *ppRec = 0; - return SQLITE_NOMEM; - } - pRec->pKeyInfo->enc = ENC(pParse->db); - pRec->flags = UNPACKED_PREFIX_MATCH; - pRec->aMem = (Mem *)&pRec[1]; - for(i=0; inColumn; i++){ - pRec->aMem[i].flags = MEM_Null; - pRec->aMem[i].type = SQLITE_NULL; - pRec->aMem[i].db = db; - } - - return SQLITE_OK; -} #endif /* ifdef SQLITE_ENABLE_STAT4 */ /* diff --git a/src/where.c b/src/where.c index a8437d843c..2af52cbb30 100644 --- a/src/where.c +++ b/src/where.c @@ -2525,8 +2525,7 @@ static int whereRangeScanEst( int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); - pRec->nField = nEq+1; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk && whereKeyStats(pParse, p, pRec, 0, a)==SQLITE_OK ){ @@ -2538,8 +2537,7 @@ static int whereRangeScanEst( int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); - pRec->nField = nEq+1; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk && whereKeyStats(pParse, p, pRec, 1, a)==SQLITE_OK ){ @@ -2547,6 +2545,7 @@ static int whereRangeScanEst( if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } } + pBuilder->pRec = pRec; if( rc==SQLITE_OK ){ WhereCost iBase = whereCost(p->aiRowEst[0]); if( iUpper>iLower ){ @@ -2629,12 +2628,11 @@ static int whereEqualScanEst( } aff = p->pTable->aCol[p->aiColumn[0]].affinity; - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq-1, &bOk); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; if( bOk==0 ) return SQLITE_NOTFOUND; - pBuilder->nRecValid = nEq; - pRec->nField = nEq; rc = whereKeyStats(pParse, p, pRec, 0, a); if( rc==SQLITE_OK ){ @@ -4575,18 +4573,11 @@ static int whereLoopAddBtree( } } + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); #ifdef SQLITE_ENABLE_STAT4 - assert( pBuilder->pRec==0 ); - rc = sqlite3Stat4ProbeNew(pWInfo->pParse, pProbe, &pBuilder->pRec); - if( rc==SQLITE_OK ){ -#endif - rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#ifdef SQLITE_ENABLE_STAT4 - sqlite3Stat4ProbeFree(pBuilder->pRec); - pBuilder->nRecValid = 0; - pBuilder->pRec = 0; - } - assert( pBuilder->pRec==0 ); + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; #endif /* If there was an INDEXED BY clause, then only that one index is