Immprove the query planner such that it is able to make use of indexed

expressions within an aggregate query with GROUP BY.  This implements
enhancement request [99378177930f87bd].

FossilOrigin-Name: b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870
This commit is contained in:
drh 2022-11-25 16:10:48 +00:00
commit ae5cfda656
6 changed files with 493 additions and 141 deletions

View File

@ -1,5 +1,5 @@
C Add\srestriction\s(9)\sto\sthe\spush-down\soptimization:\s\sIf\sthe\ssubquery\sis\na\scompound\sthen\sall\sarms\sof\sthe\scompound\smust\shave\sthe\ssame\saffinity.\ndbsqlfuzz\s3a548de406a50e896c1bf7142692d35d339d697f.
D 2022-11-25T15:52:00.241
C Immprove\sthe\squery\splanner\ssuch\sthat\sit\sis\sable\sto\smake\suse\sof\sindexed\nexpressions\swithin\san\saggregate\squery\swith\sGROUP\sBY.\s\sThis\simplements\nenhancement\srequest\s[99378177930f87bd].
D 2022-11-25T16:10:48.068
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@ -591,7 +591,7 @@ F src/date.c 94ce83b4cd848a387680a5f920c9018c16655db778c4d36525af0a0f34679ac5
F src/dbpage.c f1a87f4ebcf22284e0aaf0697862f4ccfc120dcd6db3d8dfa3b049b2580c01d8
F src/dbstat.c 861e08690fcb0f2ee1165eff0060ea8d4f3e2ea10f80dab7d32ad70443a6ff2d
F src/delete.c 86573edae75e3d3e9a8b590d87db8e47222103029df4f3e11fa56044459b514e
F src/expr.c bc6527e3dff813c8102418e6e201870626a7fa5f69329ea7b082d602e7ed1cd9
F src/expr.c 9e7fadc664b938c18f006be0d4f6669888f9302756ee204420c7eccaed5435a6
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c 722f20779f5342a787922deded3628d8c74b5249cab04098cf17ee2f2aaff002
F src/func.c 7e86074afc4dc702691a29b7801f6dcc191db092b52e8bbe69dcd2f7be52194d
@ -641,12 +641,12 @@ F src/printf.c e99ee9741e79ae3873458146f59644276657340385ade4e76a5f5d1c25793764
F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
F src/resolve.c efea4e5fbecfd6d0a9071b0be0d952620991673391b6ffaaf4c277b0bb674633
F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
F src/select.c bafe6424e942aad558b2d4be8dbcf5e35ce427ce3cf66d7f0c0ac37c366b00c6
F src/select.c 7dcab394efe24c6016c6ed7958d426bfcdef57f50c2520dcccda27d9df28fdb6
F src/shell.c.in 09cb15d7421c475f2d308f6a4312d8d690916ea5cb62ea1618f2f4ce5703af35
F src/sqlite.h.in 100fc660c2f19961b8ed8437b9d53d687de2f8eb2b96437ec6da216adcb643ca
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h c4b9fa7a7e2bcdf850cfeb4b8a91d5ec47b7a00033bc996fd2ee96cbf2741f5f
F src/sqliteInt.h 6a24230f2928b3d1d9b0fdbedb98c862b828a4b1a9170306108e74cd6277f476
F src/sqliteInt.h 5dd5d3d47f40b6a12be4a5fc131673bfe00c00373ed266ff4c4ec05d1991e69f
F src/sqliteLimit.h d7323ffea5208c6af2734574bae933ca8ed2ab728083caa117c9738581a31657
F src/status.c 160c445d7d28c984a0eae38c144f6419311ed3eace59b44ac6dafc20db4af749
F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@ -1628,6 +1628,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356
F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed
F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5
F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223
F test/tkt-99378177930f87bd.test 0f932e85fa1d41f30532cb7be9718d82e491e953123b8c4c85cf025f36ffe34b
F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667
F test/tkt-9d68c883.test 16f7cb96781ba579bc2e19bb14b4ad609d9774b6
F test/tkt-9f2eb3abac.test cb6123ac695a08b4454c3792fbe85108f67fabf8
@ -2059,8 +2060,9 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P a2b6883ac2ef878f525ee847b170beb9f9ab9d1fa67557101be2cdae1e7f7a57
R 9c0acbc69bb427be679e111e4ec839ec
P 1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56
R eb048b7302ca70762f9994708c895609
T +closed e3e1b453dc47884ddc9e51c4302fa2f4d40bca3d69ac7c13d8e2ae1adc81ac56
U drh
Z ae1fb8eb841aaf444344209ddd2c7ea7
Z 6fdb43be076439dd6a7f7f9247393df9
# Remove this line to create a well-formed Fossil manifest.

View File

@ -1 +1 @@
1ad41840c5e0fa702ba2c0df77a3ea126bd695b910b5d1271fa3129c38c58f5f
b9190d3da70c41717eb188474fd225ee43d0b46646e1b03de5967bd332553870

View File

@ -53,7 +53,7 @@ char sqlite3ExprAffinity(const Expr *pExpr){
}
op = pExpr->op;
if( op==TK_REGISTER ) op = pExpr->op2;
if( op==TK_COLUMN || op==TK_AGG_COLUMN ){
if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){
assert( ExprUseYTab(pExpr) );
assert( pExpr->y.pTab!=0 );
return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn);
@ -173,7 +173,9 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){
while( p ){
int op = p->op;
if( op==TK_REGISTER ) op = p->op2;
if( op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_TRIGGER ){
if( (op==TK_AGG_COLUMN && p->y.pTab!=0)
|| op==TK_COLUMN || op==TK_TRIGGER
){
int j;
assert( ExprUseYTab(p) );
assert( p->y.pTab!=0 );
@ -4128,15 +4130,16 @@ expr_code_doover:
assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn );
pCol = &pAggInfo->aCol[pExpr->iAgg];
if( !pAggInfo->directMode ){
assert( pCol->iMem>0 );
return pCol->iMem;
return AggInfoColumnReg(pAggInfo, pExpr->iAgg);
}else if( pAggInfo->useSortingIdx ){
Table *pTab = pCol->pTab;
sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab,
pCol->iSorterColumn, target);
if( pCol->iColumn<0 ){
if( pTab==0 ){
/* No comment added */
}else if( pCol->iColumn<0 ){
VdbeComment((v,"%s.rowid",pTab->zName));
}else if( ALWAYS(pTab!=0) ){
}else{
VdbeComment((v,"%s.%s",
pTab->zName, pTab->aCol[pCol->iColumn].zCnName));
if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){
@ -4144,6 +4147,11 @@ expr_code_doover:
}
}
return target;
}else if( pExpr->y.pTab==0 ){
/* This case happens when the argument to an aggregate function
** is rewritten by aggregateConvertIndexedExprRefToColumn() */
sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target);
return target;
}
/* Otherwise, fall thru into the TK_COLUMN case */
/* no break */ deliberate_fall_through
@ -4441,7 +4449,7 @@ expr_code_doover:
assert( !ExprHasProperty(pExpr, EP_IntValue) );
sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr);
}else{
return pInfo->aFunc[pExpr->iAgg].iMem;
return AggInfoFuncReg(pInfo, pExpr->iAgg);
}
break;
}
@ -4730,7 +4738,7 @@ expr_code_doover:
if( pAggInfo ){
assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn );
if( !pAggInfo->directMode ){
inReg = pAggInfo->aCol[pExpr->iAgg].iMem;
inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg);
break;
}
if( pExpr->pAggInfo->useSortingIdx ){
@ -6243,58 +6251,41 @@ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){
}
/*
** This is the xExprCallback for a tree walker. It is used to
** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates
** for additional information.
** Search the AggInfo object for an aCol[] entry that has iTable and iColumn.
** Return the index in aCol[] of the entry that describes that column.
**
** If no prior entry is found, create a new one and return -1. The
** new column will have an idex of pAggInfo->nColumn-1.
*/
static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
int i;
NameContext *pNC = pWalker->u.pNC;
Parse *pParse = pNC->pParse;
SrcList *pSrcList = pNC->pSrcList;
AggInfo *pAggInfo = pNC->uNC.pAggInfo;
assert( pNC->ncFlags & NC_UAggInfo );
switch( pExpr->op ){
case TK_IF_NULL_ROW:
case TK_AGG_COLUMN:
case TK_COLUMN: {
testcase( pExpr->op==TK_AGG_COLUMN );
testcase( pExpr->op==TK_COLUMN );
testcase( pExpr->op==TK_IF_NULL_ROW );
/* Check to see if the column is in one of the tables in the FROM
** clause of the aggregate query */
if( ALWAYS(pSrcList!=0) ){
SrcItem *pItem = pSrcList->a;
for(i=0; i<pSrcList->nSrc; i++, pItem++){
static void findOrCreateAggInfoColumn(
Parse *pParse, /* Parsing context */
AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */
Expr *pExpr /* Expr describing the column to find or insert */
){
struct AggInfo_col *pCol;
assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
if( pExpr->iTable==pItem->iCursor ){
/* If we reach this point, it means that pExpr refers to a table
** that is in the FROM clause of the aggregate query.
**
** Make an entry for the column in pAggInfo->aCol[] if there
** is not an entry there already.
*/
int k;
assert( pAggInfo->iFirstReg==0 );
pCol = pAggInfo->aCol;
for(k=0; k<pAggInfo->nColumn; k++, pCol++){
if( pCol->iTable==pExpr->iTable
&& pCol->iColumn==pExpr->iColumn
&& pExpr->op!=TK_IF_NULL_ROW
){
break;
goto fix_up_expr;
}
}
if( (k>=pAggInfo->nColumn)
&& (k = addAggInfoColumn(pParse->db, pAggInfo))>=0
){
k = addAggInfoColumn(pParse->db, pAggInfo);
if( k<0 ){
/* OOM on resize */
assert( pParse->db->mallocFailed );
return;
}
pCol = &pAggInfo->aCol[k];
assert( ExprUseYTab(pExpr) );
pCol->pTab = pExpr->y.pTab;
pCol->iTable = pExpr->iTable;
pCol->iColumn = pExpr->iColumn;
pCol->iMem = ++pParse->nMem;
pCol->iSorterColumn = -1;
pCol->pCExpr = pExpr;
if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){
@ -6316,18 +6307,72 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
if( pCol->iSorterColumn<0 ){
pCol->iSorterColumn = pAggInfo->nSortingColumn++;
}
}
/* There is now an entry for pExpr in pAggInfo->aCol[] (either
** because it was there before or because we just created it).
** Convert the pExpr to be a TK_AGG_COLUMN referring to that
** pAggInfo->aCol[] entry.
*/
fix_up_expr:
ExprSetVVAProperty(pExpr, EP_NoReduce);
pExpr->pAggInfo = pAggInfo;
if( pExpr->op==TK_COLUMN ){
pExpr->op = TK_AGG_COLUMN;
}
pExpr->iAgg = (i16)k;
}
/*
** This is the xExprCallback for a tree walker. It is used to
** implement sqlite3ExprAnalyzeAggregates(). See sqlite3ExprAnalyzeAggregates
** for additional information.
*/
static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
int i;
NameContext *pNC = pWalker->u.pNC;
Parse *pParse = pNC->pParse;
SrcList *pSrcList = pNC->pSrcList;
AggInfo *pAggInfo = pNC->uNC.pAggInfo;
assert( pNC->ncFlags & NC_UAggInfo );
assert( pAggInfo->iFirstReg==0 );
switch( pExpr->op ){
default: {
IndexedExpr *pIEpr;
Expr tmp;
assert( pParse->iSelfTab==0 );
if( (pNC->ncFlags & NC_InAggFunc)==0 ) break;
if( pParse->pIdxEpr==0 ) break;
for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){
int iDataCur = pIEpr->iDataCur;
if( iDataCur<0 ) continue;
if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break;
}
if( pIEpr==0 ) break;
if( NEVER(!ExprUseYTab(pExpr)) ) break;
/* If we reach this point, it means that expression pExpr can be
** translated into a reference to an index column as described by
** pIEpr.
*/
memset(&tmp, 0, sizeof(tmp));
tmp.op = TK_AGG_COLUMN;
tmp.iTable = pIEpr->iIdxCur;
tmp.iColumn = pIEpr->iIdxCol;
findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp);
pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr;
pExpr->pAggInfo = pAggInfo;
pExpr->iAgg = tmp.iAgg;
return WRC_Prune;
}
case TK_IF_NULL_ROW:
case TK_AGG_COLUMN:
case TK_COLUMN: {
testcase( pExpr->op==TK_AGG_COLUMN );
testcase( pExpr->op==TK_COLUMN );
testcase( pExpr->op==TK_IF_NULL_ROW );
/* Check to see if the column is in one of the tables in the FROM
** clause of the aggregate query */
if( ALWAYS(pSrcList!=0) ){
SrcItem *pItem = pSrcList->a;
for(i=0; i<pSrcList->nSrc; i++, pItem++){
assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) );
if( pExpr->iTable==pItem->iCursor ){
findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr);
break;
} /* endif pExpr->iTable==pItem->iCursor */
} /* end loop over pSrcList */
@ -6357,7 +6402,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
assert( !ExprHasProperty(pExpr, EP_xIsSelect) );
pItem = &pAggInfo->aFunc[i];
pItem->pFExpr = pExpr;
pItem->iMem = ++pParse->nMem;
assert( ExprUseUToken(pExpr) );
pItem->pFunc = sqlite3FindFunction(pParse->db,
pExpr->u.zToken,

View File

@ -6229,6 +6229,172 @@ void sqlite3SelectPrep(
sqlite3SelectAddTypeInfo(pParse, p);
}
#if TREETRACE_ENABLED
/*
** Display all information about an AggInfo object
*/
static void printAggInfo(AggInfo *pAggInfo){
int ii;
for(ii=0; ii<pAggInfo->nColumn; ii++){
struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
sqlite3DebugPrintf(
"agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
" iSorterColumn=%d %s\n",
ii, pCol->pTab ? pCol->pTab->zName : "NULL",
pCol->iTable, pCol->iColumn, AggInfoColumnReg(pAggInfo,ii),
pCol->iSorterColumn,
ii>=pAggInfo->nAccumulator ? "" : " Accumulator");
sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
}
for(ii=0; ii<pAggInfo->nFunc; ii++){
sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
ii, AggInfoFuncReg(pAggInfo,ii));
sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
}
}
#endif /* TREETRACE_ENABLED */
/*
** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[]
** entries for columns that are arguments to aggregate functions but which
** are not otherwise used.
**
** The aCol[] entries in AggInfo prior to nAccumulator are columns that
** are referenced outside of aggregate functions. These might be columns
** that are part of the GROUP by clause, for example. Other database engines
** would throw an error if there is a column reference that is not in the
** GROUP BY clause and that is not part of an aggregate function argument.
** But SQLite allows this.
**
** The aCol[] entries beginning with the aCol[nAccumulator] and following
** are column references that are used exclusively as arguments to
** aggregate functions. This routine is responsible for computing
** (or recomputing) those aCol[] entries.
*/
static void analyzeAggFuncArgs(
Parse *pParse,
AggInfo *pAggInfo,
NameContext *pNC
){
int i;
assert( pAggInfo!=0 );
assert( pAggInfo->iFirstReg==0 );
pNC->ncFlags |= NC_InAggFunc;
for(i=0; i<pAggInfo->nFunc; i++){
Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
assert( ExprUseXList(pExpr) );
sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList);
#ifndef SQLITE_OMIT_WINDOWFUNC
assert( !IsWindowFunc(pExpr) );
if( ExprHasProperty(pExpr, EP_WinFunc) ){
sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter);
}
#endif
}
pNC->ncFlags &= ~NC_InAggFunc;
}
/*
** An index on expressions is being used in the inner loop of an
** aggregate query with a GROUP BY clause. This routine attempts
** to adjust the AggInfo object to take advantage of index and to
** perhaps use the index as a covering index.
**
*/
static void optimizeAggregateUseOfIndexedExpr(
Parse *pParse, /* Parsing context */
Select *pSelect, /* The SELECT statement being processed */
AggInfo *pAggInfo, /* The aggregate info */
NameContext *pNC /* Name context used to resolve agg-func args */
){
assert( pAggInfo->iFirstReg==0 );
pAggInfo->nColumn = pAggInfo->nAccumulator;
if( ALWAYS(pAggInfo->nSortingColumn>0) ){
if( pAggInfo->nColumn==0 ){
pAggInfo->nSortingColumn = 0;
}else{
pAggInfo->nSortingColumn =
pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1;
}
}
analyzeAggFuncArgs(pParse, pAggInfo, pNC);
#if TREETRACE_ENABLED
if( sqlite3TreeTrace & 0x20 ){
IndexedExpr *pIEpr;
TREETRACE(0x20, pParse, pSelect,
("AggInfo (possibly) adjusted for Indexed Exprs\n"));
sqlite3TreeViewSelect(0, pSelect, 0);
for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){
printf("data-cursor=%d index={%d,%d}\n",
pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol);
sqlite3TreeViewExpr(0, pIEpr->pExpr, 0);
}
printAggInfo(pAggInfo);
}
#else
(void)pSelect; /* Suppress unused-parameter warnings */
#endif
}
/*
** Walker callback for aggregateConvertIndexedExprRefToColumn().
*/
static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){
AggInfo *pAggInfo;
struct AggInfo_col *pCol;
if( pExpr->pAggInfo==0 ) return WRC_Continue;
if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue;
if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue;
if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue;
pAggInfo = pExpr->pAggInfo;
assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn );
pCol = &pAggInfo->aCol[pExpr->iAgg];
pExpr->op = TK_AGG_COLUMN;
pExpr->iTable = pCol->iTable;
pExpr->iColumn = pCol->iColumn;
return WRC_Prune;
}
/*
** Convert every pAggInfo->aFunc[].pExpr such that any node within
** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN
** opcode.
*/
static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){
int i;
Walker w;
memset(&w, 0, sizeof(w));
w.xExprCallback = aggregateIdxEprRefToColCallback;
for(i=0; i<pAggInfo->nFunc; i++){
sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr);
}
}
/*
** Allocate a block of registers so that there is one register for each
** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first
** register in this block is stored in pAggInfo->iFirstReg.
**
** This routine may only be called once for each AggInfo object. Prior
** to calling this routine:
**
** * The aCol[] and aFunc[] arrays may be modified
** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used
**
** After clling this routine:
**
** * The aCol[] and aFunc[] arrays are fixed
** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used
**
*/
static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){
assert( pAggInfo!=0 );
assert( pAggInfo->iFirstReg==0 );
pAggInfo->iFirstReg = pParse->nMem + 1;
pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc;
}
/*
** Reset the aggregate accumulator.
**
@ -6242,24 +6408,13 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){
int i;
struct AggInfo_func *pFunc;
int nReg = pAggInfo->nFunc + pAggInfo->nColumn;
assert( pAggInfo->iFirstReg>0 );
assert( pParse->db->pParse==pParse );
assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 );
if( nReg==0 ) return;
if( pParse->nErr ) return;
#ifdef SQLITE_DEBUG
/* Verify that all AggInfo registers are within the range specified by
** AggInfo.mnReg..AggInfo.mxReg */
assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 );
for(i=0; i<pAggInfo->nColumn; i++){
assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg
&& pAggInfo->aCol[i].iMem<=pAggInfo->mxReg );
}
for(i=0; i<pAggInfo->nFunc; i++){
assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg
&& pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg );
}
#endif
sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg);
sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg,
pAggInfo->iFirstReg+nReg-1);
for(pFunc=pAggInfo->aFunc, i=0; i<pAggInfo->nFunc; i++, pFunc++){
if( pFunc->iDistinct>=0 ){
Expr *pE = pFunc->pFExpr;
@ -6291,15 +6446,16 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
ExprList *pList;
assert( ExprUseXList(pF->pFExpr) );
pList = pF->pFExpr->x.pList;
sqlite3VdbeAddOp2(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0);
sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i),
pList ? pList->nExpr : 0);
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
}
}
/*
** Update the accumulator memory cells for an aggregate based on
** the current cursor position.
** Generate code that will update the accumulator memory cells for an
** aggregate based on the current cursor position.
**
** If regAcc is non-zero and there are no min() or max() aggregates
** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator
@ -6319,6 +6475,8 @@ static void updateAccumulator(
struct AggInfo_func *pF;
struct AggInfo_col *pC;
assert( pAggInfo->iFirstReg>0 );
if( pParse->nErr ) return;
pAggInfo->directMode = 1;
for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
int nArg;
@ -6379,7 +6537,7 @@ static void updateAccumulator(
if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem;
sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ);
}
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem);
sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i));
sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF);
sqlite3VdbeChangeP5(v, (u8)nArg);
sqlite3ReleaseTempRange(pParse, regAgg, nArg);
@ -6394,7 +6552,7 @@ static void updateAccumulator(
addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v);
}
for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){
sqlite3ExprCode(pParse, pC->pCExpr, pC->iMem);
sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i));
}
pAggInfo->directMode = 0;
@ -6654,30 +6812,6 @@ static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){
return 0;
}
#if TREETRACE_ENABLED
/*
** Display all information about an AggInfo object
*/
static void printAggInfo(AggInfo *pAggInfo){
int ii;
for(ii=0; ii<pAggInfo->nColumn; ii++){
struct AggInfo_col *pCol = &pAggInfo->aCol[ii];
sqlite3DebugPrintf(
"agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d"
" iSorterColumn=%d\n",
ii, pCol->pTab ? pCol->pTab->zName : "NULL",
pCol->iTable, pCol->iColumn, pCol->iMem,
pCol->iSorterColumn);
sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0);
}
for(ii=0; ii<pAggInfo->nFunc; ii++){
sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n",
ii, pAggInfo->aFunc[ii].iMem);
sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0);
}
}
#endif /* TREETRACE_ENABLED */
/*
** Generate code for the SELECT statement given in the p argument.
**
@ -7439,7 +7573,6 @@ int sqlite3Select(
sNC.pSrcList = pTabList;
sNC.uNC.pAggInfo = pAggInfo;
VVA_ONLY( sNC.ncFlags = NC_UAggInfo; )
pAggInfo->mnReg = pParse->nMem+1;
pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0;
pAggInfo->pGroupBy = pGroupBy;
sqlite3ExprAnalyzeAggList(&sNC, pEList);
@ -7460,20 +7593,7 @@ int sqlite3Select(
}else{
minMaxFlag = WHERE_ORDERBY_NORMAL;
}
for(i=0; i<pAggInfo->nFunc; i++){
Expr *pExpr = pAggInfo->aFunc[i].pFExpr;
assert( ExprUseXList(pExpr) );
sNC.ncFlags |= NC_InAggFunc;
sqlite3ExprAnalyzeAggList(&sNC, pExpr->x.pList);
#ifndef SQLITE_OMIT_WINDOWFUNC
assert( !IsWindowFunc(pExpr) );
if( ExprHasProperty(pExpr, EP_WinFunc) ){
sqlite3ExprAnalyzeAggregates(&sNC, pExpr->y.pWin->pFilter);
}
#endif
sNC.ncFlags &= ~NC_InAggFunc;
}
pAggInfo->mxReg = pParse->nMem;
analyzeAggFuncArgs(pParse, pAggInfo, &sNC);
if( db->mallocFailed ) goto select_end;
#if TREETRACE_ENABLED
if( sqlite3TreeTrace & 0x20 ){
@ -7561,6 +7681,10 @@ int sqlite3Select(
sqlite3ExprListDelete(db, pDistinct);
goto select_end;
}
if( pParse->pIdxEpr ){
optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC);
}
assignAggregateRegisters(pParse, pAggInfo);
eDist = sqlite3WhereIsDistinct(pWInfo);
TREETRACE(0x2,pParse,p,("WhereBegin returns\n"));
if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){
@ -7621,6 +7745,23 @@ int sqlite3Select(
pAggInfo->useSortingIdx = 1;
}
/* If there entries in pAgggInfo->aFunc[] that contain subexpressions
** that are indexed (and that were previously identified and tagged
** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions
** must now be converted into a TK_AGG_COLUMN node so that the value
** is correctly pulled from the index rather than being recomputed. */
if( pParse->pIdxEpr ){
aggregateConvertIndexedExprRefToColumn(pAggInfo);
#if TREETRACE_ENABLED
if( sqlite3TreeTrace & 0x20 ){
TREETRACE(0x20, pParse, p,
("AggInfo function expressions converted to reference index\n"));
sqlite3TreeViewSelect(0, p, 0);
printAggInfo(pAggInfo);
}
#endif
}
/* If the index or temporary table used by the GROUP BY sort
** will naturally deliver rows in the order required by the ORDER BY
** clause, cancel the ephemeral table open coded earlier.
@ -7799,7 +7940,8 @@ int sqlite3Select(
if( pKeyInfo ){
sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO);
}
sqlite3VdbeAddOp2(v, OP_Count, iCsr, pAggInfo->aFunc[0].iMem);
assignAggregateRegisters(pParse, pAggInfo);
sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0));
sqlite3VdbeAddOp1(v, OP_Close, iCsr);
explainSimpleCount(pParse, pTab, pBest);
}else{
@ -7835,6 +7977,7 @@ int sqlite3Select(
pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList;
distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0;
}
assignAggregateRegisters(pParse, pAggInfo);
/* This case runs if the aggregate has no GROUP BY clause. The
** processing is much simpler since there is only a single row
@ -7918,7 +8061,7 @@ select_end:
if( pAggInfo && !db->mallocFailed ){
for(i=0; i<pAggInfo->nColumn; i++){
Expr *pExpr = pAggInfo->aCol[i].pCExpr;
assert( pExpr!=0 );
if( pExpr==0 ) continue;
assert( pExpr->pAggInfo==pAggInfo );
assert( pExpr->iAgg==i );
}

View File

@ -2739,16 +2739,15 @@ struct AggInfo {
** from source tables rather than from accumulators */
u8 useSortingIdx; /* In direct mode, reference the sorting index rather
** than the source table */
u16 nSortingColumn; /* Number of columns in the sorting index */
int sortingIdx; /* Cursor number of the sorting index */
int sortingIdxPTab; /* Cursor number of pseudo-table */
int nSortingColumn; /* Number of columns in the sorting index */
int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */
int iFirstReg; /* First register in range for aCol[] and aFunc[] */
ExprList *pGroupBy; /* The group by clause */
struct AggInfo_col { /* For each column used in source tables */
Table *pTab; /* Source table */
Expr *pCExpr; /* The original expression */
int iTable; /* Cursor number of the source table */
int iMem; /* Memory location that acts as accumulator */
i16 iColumn; /* Column number within the source table */
i16 iSorterColumn; /* Column number in the sorting index */
} *aCol;
@ -2759,7 +2758,6 @@ struct AggInfo {
struct AggInfo_func { /* For each aggregate function */
Expr *pFExpr; /* Expression encoding the function */
FuncDef *pFunc; /* The aggregate function implementation */
int iMem; /* Memory location that acts as accumulator */
int iDistinct; /* Ephemeral table used to enforce DISTINCT */
int iDistAddr; /* Address of OP_OpenEphemeral */
} *aFunc;
@ -2767,6 +2765,17 @@ struct AggInfo {
u32 selId; /* Select to which this AggInfo belongs */
};
/*
** Macros to compute aCol[] and aFunc[] register numbers.
**
** These macros should not be used prior to the call to
** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg.
** The assert()s that are part of this macro verify that constraint.
*/
#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I))
#define AggInfoFuncReg(A,I) \
(assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I))
/*
** The datatype ynVar is a signed integer, either 16-bit or 32-bit.
** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater

View File

@ -0,0 +1,154 @@
# 2022-11-23
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
# This file implements regression tests for SQLite library.
#
# This file implements tests to verify that the enhancement
# request documented by ticket 99378177930f87bd is working.
#
# The enhancement is that if an aggregate query with a GROUP BY clause
# uses subexpressions in the arguments to aggregate functions that are
# also columns of an index, then the values are pulled from the index
# rather than being recomputed. This has the potential to make some
# indexed queries works as if the index were covering.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
do_execsql_test tkt-99378-100 {
CREATE TABLE t1(a INT, b TEXT, c INT, d INT);
INSERT INTO t1(a,b,c,d) VALUES
(1, '{"x":1}', 12, 3),
(1, '{"x":2}', 4, 5),
(1, '{"x":1}', 6, 11),
(2, '{"x":1}', 22, 3),
(2, '{"x":2}', 4, 5),
(3, '{"x":1}', 6, 7);
CREATE INDEX t1x ON t1(d, a, b->>'x', c);
} {}
do_execsql_test tkt-99378-110 {
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {
1 2 1 16 12
2 2 1 26 22
3 1 1 6 6
}
# The proof that the index on the expression is being used is in the
# fact that the byte code contains no "Function" opcodes. In other words,
# the ->> operator (which is implemented by a function) is never invoked.
# Instead, the b->>'x' value is pulled out of the index.
#
do_execsql_test tkt-99378-120 {
EXPLAIN
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {~/Function/}
do_execsql_test tkt-99378-130 {
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY +a;
} {
1 2 1 16 12
2 2 1 26 22
3 1 1 6 6
}
do_execsql_test tkt-99378-140 {
EXPLAIN
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY +a;
} {~/Function/}
do_execsql_test tkt-99378-200 {
DROP INDEX t1x;
CREATE INDEX t1x ON t1(a, d, b->>'x', c);
}
do_execsql_test tkt-99378-210 {
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {
1 2 1 16 12
2 2 1 26 22
3 1 1 6 6
}
do_execsql_test tkt-99378-220 {
EXPLAIN
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {~/Function/}
do_execsql_test tkt-99378-230 {
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {
1 2 1 16 12
2 2 1 26 22
3 1 1 6 6
}
do_execsql_test tkt-99378-240 {
EXPLAIN
SELECT a,
SUM(1) AS t1,
SUM(CASE WHEN b->>'x'=1 THEN 1 END) AS t2,
SUM(c) AS t3,
SUM(CASE WHEN b->>'x'=1 THEN c END) AS t4
FROM t1
WHERE d BETWEEN 0 and 10
GROUP BY a;
} {~/Function/}
finish_test