mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Experimental change to "SELECT * FROM ... ORDER BY" processing to load some

Browse Source 
column values from the db after sorting.

FossilOrigin-Name: 9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d
This commit is contained in:
dan 2018-04-14 18:46:20 +00:00
parent 54f2cd905b
commit 24e25d32ce
6 changed files with 268 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
manifest
View File

@ -1,5 +1,5 @@
C Remove\san\salways-true\sbranch\sfrom\sthe\spreupdate\shook\slogic.
D 2018-04-13T16:23:22.787
C Experimental\schange\sto\s"SELECT\s*\sFROM\s...\sORDER\sBY"\sprocessing\sto\sload\ssome\ncolumn\svalues\sfrom\sthe\sdb\safter\ssorting.
D 2018-04-14T18:46:20.429
F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
F Makefile.in 7016fc56c6b9bfe5daac4f34be8be38d8c0b5fab79ccbfb764d3b23bf1c6fff3
@ -443,7 +443,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
F src/dbpage.c 8db4c97f630e7d83f884ea75caf1ffd0988c160e9d530194d93721c80821e0f6
F src/dbstat.c edabb82611143727511a45ca0859b8cd037851ebe756ae3db289859dd18b6f91
F src/delete.c 20c8788451dc737a967c87ea53ad43544d617f5b57d32ccce8bd52a0daf9e89b
F src/expr.c 6a41ceb27924dcfb6dc910a283ce74e136c9c305aba87a5acbfca32f5c49caa7
F src/expr.c 2448a255ce627c4e772bd68cf5529877c2bdfb6b580803d5fadc8528bdf7c1ef
F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
F src/fkey.c d617daf66b5515e2b42c1405b2b4984c30ca50fb705ab164271a9bf66c69e331
F src/func.c 94f42cba2cc1c34aeaa441022ba0170ec3fec4bba54db4e0ded085c6dc0fdc51
@ -491,12 +491,12 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
F src/resolve.c 66c73fcb7719b8ff0e841b58338f13604ff3e2b50a723f9b8f383595735262f6
F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
F src/select.c 41e857c107be5fb972b87a4cbb948c8b1191f339c1e32a65742581573932e119
F src/select.c c87425f0d56340a4ccc5aac52d07264b73c348ca45103caef9f289b09cb0e052
F src/shell.c.in cc960721e56ebc1a78773bb5d2f5608b54275f945cbe49e4afe919d6888062a7
F src/sqlite.h.in e0be726ea6e4e6571724d39d242472ecd8bd1ba6f84ade88e1641bde98a6d02b
F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
F src/sqliteInt.h a4837c57f9a3e2af100bc59f4be60d16b823f18131f8cef6a6685440f775eebd
F src/sqliteInt.h 31a7f214a6b52d413f144d303d22c6e9a940059d9fb86e6f49e5ec7ad65636e9
F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@ -1262,6 +1262,7 @@ F test/sort2.test cc23b7c19d684657559e8a55b02f7fcee03851d0
F test/sort3.test 1480ed7c4c157682542224e05e3b75faf4a149e5
F test/sort4.test 5c34d9623a4ae5921d956dfa2b70e77ed0fc6e5c
F test/sort5.test 6b43ae0e2169b5ceed441844492e55ba7f1ae0790528395ddf7888ab3094525d
F test/sorterref.test a13ed207a0eea3c7898f308f979bfb518f68c598ec737d2c494dfd3deaa83506
F test/sortfault.test d4ccf606a0c77498e2beb542764fd9394acb4d66
F test/speed1.test f2974a91d79f58507ada01864c0e323093065452
F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb
@ -1717,7 +1718,11 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P d7ec7b5970df9e542401f8299e098a5039d1875e2e3383dad06a0ec998589b6e
R f9cb5caefaf1e23db60ecde977cce853
U drh
Z 2e05817b48e1d8380863fbfa532bc935
P 0ab4518811b23bdb57feba55014cef07de66028f8fcbf8cf0831a712b2954b91
R 58c773e5cd03f7f4dcbed950a8e0accb
T *branch * sorter-reference
T *sym-sorter-reference *
T +closed caee77d9a8e1da2551cef44ba7a11e2de36eb8c047273148b4126bfc1edd21da
T -sym-trunk *
U dan
Z 5d1ae4ddc55c798d695f83fc2835288d

2
manifest.uuid
View File

@ -1 +1 @@
0ab4518811b23bdb57feba55014cef07de66028f8fcbf8cf0831a712b2954b91
9719cb46bbf501ce80f185159d594f593dd0b2639b9ef5a71a6c7b70046cb08d

7
src/expr.c
View File

@ -1363,6 +1363,7 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){
pItem->sortOrder = pOldItem->sortOrder;
pItem->done = 0;
pItem->bSpanIsTab = pOldItem->bSpanIsTab;
pItem->bSorterRef = pOldItem->bSorterRef;
pItem->u = pOldItem->u;
}
return pNew;
@ -4371,6 +4372,12 @@ int sqlite3ExprCodeExprList(
if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR;
for(pItem=pList->a, i=0; i<n; i++, pItem++){
Expr *pExpr = pItem->pExpr;
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
if( pItem->bSorterRef ){
i--;
n--;
}else
#endif
if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){
if( flags & SQLITE_ECEL_OMITREF ){
i--;

206
src/select.c
View File

@ -44,6 +44,20 @@ struct DistinctCtx {
/*
** An instance of the following object is used to record information about
** the ORDER BY (or GROUP BY) clause of query is being coded.
**
** The aDefer[] array is used by the sorter-references optimization. For
** example, assuming there is no index that can be used for the ORDER BY,
** for the query:
**
** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10;
**
** it may be more efficient to add just the "a" values to the sorter, and
** retrieve the associated "bigblob" values directly from table t1 as the
** 10 smallest "a" values are extracted from the sorter.
**
** When the sorter-reference optimization is used, there is one entry in the
** aDefer[] array for each database table that may be read as values are
** extracted from the sorter.
*/
typedef struct SortCtx SortCtx;
struct SortCtx {
@ -56,6 +70,14 @@ struct SortCtx {
int labelDone; /* Jump here when done, ex: LIMIT reached */
u8 sortFlags; /* Zero or more SORTFLAG_* bits */
u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
u8 nDefer; /* Number of valid entries in aDefer[] */
struct DeferredCsr {
Table *pTab; /* Table definition */
int iCsr; /* Cursor number for table */
int nKey; /* Number of PK columns for table pTab (>=1) */
} aDefer[4];
#endif
};
#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */
@ -678,6 +700,87 @@ static void codeDistinct(
sqlite3ReleaseTempReg(pParse, r1);
}
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
/*
** This function is called as part of inner-loop generation for a SELECT
** statement with an ORDER BY that is not optimized by an index. It
** determines the expressions, if any, that the sorter-reference
** optimization should be used for. The sorter-reference optimization
** is used for SELECT queries like:
**
** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10
**
** If the optimization is used for expression "bigblob", then instead of
** storing values read from that column in the sorter records, the PK of
** the row from table t1 is stored instead. Then, as records are extracted from
** the sorter to return to the user, the required value of bigblob is
** retrieved directly from table t1. If the values are very large, this
** can be more efficient than storing them directly in the sorter records.
**
** The ExprList_item.bSorterRef flag is set for each expression in pEList
** for which the sorter-reference optimization should be enabled.
** Additionally, the pSort->aDefer[] array is populated with entries
** for all cursors required to evaluate all selected expressions. Finally.
** output variable (*ppExtra) is set to an expression list containing
** expressions for all extra PK values that should be stored in the
** sorter records.
*/
static void selectExprDefer(
Parse *pParse, /* Leave any error here */
SortCtx *pSort, /* Sorter context */
ExprList *pEList, /* Expressions destined for sorter */
ExprList **ppExtra /* Expressions to append to sorter record */
){
int i;
int nDefer = 0;
ExprList *pExtra = 0;
for(i=0; i<pEList->nExpr; i++){
struct ExprList_item *pItem = &pEList->a[i];
if( pItem->u.x.iOrderByCol==0 ){
Expr *pExpr = pItem->pExpr;
Table *pTab = pExpr->pTab;
if( pExpr->op==TK_COLUMN && pTab && pTab->pSchema && pTab->pSelect==0
&& !IsVirtual(pTab)
){
int j;
for(j=0; j<nDefer; j++){
if( pSort->aDefer[j].iCsr==pExpr->iTable ) break;
}
if( j==nDefer ){
if( nDefer==ArraySize(pSort->aDefer) ){
continue;
}else{
int nKey = 1;
int k;
Index *pPk = 0;
if( !HasRowid(pTab) ){
pPk = sqlite3PrimaryKeyIndex(pTab);
nKey = pPk->nKeyCol;
}
for(k=0; k<nKey; k++){
Expr *pNew = sqlite3PExpr(pParse, TK_COLUMN, 0, 0);
if( pNew ){
pNew->iTable = pExpr->iTable;
pNew->pTab = pExpr->pTab;
pNew->iColumn = pPk ? pPk->aiColumn[k] : -1;
pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew);
}
}
pSort->aDefer[nDefer].pTab = pExpr->pTab;
pSort->aDefer[nDefer].iCsr = pExpr->iTable;
pSort->aDefer[nDefer].nKey = nKey;
nDefer++;
}
}
pItem->bSorterRef = 1;
}
}
}
pSort->nDefer = (u8)nDefer;
*ppExtra = pExtra;
}
#endif
/*
** This routine generates the code for the inside of the inner loop
** of a SELECT.
@ -750,6 +853,9 @@ static void selectInnerLoop(
VdbeComment((v, "%s", p->pEList->a[i].zName));
}
}else if( eDest!=SRT_Exists ){
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
ExprList *pExtra = 0;
#endif
/* If the destination is an EXISTS(...) expression, the actual
** values returned by the SELECT are not required.
*/
@ -773,12 +879,34 @@ static void selectInnerLoop(
p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat;
}
}
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
selectExprDefer(pParse, pSort, p->pEList, &pExtra);
if( pExtra ){
/* If there are any extra PK columns to add to the sorter records,
** allocate extra memory cells and adjust the OpenEphemeral
** instruction to account for the larger records. This is only
** required if there are one or more WITHOUT ROWID tables with
** composite primary keys in the SortCtx.aDefer[] array. */
VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex);
pOp->p2 += (pExtra->nExpr - pSort->nDefer);
pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer);
pParse->nMem += pExtra->nExpr;
}
#endif
regOrig = 0;
assert( eDest==SRT_Set || eDest==SRT_Mem
|| eDest==SRT_Coroutine || eDest==SRT_Output );
}
nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,
0,ecelFlags);
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
if( pExtra ){
nResultCol += sqlite3ExprCodeExprList(
pParse, pExtra, regResult + nResultCol, 0, 0
);
sqlite3ExprListDelete(pParse->db, pExtra);
}
#endif
}
/* If the DISTINCT keyword was present on the SELECT statement
@ -1236,7 +1364,7 @@ static void generateSortTail(
Vdbe *v = pParse->pVdbe; /* The prepared statement */
int addrBreak = pSort->labelDone; /* Jump here to exit loop */
int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */
int addr;
int addr; /* Top of output loop. Jump for Next. */
int addrOnce = 0;
int iTab;
ExprList *pOrderBy = pSort->pOrderBy;
@ -1245,10 +1373,11 @@ static void generateSortTail(
int regRow;
int regRowid;
int iCol;
int nKey;
int nKey; /* Number of key columns in sorter record */
int iSortTab; /* Sorter cursor to read from */
int i;
int bSeq; /* True if sorter record includes seq. no. */
int nRefKey = 0;
struct ExprList_item *aOutEx = p->pEList->a;
assert( addrBreak<0 );
@ -1257,6 +1386,17 @@ static void generateSortTail(
sqlite3VdbeGoto(v, addrBreak);
sqlite3VdbeResolveLabel(v, pSort->labelBkOut);
}
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
/* Open any cursors needed for sorter-reference expressions */
for(i=0; i<pSort->nDefer; i++){
Table *pTab = pSort->aDefer[i].pTab;
int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead);
nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey);
}
#endif
iTab = pSort->iECursor;
if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){
regRowid = 0;
@ -1272,7 +1412,8 @@ static void generateSortTail(
if( pSort->labelBkOut ){
addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
}
sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nColumn);
sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut,
nKey+1+nColumn+nRefKey);
if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce);
addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak);
VdbeCoverage(v);
@ -1286,17 +1427,59 @@ static void generateSortTail(
bSeq = 1;
}
for(i=0, iCol=nKey+bSeq-1; i<nColumn; i++){
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
if( aOutEx[i].bSorterRef ) continue;
#endif
if( aOutEx[i].u.x.iOrderByCol==0 ) iCol++;
}
for(i=nColumn-1; i>=0; i--){
int iRead;
if( aOutEx[i].u.x.iOrderByCol ){
iRead = aOutEx[i].u.x.iOrderByCol-1;
}else{
iRead = iCol--;
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
if( pSort->nDefer ){
int iKey = iCol+1;
int regKey = sqlite3GetTempRange(pParse, nRefKey);
for(i=0; i<pSort->nDefer; i++){
int iCsr = pSort->aDefer[i].iCsr;
Table *pTab = pSort->aDefer[i].pTab;
int nKey = pSort->aDefer[i].nKey;
sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
if( HasRowid(pTab) ){
sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey);
sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr,
sqlite3VdbeCurrentAddr(v)+1, regKey);
}else{
Index *pPk = sqlite3PrimaryKeyIndex(pTab);
int k;
int iJmp;
assert( pPk->nKeyCol==nKey );
for(k=0; k<nKey; k++){
sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey+k);
}
iJmp = sqlite3VdbeCurrentAddr(v);
sqlite3VdbeAddOp4Int(v, OP_SeekGE, iCsr, iJmp+2, regKey, nKey);
sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp+3, regKey, nKey);
sqlite3VdbeAddOp1(v, OP_NullRow, iCsr);
}
}
sqlite3ReleaseTempRange(pParse, regKey, nRefKey);
}
#endif
for(i=nColumn-1; i>=0; i--){
#ifdef SQLITE_ENABLE_SORTER_REFERENCES
if( aOutEx[i].bSorterRef ){
sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i);
}else
#endif
{
int iRead;
if( aOutEx[i].u.x.iOrderByCol ){
iRead = aOutEx[i].u.x.iOrderByCol-1;
}else{
iRead = iCol--;
}
sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
VdbeComment((v, "%s", aOutEx[i].zName?aOutEx[i].zName : aOutEx[i].zSpan));
}
sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i);
VdbeComment((v, "%s", aOutEx[i].zName ? aOutEx[i].zName : aOutEx[i].zSpan));
}
switch( eDest ){
case SRT_Table:
@ -6072,6 +6255,7 @@ int sqlite3Select(
if( sSort.pOrderBy ){
explainTempTable(pParse,
sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY");
assert( p->pEList==pEList );
generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest);
}

1
src/sqliteInt.h
View File

@ -2499,6 +2499,7 @@ struct ExprList {
unsigned done :1; /* A flag to indicate when processing is finished */
unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */
unsigned reusable :1; /* Constant expression is reusable */
unsigned bSorterRef :1; /* Defer evaluation until after sorting */
union {
struct {
u16 iOrderByCol; /* For ORDER BY, column number in result set */

50
test/sorterref.test Normal file
View File

@ -0,0 +1,50 @@
# 2018 April 14.
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix sorterref
do_execsql_test 1.0 {
CREATE TABLE t1(a, b, c);
INSERT INTO t1 VALUES(1, 2, 3);
INSERT INTO t1 VALUES(4, 5, 6);
ALTER TABLE t1 ADD COLUMN d DEFAULT 'string';
INSERT INTO t1 VALUES(7, 8, 9, 'text');
}
do_execsql_test 1.1 {
SELECT * FROM t1 ORDER BY b;
} {
1 2 3 string 4 5 6 string 7 8 9 text
}
do_execsql_test 2.0 {
DROP TABLE IF EXISTS t1;
CREATE TABLE t1(a, b);
CREATE TABLE t2(c, d, PRIMARY KEY(c)) WITHOUT ROWID;
INSERT INTO t1 VALUES(1, 2);
INSERT INTO t1 VALUES(2, 3);
INSERT INTO t1 VALUES(3, 4);
INSERT INTO t2 VALUES(1, 'one');
INSERT INTO t2 VALUES(3, 'three');
}
do_execsql_test 2.1 {
SELECT * FROM t1 LEFT JOIN t2 ON (a=c) ORDER BY b;
} {1 2 1 one 2 3 {} {} 3 4 3 three}
finish_test