mirror of https://github.com/sqlite/sqlite
Modify the code in vdbesort.c so that most reads and writes to temporary files are aligned page-sized blocks.
FossilOrigin-Name: 55e47ef338c42f95f0f071d6ec92cd2480f9f1fe
This commit is contained in:
parent
5373b76bdc
commit
3b2c9b3250
17
manifest
17
manifest
|
@ -1,5 +1,5 @@
|
|||
C Ensure\sthat\sthere\sis\salways\sat\sleast\sone\saReadMark\sslot\susable\sby\san\sunprivileged\sreader\swhile\sa\scheckpoint\sis\srunning.\sAlso,\sif\sone\sor\smore\stransactions\sare\srecovered\sfrom\sa\slog\sfile,\sinitialize\sone\sof\sthe\saReadMark\sslots\sto\scontain\smxFrame\sas\spart\sof\sthe\srecovery\sprocess.
|
||||
D 2012-07-17T14:37:12.494
|
||||
C Modify\sthe\scode\sin\svdbesort.c\sso\sthat\smost\sreads\sand\swrites\sto\stemporary\sfiles\sare\saligned\spage-sized\sblocks.
|
||||
D 2012-07-23T19:25:39.921
|
||||
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
|
||||
F Makefile.in 8f6d858bf3df9978ba43df19985146a1173025e4
|
||||
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
|
||||
|
@ -244,7 +244,7 @@ F src/vdbeapi.c 88ea823bbcb4320f5a6607f39cd7c2d3cc4c26b1
|
|||
F src/vdbeaux.c dce80038c3c41f2680e5ab4dd0f7e0d8b7ff9071
|
||||
F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb
|
||||
F src/vdbemem.c cb55e84b8e2c15704968ee05f0fae25883299b74
|
||||
F src/vdbesort.c b25814d385895544ebc8118245c8311ded7f81c9
|
||||
F src/vdbesort.c 6822221af97e57bd17091cfe5abec902c1ef04ad
|
||||
F src/vdbetrace.c 79059ebd17b3c8545fab2a24253713e77e4ab392
|
||||
F src/vtab.c bb8ea3a26608bb1357538a5d2fc72beba6638998
|
||||
F src/wal.c 9294df6f96aae5909ae1a9b733fd1e1b4736978b
|
||||
|
@ -530,7 +530,7 @@ F test/incrvacuum_ioerr.test 22f208d01c528403240e05beecc41dc98ed01637
|
|||
F test/index.test b5429732b3b983fa810e3ac867d7ca85dae35097
|
||||
F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6
|
||||
F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7
|
||||
F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026
|
||||
F test/index4.test 1e299862024012e0165531cce251572f7f084d15
|
||||
F test/indexedby.test be501e381b82b2f8ab406309ba7aac46e221f4ad
|
||||
F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d
|
||||
F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
|
||||
|
@ -1005,7 +1005,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
|
|||
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
|
||||
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
|
||||
F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
|
||||
P 8c9ee1d78f99394eef73a177141ca9e1c67e4e07
|
||||
R 21a0c6942de3e9593e3e93f462c443c7
|
||||
P e4163596339c2166f9c4356ab824fff8bda8d0b0
|
||||
R ed9539f5559c8070d5186f316e47e4b4
|
||||
T *branch * sorter-coalesce-writes
|
||||
T *sym-sorter-coalesce-writes *
|
||||
T -sym-trunk *
|
||||
U dan
|
||||
Z a98d8c358c03bd601d60af308961371e
|
||||
Z 0e4fc9d4c65d0946db257b091fe0d573
|
||||
|
|
|
@ -1 +1 @@
|
|||
e4163596339c2166f9c4356ab824fff8bda8d0b0
|
||||
55e47ef338c42f95f0f071d6ec92cd2480f9f1fe
|
385
src/vdbesort.c
385
src/vdbesort.c
|
@ -22,6 +22,7 @@
|
|||
|
||||
typedef struct VdbeSorterIter VdbeSorterIter;
|
||||
typedef struct SorterRecord SorterRecord;
|
||||
typedef struct FileWriter FileWriter;
|
||||
|
||||
/*
|
||||
** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES:
|
||||
|
@ -119,6 +120,22 @@ struct VdbeSorterIter {
|
|||
sqlite3_file *pFile; /* File iterator is reading from */
|
||||
u8 *aAlloc; /* Allocated space */
|
||||
u8 *aKey; /* Pointer to current key */
|
||||
u8 *aBuffer; /* Current read buffer */
|
||||
int nBuffer; /* Size of read buffer in bytes */
|
||||
};
|
||||
|
||||
/*
|
||||
** An instance of this structure is used to separate the stream of records
|
||||
** being written to files by the merge-sort code into aligned, page-sized
|
||||
** blocks.
|
||||
*/
|
||||
struct FileWriter {
|
||||
u8 *aBuffer; /* Pointer to write buffer */
|
||||
int nBuffer; /* Size of write buffer in bytes */
|
||||
int iBufStart; /* First byte of buffer to write */
|
||||
int iBufEnd; /* Last byte of buffer to write */
|
||||
i64 iWriteOff; /* Offset of start of buffer in file */
|
||||
sqlite3_file *pFile; /* File to write to */
|
||||
};
|
||||
|
||||
/*
|
||||
|
@ -144,9 +161,107 @@ struct SorterRecord {
|
|||
*/
|
||||
static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){
|
||||
sqlite3DbFree(db, pIter->aAlloc);
|
||||
sqlite3DbFree(db, pIter->aBuffer);
|
||||
memset(pIter, 0, sizeof(VdbeSorterIter));
|
||||
}
|
||||
|
||||
/*
|
||||
** Read nByte bytes of data from the stream of data iterated by object p.
|
||||
** If successful, set *ppOut to point to a buffer containing the data
|
||||
** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite
|
||||
** error code.
|
||||
**
|
||||
** The buffer indicated by *ppOut may only be considered valid until the
|
||||
** next call to this function.
|
||||
*/
|
||||
static int vdbeSorterIterRead(
|
||||
sqlite3 *db, /* Database handle (for malloc) */
|
||||
VdbeSorterIter *p, /* Iterator */
|
||||
int nByte, /* Bytes of data to read */
|
||||
u8 **ppOut /* OUT: Pointer to buffer containing data */
|
||||
){
|
||||
int iBuf;
|
||||
int nAvail;
|
||||
assert( p->aBuffer );
|
||||
|
||||
iBuf = p->iReadOff % p->nBuffer;
|
||||
if( iBuf==0 ){
|
||||
int nRead;
|
||||
int rc;
|
||||
|
||||
nRead = p->iEof - p->iReadOff;
|
||||
if( nRead>p->nBuffer ) nRead = p->nBuffer;
|
||||
assert( nRead>0 );
|
||||
rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff);
|
||||
assert( rc!=SQLITE_IOERR_SHORT_READ );
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
}
|
||||
nAvail = p->nBuffer - iBuf;
|
||||
|
||||
if( nByte<=nAvail ){
|
||||
*ppOut = &p->aBuffer[iBuf];
|
||||
p->iReadOff += nByte;
|
||||
}else{
|
||||
int nRem;
|
||||
if( p->nAlloc<nByte ){
|
||||
int nNew = p->nAlloc*2;
|
||||
while( nByte>nNew ) nNew = nNew*2;
|
||||
|
||||
p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew);
|
||||
if( !p->aAlloc ) return SQLITE_NOMEM;
|
||||
}
|
||||
|
||||
memcpy(p->aAlloc, &p->aBuffer[iBuf], nAvail);
|
||||
p->iReadOff += nAvail;
|
||||
nRem = nByte - nAvail;
|
||||
while( nRem>0 ){
|
||||
int rc;
|
||||
int nCopy;
|
||||
u8 *aNext;
|
||||
|
||||
nCopy = nRem;
|
||||
if( nRem>p->nBuffer ) nCopy = p->nBuffer;
|
||||
rc = vdbeSorterIterRead(db, p, nCopy, &aNext);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
assert( aNext!=p->aAlloc );
|
||||
|
||||
memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy);
|
||||
nRem -= nCopy;
|
||||
}
|
||||
|
||||
*ppOut = p->aAlloc;
|
||||
}
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/*
|
||||
** Read a varint from the stream of data accessed by p. Set *pnOut to
|
||||
** the value read.
|
||||
*/
|
||||
static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){
|
||||
int iBuf;
|
||||
|
||||
iBuf = p->iReadOff % p->nBuffer;
|
||||
if( iBuf && (p->nBuffer-iBuf)>=9 ){
|
||||
p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut);
|
||||
}else{
|
||||
u8 aVarint[9];
|
||||
int i;
|
||||
for(i=0; i<sizeof(aVarint); i++){
|
||||
u8 *a;
|
||||
int rc = vdbeSorterIterRead(db, p, 1, &a);
|
||||
if( rc ) return rc;
|
||||
aVarint[i] = *a;
|
||||
if( (aVarint[i] & 0x80)==0 ) break;
|
||||
}
|
||||
sqlite3GetVarint(aVarint, pnOut);
|
||||
}
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if
|
||||
** no error occurs, or an SQLite error code if one does.
|
||||
|
@ -156,96 +271,18 @@ static int vdbeSorterIterNext(
|
|||
VdbeSorterIter *pIter /* Iterator to advance */
|
||||
){
|
||||
int rc; /* Return Code */
|
||||
int nRead; /* Number of bytes read */
|
||||
int nRec = 0; /* Size of record in bytes */
|
||||
int iOff = 0; /* Size of serialized size varint in bytes */
|
||||
u64 nRec = 0; /* Size of record in bytes */
|
||||
|
||||
assert( pIter->iEof>=pIter->iReadOff );
|
||||
if( pIter->iEof-pIter->iReadOff>5 ){
|
||||
nRead = 5;
|
||||
}else{
|
||||
nRead = (int)(pIter->iEof - pIter->iReadOff);
|
||||
}
|
||||
if( nRead<=0 ){
|
||||
if( pIter->iReadOff>=pIter->iEof ){
|
||||
/* This is an EOF condition */
|
||||
vdbeSorterIterZero(db, pIter);
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
rc = sqlite3OsRead(pIter->pFile, pIter->aAlloc, nRead, pIter->iReadOff);
|
||||
rc = vdbeSorterIterVarint(db, pIter, &nRec);
|
||||
if( rc==SQLITE_OK ){
|
||||
iOff = getVarint32(pIter->aAlloc, nRec);
|
||||
if( (iOff+nRec)>nRead ){
|
||||
int nRead2; /* Number of extra bytes to read */
|
||||
if( (iOff+nRec)>pIter->nAlloc ){
|
||||
int nNew = pIter->nAlloc*2;
|
||||
while( (iOff+nRec)>nNew ) nNew = nNew*2;
|
||||
pIter->aAlloc = sqlite3DbReallocOrFree(db, pIter->aAlloc, nNew);
|
||||
if( !pIter->aAlloc ) return SQLITE_NOMEM;
|
||||
pIter->nAlloc = nNew;
|
||||
}
|
||||
|
||||
nRead2 = iOff + nRec - nRead;
|
||||
rc = sqlite3OsRead(
|
||||
pIter->pFile, &pIter->aAlloc[nRead], nRead2, pIter->iReadOff+nRead
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
assert( rc!=SQLITE_OK || nRec>0 );
|
||||
pIter->iReadOff += iOff+nRec;
|
||||
pIter->nKey = nRec;
|
||||
pIter->aKey = &pIter->aAlloc[iOff];
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Write a single varint, value iVal, to file-descriptor pFile. Return
|
||||
** SQLITE_OK if successful, or an SQLite error code if some error occurs.
|
||||
**
|
||||
** The value of *piOffset when this function is called is used as the byte
|
||||
** offset in file pFile to write to. Before returning, *piOffset is
|
||||
** incremented by the number of bytes written.
|
||||
*/
|
||||
static int vdbeSorterWriteVarint(
|
||||
sqlite3_file *pFile, /* File to write to */
|
||||
i64 iVal, /* Value to write as a varint */
|
||||
i64 *piOffset /* IN/OUT: Write offset in file pFile */
|
||||
){
|
||||
u8 aVarint[9]; /* Buffer large enough for a varint */
|
||||
int nVarint; /* Number of used bytes in varint */
|
||||
int rc; /* Result of write() call */
|
||||
|
||||
nVarint = sqlite3PutVarint(aVarint, iVal);
|
||||
rc = sqlite3OsWrite(pFile, aVarint, nVarint, *piOffset);
|
||||
*piOffset += nVarint;
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Read a single varint from file-descriptor pFile. Return SQLITE_OK if
|
||||
** successful, or an SQLite error code if some error occurs.
|
||||
**
|
||||
** The value of *piOffset when this function is called is used as the
|
||||
** byte offset in file pFile from whence to read the varint. If successful
|
||||
** (i.e. if no IO error occurs), then *piOffset is set to the offset of
|
||||
** the first byte past the end of the varint before returning. *piVal is
|
||||
** set to the integer value read. If an error occurs, the final values of
|
||||
** both *piOffset and *piVal are undefined.
|
||||
*/
|
||||
static int vdbeSorterReadVarint(
|
||||
sqlite3_file *pFile, /* File to read from */
|
||||
i64 *piOffset, /* IN/OUT: Read offset in pFile */
|
||||
i64 *piVal /* OUT: Value read from file */
|
||||
){
|
||||
u8 aVarint[9]; /* Buffer large enough for a varint */
|
||||
i64 iOff = *piOffset; /* Offset in file to read from */
|
||||
int rc; /* Return code */
|
||||
|
||||
rc = sqlite3OsRead(pFile, aVarint, 9, iOff);
|
||||
if( rc==SQLITE_OK ){
|
||||
*piOffset += getVarint(aVarint, (u64 *)piVal);
|
||||
pIter->nKey = (int)nRec;
|
||||
rc = vdbeSorterIterRead(db, pIter, nRec, &pIter->aKey);
|
||||
}
|
||||
|
||||
return rc;
|
||||
|
@ -264,22 +301,43 @@ static int vdbeSorterIterInit(
|
|||
VdbeSorterIter *pIter, /* Iterator to populate */
|
||||
i64 *pnByte /* IN/OUT: Increment this value by PMA size */
|
||||
){
|
||||
int rc;
|
||||
int rc = SQLITE_OK;
|
||||
int nBuf;
|
||||
|
||||
nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
|
||||
|
||||
assert( pSorter->iWriteOff>iStart );
|
||||
assert( pIter->aAlloc==0 );
|
||||
assert( pIter->aBuffer==0 );
|
||||
pIter->pFile = pSorter->pTemp1;
|
||||
pIter->iReadOff = iStart;
|
||||
pIter->nAlloc = 128;
|
||||
pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc);
|
||||
if( !pIter->aAlloc ){
|
||||
pIter->nBuffer = nBuf;
|
||||
pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf);
|
||||
|
||||
if( !pIter->aBuffer ){
|
||||
rc = SQLITE_NOMEM;
|
||||
}else{
|
||||
i64 nByte; /* Total size of PMA in bytes */
|
||||
rc = vdbeSorterReadVarint(pSorter->pTemp1, &pIter->iReadOff, &nByte);
|
||||
*pnByte += nByte;
|
||||
pIter->iEof = pIter->iReadOff + nByte;
|
||||
int iBuf;
|
||||
|
||||
iBuf = pIter->iReadOff % nBuf;
|
||||
if( iBuf ){
|
||||
rc = sqlite3OsRead(
|
||||
pSorter->pTemp1, &pIter->aBuffer[iBuf], nBuf-iBuf, iStart
|
||||
);
|
||||
assert( rc!=SQLITE_IOERR_SHORT_READ );
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
u64 nByte; /* Size of PMA in bytes */
|
||||
pIter->iEof = iStart + pIter->nBuffer;
|
||||
rc = vdbeSorterIterVarint(db, pIter, &nByte);
|
||||
pIter->iEof = pIter->iReadOff + nByte;
|
||||
*pnByte += nByte;
|
||||
}
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = vdbeSorterIterNext(db, pIter);
|
||||
}
|
||||
|
@ -531,6 +589,92 @@ static int vdbeSorterSort(VdbeCursor *pCsr){
|
|||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/*
|
||||
** Initialize a file-writer object.
|
||||
*/
|
||||
static int fileWriterInit(
|
||||
sqlite3 *db, /* Database (for malloc) */
|
||||
sqlite3_file *pFile, /* File to write to */
|
||||
FileWriter *p, /* Object to populate */
|
||||
i64 iStart /* Offset of pFile to begin writing at */
|
||||
){
|
||||
int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt);
|
||||
|
||||
memset(p, 0, sizeof(FileWriter));
|
||||
p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf);
|
||||
if( !p->aBuffer ) return SQLITE_NOMEM;
|
||||
|
||||
p->iBufEnd = p->iBufStart = (iStart % nBuf);
|
||||
p->iWriteOff = iStart - p->iBufStart;
|
||||
p->nBuffer = nBuf;
|
||||
p->pFile = pFile;
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/*
|
||||
** Write nData bytes of data to the file-write object. Return SQLITE_OK
|
||||
** if successful, or an SQLite error code if an error occurs.
|
||||
*/
|
||||
static int fileWriterWrite(FileWriter *p, u8 *pData, int nData){
|
||||
int nRem = nData;
|
||||
while( nRem>0 ){
|
||||
int nCopy = nRem;
|
||||
if( nCopy>(p->nBuffer - p->iBufEnd) ){
|
||||
nCopy = p->nBuffer - p->iBufEnd;
|
||||
}
|
||||
|
||||
memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy);
|
||||
p->iBufEnd += nCopy;
|
||||
if( p->iBufEnd==p->nBuffer ){
|
||||
int rc = sqlite3OsWrite(p->pFile,
|
||||
&p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart,
|
||||
p->iWriteOff + p->iBufStart
|
||||
);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
p->iBufStart = p->iBufEnd = 0;
|
||||
p->iWriteOff += p->nBuffer;
|
||||
}
|
||||
assert( p->iBufEnd<p->nBuffer );
|
||||
|
||||
nRem -= nCopy;
|
||||
}
|
||||
|
||||
return SQLITE_OK;
|
||||
}
|
||||
|
||||
/*
|
||||
** Flush any buffered data to disk and clean up the file-writer object.
|
||||
** The results of using the file-writer after this call are undefined.
|
||||
** Return SQLITE_OK if flushing the buffered data succeeds or is not
|
||||
** required. Otherwise, return an SQLite error code.
|
||||
**
|
||||
** Before returning, set *piEof to the offset immediately following the
|
||||
** last byte written to the file.
|
||||
*/
|
||||
static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){
|
||||
int rc = SQLITE_OK;
|
||||
if( p->aBuffer && p->iBufEnd>p->iBufStart ){
|
||||
rc = sqlite3OsWrite(p->pFile,
|
||||
&p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart,
|
||||
p->iWriteOff + p->iBufStart
|
||||
);
|
||||
}
|
||||
*piEof = (p->iWriteOff + p->iBufEnd);
|
||||
sqlite3DbFree(db, p->aBuffer);
|
||||
memset(p, 0, sizeof(FileWriter));
|
||||
return rc;
|
||||
}
|
||||
|
||||
/*
|
||||
** Write value iVal encoded as a varint to the file-write object. Return
|
||||
** SQLITE_OK if successful, or an SQLite error code if an error occurs.
|
||||
*/
|
||||
static int fileWriterWriteVarint(FileWriter *p, u64 iVal){
|
||||
int nByte;
|
||||
u8 aByte[10];
|
||||
nByte = sqlite3PutVarint(aByte, iVal);
|
||||
return fileWriterWrite(p, aByte, nByte);
|
||||
}
|
||||
|
||||
/*
|
||||
** Write the current contents of the in-memory linked-list to a PMA. Return
|
||||
|
@ -547,7 +691,11 @@ static int vdbeSorterSort(VdbeCursor *pCsr){
|
|||
*/
|
||||
static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){
|
||||
int rc = SQLITE_OK; /* Return code */
|
||||
int rc2; /* fileWriterFinish return code */
|
||||
VdbeSorter *pSorter = pCsr->pSorter;
|
||||
FileWriter writer;
|
||||
|
||||
memset(&writer, 0, sizeof(FileWriter));
|
||||
|
||||
if( pSorter->nInMemory==0 ){
|
||||
assert( pSorter->pRecord==0 );
|
||||
|
@ -565,41 +713,32 @@ static int vdbeSorterListToPMA(sqlite3 *db, VdbeCursor *pCsr){
|
|||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
i64 iOff = pSorter->iWriteOff;
|
||||
rc = fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff);
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
SorterRecord *p;
|
||||
SorterRecord *pNext = 0;
|
||||
static const char eightZeros[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
|
||||
|
||||
|
||||
pSorter->nPMA++;
|
||||
rc = vdbeSorterWriteVarint(pSorter->pTemp1, pSorter->nInMemory, &iOff);
|
||||
rc = fileWriterWriteVarint(&writer, pSorter->nInMemory);
|
||||
for(p=pSorter->pRecord; rc==SQLITE_OK && p; p=pNext){
|
||||
pNext = p->pNext;
|
||||
rc = vdbeSorterWriteVarint(pSorter->pTemp1, p->nVal, &iOff);
|
||||
|
||||
rc = fileWriterWriteVarint(&writer, p->nVal);
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3OsWrite(pSorter->pTemp1, p->pVal, p->nVal, iOff);
|
||||
iOff += p->nVal;
|
||||
rc = fileWriterWrite(&writer, p->pVal, p->nVal);
|
||||
}
|
||||
|
||||
sqlite3DbFree(db, p);
|
||||
}
|
||||
|
||||
/* This assert verifies that unless an error has occurred, the size of
|
||||
** the PMA on disk is the same as the expected size stored in
|
||||
** pSorter->nInMemory. */
|
||||
assert( rc!=SQLITE_OK || pSorter->nInMemory==(
|
||||
iOff-pSorter->iWriteOff-sqlite3VarintLen(pSorter->nInMemory)
|
||||
));
|
||||
|
||||
pSorter->iWriteOff = iOff;
|
||||
if( rc==SQLITE_OK ){
|
||||
/* Terminate each file with 8 extra bytes so that from any offset
|
||||
** in the file we can always read 9 bytes without a SHORT_READ error */
|
||||
rc = sqlite3OsWrite(pSorter->pTemp1, eightZeros, 8, iOff);
|
||||
}
|
||||
pSorter->pRecord = p;
|
||||
}
|
||||
|
||||
rc2 = fileWriterFinish(db, &writer, &pSorter->iWriteOff);
|
||||
if( rc==SQLITE_OK ) rc = rc2;
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
@ -642,8 +781,14 @@ int sqlite3VdbeSorterWrite(
|
|||
(pSorter->nInMemory>pSorter->mxPmaSize)
|
||||
|| (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull())
|
||||
)){
|
||||
#ifdef SQLITE_DEBUG
|
||||
i64 nExpect = pSorter->iWriteOff
|
||||
+ sqlite3VarintLen(pSorter->nInMemory)
|
||||
+ pSorter->nInMemory;
|
||||
#endif
|
||||
rc = vdbeSorterListToPMA(db, pCsr);
|
||||
pSorter->nInMemory = 0;
|
||||
assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) );
|
||||
}
|
||||
|
||||
return rc;
|
||||
|
@ -704,7 +849,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
|
|||
return vdbeSorterSort(pCsr);
|
||||
}
|
||||
|
||||
/* Write the current b-tree to a PMA. Close the b-tree cursor. */
|
||||
/* Write the current in-memory list to a PMA. */
|
||||
rc = vdbeSorterListToPMA(db, pCsr);
|
||||
if( rc!=SQLITE_OK ) return rc;
|
||||
|
||||
|
@ -726,8 +871,12 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
|
|||
rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNT<pSorter->nPMA;
|
||||
iNew++
|
||||
){
|
||||
int rc2; /* Return code from fileWriterFinish() */
|
||||
FileWriter writer; /* Object used to write to disk */
|
||||
i64 nWrite; /* Number of bytes in new PMA */
|
||||
|
||||
memset(&writer, 0, sizeof(FileWriter));
|
||||
|
||||
/* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1,
|
||||
** initialize an iterator for each of them and break out of the loop.
|
||||
** These iterators will be incrementally merged as the VDBE layer calls
|
||||
|
@ -749,24 +898,30 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, VdbeCursor *pCsr, int *pbEof){
|
|||
rc = vdbeSorterOpenTempFile(db, &pTemp2);
|
||||
}
|
||||
|
||||
rc = fileWriterInit(db, pTemp2, &writer, iWrite2);
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = vdbeSorterWriteVarint(pTemp2, nWrite, &iWrite2);
|
||||
rc = fileWriterWriteVarint(&writer, nWrite);
|
||||
}
|
||||
|
||||
if( rc==SQLITE_OK ){
|
||||
int bEof = 0;
|
||||
while( rc==SQLITE_OK && bEof==0 ){
|
||||
int nToWrite;
|
||||
VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ];
|
||||
assert( pIter->pFile );
|
||||
nToWrite = pIter->nKey + sqlite3VarintLen(pIter->nKey);
|
||||
rc = sqlite3OsWrite(pTemp2, pIter->aAlloc, nToWrite, iWrite2);
|
||||
iWrite2 += nToWrite;
|
||||
|
||||
rc = fileWriterWriteVarint(&writer, pIter->nKey);
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = fileWriterWrite(&writer, pIter->aKey, pIter->nKey);
|
||||
}
|
||||
if( rc==SQLITE_OK ){
|
||||
rc = sqlite3VdbeSorterNext(db, pCsr, &bEof);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rc2 = fileWriterFinish(db, &writer, &iWrite2);
|
||||
if( rc==SQLITE_OK ) rc = rc2;
|
||||
}
|
||||
|
||||
if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){
|
||||
|
|
|
@ -17,6 +17,30 @@ source $testdir/tester.tcl
|
|||
|
||||
set testprefix index4
|
||||
|
||||
#proc str {n} { string range [string repeat [format %.06d. $n] 20] 0 101 }
|
||||
#db func str str
|
||||
#do_execsql_test 1.1 {
|
||||
# BEGIN;
|
||||
# CREATE TABLE t1(x);
|
||||
# INSERT INTO t1 VALUES(str(1));
|
||||
# INSERT INTO t1 SELECT str(rowid + 1) FROM t1; -- 2
|
||||
# INSERT INTO t1 SELECT str(rowid + 2) FROM t1; -- 4
|
||||
# INSERT INTO t1 SELECT str(rowid + 4) FROM t1; -- 8
|
||||
# INSERT INTO t1 SELECT str(rowid + 8) FROM t1; -- 16
|
||||
# INSERT INTO t1 SELECT str(rowid + 16) FROM t1; -- 32
|
||||
# INSERT INTO t1 SELECT str(rowid + 32) FROM t1; -- 64
|
||||
# INSERT INTO t1 SELECT str(rowid + 64) FROM t1; -- 128
|
||||
# INSERT INTO t1 SELECT str(rowid + 128) FROM t1; -- 256
|
||||
# INSERT INTO t1 SELECT str(rowid + 256) FROM t1; -- 512
|
||||
# INSERT INTO t1 SELECT str(rowid + 512) FROM t1; -- 1024
|
||||
# INSERT INTO t1 SELECT str(rowid + 1024) FROM t1; -- 2048
|
||||
# INSERT INTO t1 SELECT str(rowid + 2048) FROM t1; -- 4096
|
||||
# INSERT INTO t1 SELECT str(rowid + 4096) FROM t1; -- 8192
|
||||
# INSERT INTO t1 SELECT str(rowid + 8192) FROM t1; -- 16384
|
||||
# INSERT INTO t1 SELECT str(rowid + 16384) FROM t1; -- 32768
|
||||
# COMMIT;
|
||||
#}
|
||||
|
||||
do_execsql_test 1.1 {
|
||||
BEGIN;
|
||||
CREATE TABLE t1(x);
|
||||
|
|
Loading…
Reference in New Issue