mirrors/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Avoid creating a temp table in the user database in the sqlite3_expert code.

Browse Source 
Trouble is, this makes sampling for stat1 data much slower.

FossilOrigin-Name: c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5
This commit is contained in:
dan 2017-04-20 16:08:33 +00:00
parent e53b4f9774
commit 2ed99def5b
3 changed files with 210 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
ext/expert/sqlite3expert.c
View File

@ -281,6 +281,59 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
return pNew;
}
/*
** An error associated with database handle db has just occurred. Pass
** the error message to callback function xOut.
*/
static void idxDatabaseError(
sqlite3 *db, /* Database handle */
char **pzErrmsg /* Write error here */
){
*pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
}
/*
** Prepare an SQL statement.
*/
static int idxPrepareStmt(
sqlite3 *db, /* Database handle to compile against */
sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
const char *zSql /* SQL statement to compile */
){
int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
if( rc!=SQLITE_OK ){
*ppStmt = 0;
idxDatabaseError(db, pzErrmsg);
}
return rc;
}
/*
** Prepare an SQL statement using the results of a printf() formatting.
*/
static int idxPrintfPrepareStmt(
sqlite3 *db, /* Database handle to compile against */
sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
const char *zFmt, /* printf() format of SQL statement */
... /* Trailing printf() arguments */
){
va_list ap;
int rc;
char *zSql;
va_start(ap, zFmt);
zSql = sqlite3_vmprintf(zFmt, ap);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
sqlite3_free(zSql);
}
va_end(ap);
return rc;
}
/*************************************************************************
** Beginning of virtual table implementation.
@ -292,6 +345,17 @@ struct ExpertVtab {
sqlite3expert *pExpert;
};
typedef struct ExpertCsr ExpertCsr;
struct ExpertCsr {
sqlite3_vtab_cursor base;
sqlite3_stmt *pData;
int iTarget; /* Target as a percentage */
double target; /* Target nRet/nRow value */
double nRow; /* Rows seen */
double nRet; /* Rows returned */
};
static char *expertDequote(const char *zIn){
int n = strlen(zIn);
char *zRet = sqlite3_malloc(n);
@ -389,7 +453,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
/* Add the constraints to the IdxScan object */
for(i=0; i<pIdxInfo->nConstraint; i++){
struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
if( pCons->usable
&& pCons->iColumn>=0
&& p->pTab->aCol[pCons->iColumn].iPk==0
&& (pCons->op & opmask)
){
IdxConstraint *pNew;
const char *zColl = sqlite3_vtab_collation(dbv, i);
pNew = idxNewConstraint(&rc, zColl);
@ -439,6 +507,122 @@ static int expertUpdate(
return SQLITE_OK;
}
/*
** Virtual table module xOpen method.
*/
static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
int rc = SQLITE_OK;
ExpertCsr *pCsr;
pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
*ppCursor = (sqlite3_vtab_cursor*)pCsr;
return rc;
}
/*
** Virtual table module xClose method.
*/
static int expertClose(sqlite3_vtab_cursor *cur){
ExpertCsr *pCsr = (ExpertCsr*)cur;
sqlite3_finalize(pCsr->pData);
sqlite3_free(pCsr);
return SQLITE_OK;
}
/*
** Virtual table module xEof method.
**
** Return non-zero if the cursor does not currently point to a valid
** record (i.e if the scan has finished), or zero otherwise.
*/
static int expertEof(sqlite3_vtab_cursor *cur){
ExpertCsr *pCsr = (ExpertCsr*)cur;
return pCsr->pData==0;
}
/*
** Virtual table module xNext method.
*/
static int expertNext(sqlite3_vtab_cursor *cur){
ExpertCsr *pCsr = (ExpertCsr*)cur;
int rc = SQLITE_OK;
int bRet;
assert( pCsr->pData );
do {
rc = sqlite3_step(pCsr->pData);
if( rc!=SQLITE_ROW ){
rc = sqlite3_finalize(pCsr->pData);
pCsr->pData = 0;
bRet = 1;
}else{
rc = SQLITE_OK;
bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
if( bRet==0 ){
unsigned short rnd;
sqlite3_randomness(2, (void*)&rnd);
bRet = ((int)rnd % 100) <= pCsr->iTarget;
}
}
pCsr->nRow += 1.0;
}while( bRet==0 );
pCsr->nRet += 1.0;
return rc;
}
/*
** Virtual table module xRowid method.
*/
static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
*pRowid = 0;
return SQLITE_OK;
}
/*
** Virtual table module xColumn method.
*/
static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
ExpertCsr *pCsr = (ExpertCsr*)cur;
sqlite3_value *pVal;
pVal = sqlite3_column_value(pCsr->pData, i);
if( pVal ){
sqlite3_result_value(ctx, pVal);
}
return SQLITE_OK;
}
/*
** Virtual table module xFilter method.
*/
static int expertFilter(
sqlite3_vtab_cursor *cur,
int idxNum, const char *idxStr,
int argc, sqlite3_value **argv
){
ExpertCsr *pCsr = (ExpertCsr*)cur;
ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
sqlite3expert *pExpert = pVtab->pExpert;
int rc;
rc = sqlite3_finalize(pCsr->pData);
pCsr->pData = 0;
pCsr->nRow = 0.0;
pCsr->nRet = 0.0;
pCsr->iTarget = pExpert->iSample;
pCsr->target = (double)pExpert->iSample / 100.0;
if( rc==SQLITE_OK ){
rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
"SELECT * FROM main.%Q", pVtab->pTab->zName
);
}
if( rc==SQLITE_OK ){
rc = expertNext(cur);
}
return rc;
}
static int idxRegisterVtab(sqlite3expert *p){
static sqlite3_module expertModule = {
2, /* iVersion */
@ -447,13 +631,13 @@ static int idxRegisterVtab(sqlite3expert *p){
expertBestIndex, /* xBestIndex - Determine search strategy */
expertDisconnect, /* xDisconnect - Disconnect from a table */
expertDisconnect, /* xDestroy - Drop a table */
0, /* xOpen - open a cursor */
0, /* xClose - close a cursor */
0, /* xFilter - configure scan constraints */
0, /* xNext - advance a cursor */
0, /* xEof */
0, /* xColumn - read data */
0, /* xRowid - read data */
expertOpen, /* xOpen - open a cursor */
expertClose, /* xClose - close a cursor */
expertFilter, /* xFilter - configure scan constraints */
expertNext, /* xNext - advance a cursor */
expertEof, /* xEof */
expertColumn, /* xColumn - read data */
expertRowid, /* xRowid - read data */
expertUpdate, /* xUpdate - write data */
0, /* xBegin - begin transaction */
0, /* xSync - sync transaction */
@ -471,60 +655,6 @@ static int idxRegisterVtab(sqlite3expert *p){
/*
** End of virtual table implementation.
*************************************************************************/
/*
** An error associated with database handle db has just occurred. Pass
** the error message to callback function xOut.
*/
static void idxDatabaseError(
sqlite3 *db, /* Database handle */
char **pzErrmsg /* Write error here */
){
*pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
}
/*
** Prepare an SQL statement.
*/
static int idxPrepareStmt(
sqlite3 *db, /* Database handle to compile against */
sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
const char *zSql /* SQL statement to compile */
){
int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
if( rc!=SQLITE_OK ){
*ppStmt = 0;
idxDatabaseError(db, pzErrmsg);
}
return rc;
}
/*
** Prepare an SQL statement using the results of a printf() formatting.
*/
static int idxPrintfPrepareStmt(
sqlite3 *db, /* Database handle to compile against */
sqlite3_stmt **ppStmt, /* OUT: Compiled SQL statement */
char **pzErrmsg, /* OUT: sqlite3_malloc()ed error message */
const char *zFmt, /* printf() format of SQL statement */
... /* Trailing printf() arguments */
){
va_list ap;
int rc;
char *zSql;
va_start(ap, zFmt);
zSql = sqlite3_vmprintf(zFmt, ap);
if( zSql==0 ){
rc = SQLITE_NOMEM;
}else{
rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
sqlite3_free(zSql);
}
va_end(ap);
return rc;
}
/*
** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
** is called, set it to the return value of sqlite3_finalize() before
@ -557,7 +687,7 @@ static int idxGetTableInfo(
int nByte = sizeof(IdxTable) + nTab + 1;
IdxTable *pNew = 0;
int rc, rc2;
char *pCsr;
char *pCsr = 0;
rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
@ -1425,7 +1555,8 @@ static int idxPopulateOneStat1(
/* Formulate the query text */
if( rc==SQLITE_OK ){
rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
}
sqlite3_free(zQuery);
@ -1481,16 +1612,14 @@ static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
int rc;
char *zSql;
rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
if( rc!=SQLITE_OK ) return rc;
zSql = sqlite3_mprintf(
"CREATE TABLE temp." UNIQUE_TABLE_NAME
" AS SELECT * FROM %Q WHERE sample()"
, zTab
"CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
);
if( zSql==0 ) return SQLITE_NOMEM;
rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
sqlite3_free(zSql);
return rc;
@ -1537,8 +1666,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
}
if( rc==SQLITE_OK ){
sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
rc = sqlite3_create_function(
p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
);
}
if( rc==SQLITE_OK ){

15
manifest
View File

@ -1,5 +1,5 @@
C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
D 2017-04-20T09:54:04.700
C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
D 2017-04-20T16:08:33.333
F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@ -1579,7 +1579,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
R 2a75643c384b85cff6c5d9d5615ac2a2
P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
R 02fc06fd99e34f4a0d95661da624b768
T *branch * schemalint-failure
T *sym-schemalint-failure *
T -sym-schemalint *
U dan
Z 7876a034c88841c26664b8a83eb3817f
Z 8a7b9cb83c65b8cfdb4d171254abdcee

2
manifest.uuid
View File

@ -1 +1 @@
c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5