Avoid creating a temp table in the user database in the sqlite3_expert code.

Trouble is, this makes sampling for stat1 data much slower.

FossilOrigin-Name: c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5

ext/expert/sqlite3expert.c

@@ -281,6 +281,59 @@ static IdxConstraint *idxNewConstraint(int *pRc, const char *zColl){
   return pNew;
 }
 
+/*
+** An error associated with database handle db has just occurred. Pass
+** the error message to callback function xOut.
+*/
+static void idxDatabaseError(
+  sqlite3 *db,                    /* Database handle */
+  char **pzErrmsg                 /* Write error here */
+){
+  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
+}
+
+/*
+** Prepare an SQL statement.
+*/
+static int idxPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zSql                /* SQL statement to compile */
+){
+  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
+  if( rc!=SQLITE_OK ){
+    *ppStmt = 0;
+    idxDatabaseError(db, pzErrmsg);
+  }
+  return rc;
+}
+
+/*
+** Prepare an SQL statement using the results of a printf() formatting.
+*/
+static int idxPrintfPrepareStmt(
+  sqlite3 *db,                    /* Database handle to compile against */
+  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
+  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
+  const char *zFmt,               /* printf() format of SQL statement */
+  ...                             /* Trailing printf() arguments */
+){
+  va_list ap;
+  int rc;
+  char *zSql;
+  va_start(ap, zFmt);
+  zSql = sqlite3_vmprintf(zFmt, ap);
+  if( zSql==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
+    sqlite3_free(zSql);
+  }
+  va_end(ap);
+  return rc;
+}
+
 
 /*************************************************************************
 ** Beginning of virtual table implementation.
@@ -292,6 +345,17 @@ struct ExpertVtab {
   sqlite3expert *pExpert;
 };
 
+typedef struct ExpertCsr ExpertCsr;
+struct ExpertCsr {
+  sqlite3_vtab_cursor base;
+  sqlite3_stmt *pData;
+
+  int iTarget;                    /* Target as a percentage */
+  double target;                  /* Target nRet/nRow value */
+  double nRow;                    /* Rows seen */
+  double nRet;                    /* Rows returned */
+};
+
 static char *expertDequote(const char *zIn){
   int n = strlen(zIn);
   char *zRet = sqlite3_malloc(n);
@@ -389,7 +453,11 @@ static int expertBestIndex(sqlite3_vtab *pVtab, sqlite3_index_info *pIdxInfo){
     /* Add the constraints to the IdxScan object */
     for(i=0; i<pIdxInfo->nConstraint; i++){
       struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
-      if( pCons->usable && pCons->iColumn>=0 && (pCons->op & opmask) ){
+      if( pCons->usable 
+       && pCons->iColumn>=0 
+       && p->pTab->aCol[pCons->iColumn].iPk==0
+       && (pCons->op & opmask) 
+      ){
         IdxConstraint *pNew;
         const char *zColl = sqlite3_vtab_collation(dbv, i);
         pNew = idxNewConstraint(&rc, zColl);
@@ -439,6 +507,122 @@ static int expertUpdate(
   return SQLITE_OK;
 }
 
+/* 
+** Virtual table module xOpen method.
+*/
+static int expertOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  int rc = SQLITE_OK;
+  ExpertCsr *pCsr;
+  pCsr = idxMalloc(&rc, sizeof(ExpertCsr));
+  *ppCursor = (sqlite3_vtab_cursor*)pCsr;
+  return rc;
+}
+
+/* 
+** Virtual table module xClose method.
+*/
+static int expertClose(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_finalize(pCsr->pData);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Virtual table module xEof method.
+**
+** Return non-zero if the cursor does not currently point to a valid 
+** record (i.e if the scan has finished), or zero otherwise.
+*/
+static int expertEof(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  return pCsr->pData==0;
+}
+
+/* 
+** Virtual table module xNext method.
+*/
+static int expertNext(sqlite3_vtab_cursor *cur){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  int rc = SQLITE_OK;
+  int bRet;
+  assert( pCsr->pData );
+
+  do {
+    rc = sqlite3_step(pCsr->pData);
+    if( rc!=SQLITE_ROW ){
+      rc = sqlite3_finalize(pCsr->pData);
+      pCsr->pData = 0;
+      bRet = 1;
+    }else{
+      rc = SQLITE_OK;
+      bRet = (pCsr->nRow==0.0 || pCsr->nRow/pCsr->nRet < pCsr->target);
+      if( bRet==0 ){
+        unsigned short rnd;
+        sqlite3_randomness(2, (void*)&rnd);
+        bRet = ((int)rnd % 100) <= pCsr->iTarget;
+      }
+    }
+    pCsr->nRow += 1.0;
+  }while( bRet==0 );
+
+  pCsr->nRet += 1.0;
+  return rc;
+}
+
+/* 
+** Virtual table module xRowid method.
+*/
+static int expertRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){
+  *pRowid = 0;
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xColumn method.
+*/
+static int expertColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  sqlite3_value *pVal;
+  pVal = sqlite3_column_value(pCsr->pData, i);
+  if( pVal ){
+    sqlite3_result_value(ctx, pVal);
+  }
+  return SQLITE_OK;
+}
+
+/* 
+** Virtual table module xFilter method.
+*/
+static int expertFilter(
+  sqlite3_vtab_cursor *cur, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  ExpertCsr *pCsr = (ExpertCsr*)cur;
+  ExpertVtab *pVtab = (ExpertVtab*)(cur->pVtab);
+  sqlite3expert *pExpert = pVtab->pExpert;
+  int rc;
+
+  rc = sqlite3_finalize(pCsr->pData);
+  pCsr->pData = 0;
+  pCsr->nRow = 0.0;
+  pCsr->nRet = 0.0;
+  pCsr->iTarget = pExpert->iSample;
+  pCsr->target = (double)pExpert->iSample / 100.0;
+
+  if( rc==SQLITE_OK ){
+    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
+        "SELECT * FROM main.%Q", pVtab->pTab->zName
+    );
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = expertNext(cur);
+  }
+  return rc;
+}
+
 static int idxRegisterVtab(sqlite3expert *p){
   static sqlite3_module expertModule = {
     2,                            /* iVersion */
@@ -447,13 +631,13 @@ static int idxRegisterVtab(sqlite3expert *p){
     expertBestIndex,              /* xBestIndex - Determine search strategy */
     expertDisconnect,             /* xDisconnect - Disconnect from a table */
     expertDisconnect,             /* xDestroy - Drop a table */
-    0,                            /* xOpen - open a cursor */
+    expertOpen,                   /* xOpen - open a cursor */
-    0,                            /* xClose - close a cursor */
+    expertClose,                  /* xClose - close a cursor */
-    0,                            /* xFilter - configure scan constraints */
+    expertFilter,                 /* xFilter - configure scan constraints */
-    0,                            /* xNext - advance a cursor */
+    expertNext,                   /* xNext - advance a cursor */
-    0,                            /* xEof */
+    expertEof,                    /* xEof */
-    0,                            /* xColumn - read data */
+    expertColumn,                 /* xColumn - read data */
-    0,                            /* xRowid - read data */
+    expertRowid,                  /* xRowid - read data */
     expertUpdate,                 /* xUpdate - write data */
     0,                            /* xBegin - begin transaction */
     0,                            /* xSync - sync transaction */
@@ -471,60 +655,6 @@ static int idxRegisterVtab(sqlite3expert *p){
 /*
 ** End of virtual table implementation.
 *************************************************************************/
-
-/*
-** An error associated with database handle db has just occurred. Pass
-** the error message to callback function xOut.
-*/
-static void idxDatabaseError(
-  sqlite3 *db,                    /* Database handle */
-  char **pzErrmsg                 /* Write error here */
-){
-  *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db));
-}
-
-/*
-** Prepare an SQL statement.
-*/
-static int idxPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zSql                /* SQL statement to compile */
-){
-  int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0);
-  if( rc!=SQLITE_OK ){
-    *ppStmt = 0;
-    idxDatabaseError(db, pzErrmsg);
-  }
-  return rc;
-}
-
-/*
-** Prepare an SQL statement using the results of a printf() formatting.
-*/
-static int idxPrintfPrepareStmt(
-  sqlite3 *db,                    /* Database handle to compile against */
-  sqlite3_stmt **ppStmt,          /* OUT: Compiled SQL statement */
-  char **pzErrmsg,                /* OUT: sqlite3_malloc()ed error message */
-  const char *zFmt,               /* printf() format of SQL statement */
-  ...                             /* Trailing printf() arguments */
-){
-  va_list ap;
-  int rc;
-  char *zSql;
-  va_start(ap, zFmt);
-  zSql = sqlite3_vmprintf(zFmt, ap);
-  if( zSql==0 ){
-    rc = SQLITE_NOMEM;
-  }else{
-    rc = idxPrepareStmt(db, ppStmt, pzErrmsg, zSql);
-    sqlite3_free(zSql);
-  }
-  va_end(ap);
-  return rc;
-}
-
 /*
 ** Finalize SQL statement pStmt. If (*pRc) is SQLITE_OK when this function
 ** is called, set it to the return value of sqlite3_finalize() before
@@ -557,7 +687,7 @@ static int idxGetTableInfo(
   int nByte = sizeof(IdxTable) + nTab + 1;
   IdxTable *pNew = 0;
   int rc, rc2;
-  char *pCsr;
+  char *pCsr = 0;
 
   rc = idxPrintfPrepareStmt(db, &p1, pzErrmsg, "PRAGMA table_info=%Q", zTab);
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(p1) ){
@@ -1425,7 +1555,8 @@ static int idxPopulateOneStat1(
 
   /* Formulate the query text */
   if( rc==SQLITE_OK ){
-    rc = idxPrepareStmt(p->db, &pQuery, pzErr, zQuery);
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
+    rc = idxPrepareStmt(dbrem, &pQuery, pzErr, zQuery);
   }
   sqlite3_free(zQuery);
 
@@ -1481,16 +1612,14 @@ static int idxBuildSampleTable(sqlite3expert *p, const char *zTab){
   int rc;
   char *zSql;
 
-  rc = sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
+  rc = sqlite3_exec(p->dbv,"DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   if( rc!=SQLITE_OK ) return rc;
 
   zSql = sqlite3_mprintf(
-      "CREATE TABLE temp." UNIQUE_TABLE_NAME 
+      "CREATE TABLE temp." UNIQUE_TABLE_NAME " AS SELECT * FROM %Q", zTab
-      "  AS SELECT * FROM %Q WHERE sample()"
-      , zTab
   );
   if( zSql==0 ) return SQLITE_NOMEM;
-  rc = sqlite3_exec(p->db, zSql, 0, 0, 0);
+  rc = sqlite3_exec(p->dbv, zSql, 0, 0, 0);
   sqlite3_free(zSql);
 
   return rc;
@@ -1537,8 +1666,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
   }
 
   if( rc==SQLITE_OK ){
+    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
     rc = sqlite3_create_function(
-        p->db, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
   if( rc==SQLITE_OK ){

manifest

@@ -1,5 +1,5 @@
-C Add\san\soption\sto\sgenerate\sstat1\sdata\sbased\son\sa\ssubset\sof\sthe\suser\sdatabase\ntable\scontents\sto\ssqlite3_expert.
+C Avoid\screating\sa\stemp\stable\sin\sthe\suser\sdatabase\sin\sthe\ssqlite3_expert\scode.\nTrouble\sis,\sthis\smakes\ssampling\sfor\sstat1\sdata\smuch\sslower.
-D 2017-04-20T09:54:04.700
+D 2017-04-20T16:08:33.333
 F Makefile.in 1cc758ce3374a32425e4d130c2fe7b026b20de5b8843243de75f087c0a2661fb
 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434
 F Makefile.msc 6a8c838220f7c00820e1fc0ac1bccaaa8e5676067e1dbfa1bafa7a4ffecf8ae6
@@ -43,7 +43,7 @@ F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef
 F ext/expert/README.md 9f15075ec5ad772808eff55ef044c31140fd1146aa0a3c47eafd155e71851b01
 F ext/expert/expert.c 33842ef151d84c5f8000f9c7b938998c6b999eaef7ce1f4eeb0df8ffe6739496
 F ext/expert/expert1.test 1033e43071b69dc2f4e88fbf03fc7f18846c9865cac14f28c80f581437f09acb
-F ext/expert/sqlite3expert.c af3b336f83bcd2a586f6119d4040ac36ccf45162c48e3780ed63ab119fd04fe1
+F ext/expert/sqlite3expert.c 68acd26b0520408f0c5e76bd9ddaf70d776d397f8aa1bd6eda91f42f34c0a8cf
 F ext/expert/sqlite3expert.h af6354f8ee5c9e025024e63fec3bd640a802afcc3099a44d804752cf0791d811
 F ext/expert/test_expert.c b01a5115f9444a9b416582c985138f5dfdb279848ce8b7452be383530be27f01
 F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e
@@ -1579,7 +1579,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P a157fcfde5afc27ae38e7cf4669fcc8e60e23d9d301ffe2e541dd69f895b493b
+P c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
-R 2a75643c384b85cff6c5d9d5615ac2a2
+R 02fc06fd99e34f4a0d95661da624b768
+T *branch * schemalint-failure
+T *sym-schemalint-failure *
+T -sym-schemalint *
 U dan
-Z 7876a034c88841c26664b8a83eb3817f
+Z 8a7b9cb83c65b8cfdb4d171254abdcee

manifest.uuid

@@ -1 +1 @@
-c69c3e21db6e141f7e24226c6432f2ed31fe5f177bd23781915871f8600ee56a
+c62e358243d96cb38a7ce2aa679fc640b62bf46080eab4bd5fc2acf5997d6cd5