Attempt to optimize virtual table queries with 'OR' expressions in the WHERE clause. (CVS 6527)

FossilOrigin-Name: f61e4cd93682fd98bea2a71d346f9eaa68454390

Makefile.in

@@ -172,7 +172,7 @@ OBJS0 = alter.lo analyze.lo attach.lo auth.lo backup.lo bitvec.lo btmutex.lo \
         mutex.lo mutex_noop.lo mutex_os2.lo mutex_unix.lo mutex_w32.lo \
         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo os_os2.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
-        random.lo resolve.lo rowset.lo select.lo status.lo \
+        random.lo resolve.lo rowhash.lo rowset.lo select.lo status.lo \
         table.lo tokenize.lo trigger.lo update.lo \
         util.lo vacuum.lo \
         vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo \
@@ -249,6 +249,7 @@ SRC = \
   $(TOP)/src/printf.c \
   $(TOP)/src/random.c \
   $(TOP)/src/resolve.c \
+  $(TOP)/src/rowhash.c \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \
@@ -671,6 +672,9 @@ random.lo:	$(TOP)/src/random.c $(HDR)
 resolve.lo:	$(TOP)/src/resolve.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/resolve.c
 
+rowhash.lo:	$(TOP)/src/rowhash.c $(HDR)
+	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/rowhash.c
+
 rowset.lo:	$(TOP)/src/rowset.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/rowset.c
 

main.mk

@@ -61,7 +61,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \
          mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \
          notify.o opcodes.o os.o os_os2.o os_unix.o os_win.o \
          pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \
-         random.o resolve.o rowset.o rtree.o select.o status.o \
+         random.o resolve.o rowhash.o rowset.o rtree.o select.o status.o \
          table.o tokenize.o trigger.o \
          update.o util.o vacuum.o \
          vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o \
@@ -130,6 +130,7 @@ SRC = \
   $(TOP)/src/printf.c \
   $(TOP)/src/random.c \
   $(TOP)/src/resolve.c \
+  $(TOP)/src/rowhash.c \
   $(TOP)/src/rowset.c \
   $(TOP)/src/select.c \
   $(TOP)/src/status.c \

manifest

@@ -1,7 +1,7 @@
-C Change\sthe\sjournal_mode\spragma\sso\sthat\sit\salways\sreturns\sthe\scurrent\njournal\smode,\seven\son\sa\sfailed\sattempt\sto\schange\sthe\sjournal\smode.\nAllow\sthe\sjournal\smode\sto\sbe\schanged\sas\slong\sas\sthere\sis\snot\sa\spending\ntransaction.\s\sTicket\s#3811.\s(CVS\s6526)
+C Attempt\sto\soptimize\svirtual\stable\squeries\swith\s'OR'\sexpressions\sin\sthe\sWHERE\sclause.\s(CVS\s6527)
-D 2009-04-20T17:43:03
+D 2009-04-21T09:02:46
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
-F Makefile.in 583e87706abc3026960ed759aff6371faf84c211
+F Makefile.in fa5998fe08bd8c0fdc7f9f66cea16c0279f39da8
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
 F Makefile.vxworks 40c707b2589f61436b89788003b1ccf5679ec3e6
 F README b974cdc3f9f12b87e851b04e75996d720ebf81ac
@@ -83,7 +83,7 @@ F ext/rtree/tkt3363.test 6662237ea75bb431cd5d262dfc9535e1023315fc
 F ext/rtree/viewrtree.tcl 09526398dae87a5a87c5aac2b3854dbaf8376869
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
-F main.mk bbb170882a34fe51dbd2d2e9c450c6cc0dad3325
+F main.mk aeaf069e8ec8238ccfab5bdc4219149b16a3ee04
 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -154,12 +154,13 @@ F src/prepare.c 72d74e6d3b9c8eb0663b33ec6438aa718096ac79
 F src/printf.c ea2d76000cc5f4579d7e9cb2f5460433eec0d384
 F src/random.c 676b9d7ac820fe81e6fb2394ac8c10cff7f38628
 F src/resolve.c 094e44450371fb27869eb8bf679aacbe51fdc56d
+F src/rowhash.c f1ebc89222c4095caf40d18176aed408669ddaa5
 F src/rowset.c badb9f36b3a2ced9ee9551f4ce730f5fab442791
 F src/select.c 35225756c247484f473678e5bd191d70a6e4dba0
 F src/shell.c 0a11f831603f17fea20ca97133c0f64e716af4a7
 F src/sqlite.h.in 8e0e256079bac2319380bdfebf403fcbe630510f
 F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17
-F src/sqliteInt.h fcdad0896da9c8b6372db974131e33b7a06606ce
+F src/sqliteInt.h 15ae1158343bd4062424f70941c12c31fc4c0354
 F src/sqliteLimit.h ffe93f5a0c4e7bd13e70cd7bf84cfb5c3465f45d
 F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
 F src/table.c cc86ad3d6ad54df7c63a3e807b5783c90411a08d
@@ -171,7 +172,7 @@ F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c
 F src/test5.c 162a1cea2105a2c460a3f39fa6919617b562a288
 F src/test6.c 1a0a7a1f179469044b065b4a88aab9faee114101
 F src/test7.c b94e68c2236de76889d82b8d7d8e00ad6a4d80b1
-F src/test8.c dd7fc4530ec02096a6901c057a476ea0c1dc1a3c
+F src/test8.c b1061548f7ce3aeedea3cc4d649ee1487c2b4eaf
 F src/test9.c 963d380922f25c1c323712d05db01b19197ee6f7
 F src/test_async.c c820a2d21ef910cbef613ca55938fc8d7545c84a
 F src/test_autoext.c f53b0cdf7bf5f08100009572a5d65cdb540bd0ad
@@ -200,16 +201,16 @@ F src/update.c 8ededddcde6f7b6da981dd0429a5d34518a475b7
 F src/utf.c 9541d28f40441812c0b40f00334372a0542c00ff
 F src/util.c 828c552a22a1d5b650b8a5ea0009546715c45d93
 F src/vacuum.c 07121a727beeee88f27d704a00313ad6a7c9bef0
-F src/vdbe.c 88bc70921ccdcff8bfdf574f3e2285d17ab97103
+F src/vdbe.c 60db222db8d0f04a7fd2c754e99754eb83d6ed24
 F src/vdbe.h 35a648bc3279a120da24f34d9a25213ec15daf8a
-F src/vdbeInt.h df5c5a1c739c98af2c83440dde3fc361240f3a25
+F src/vdbeInt.h d3adfeccc750643ae7861f2d29f579d3dad28785
 F src/vdbeapi.c 015c9d0fb7047657a13a7bb6aa886f75e43db02d
-F src/vdbeaux.c 5ecb4c7a041b8926a8927b1a27bcbb8ff74ae5c4
+F src/vdbeaux.c 8b2ecd0ed6fb7e2113c33618ea37f2abafd97717
 F src/vdbeblob.c e67757450ae8581a8b354d9d7e467e41502dfe38
-F src/vdbemem.c 9798905787baae83d0b53b62030e32ecf7a0586f
+F src/vdbemem.c 96e57468036638c3de72e2ed8b08f308c5982053
 F src/vtab.c 6118d71c5137e20a7ac51fb5d9beb0361fbedb89
 F src/walker.c 7cdf63223c953d4343c6833e940f110281a378ee
-F src/where.c ddf26069d03f9e0c6ef14d537422df02e0c593f0
+F src/where.c 2b580cc5eb4edd943458bac16248bc927778c3ee
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 597662c5d777a122f9a3df0047ea5c5bd383a911
 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45
@@ -439,7 +440,7 @@ F test/lock5.test 6b1f78f09ad1522843dad571b76b321e6f439bf7
 F test/lock6.test 862aa71e97b288d6b3f92ba3313f51bd0b003776
 F test/lookaside.test 1dd350dc6dff015c47c07fcc5a727a72fc5bae02
 F test/main.test 187a9a1b5248ed74a83838c581c15ec6023b555b
-F test/make-where7.tcl 40bb740b37eead343eaf57b74ab72d2a5a304745
+F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9
 F test/malloc.test cd2b9f515ba98beb5e335acdd94c3ad7a6f7fc4a
 F test/malloc3.test 4bc57f850b212f706f3e1b37c4eced1d5a727cd1
 F test/malloc4.test 957337613002b7058a85116493a262f679f3a261
@@ -505,6 +506,7 @@ F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a
 F test/rdonly.test bd054831f8a3078e765a0657e247182486f0cb47
 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254
 F test/rollback.test 1f70ab4301d8d105d41438a436cad1fc8897f5e5
+F test/rowhash.test 60e82105d7a5d209a245177995f45eb172012971
 F test/rowid.test 1c8fc43c60d273e6ea44dfb992db587f3164312c
 F test/rtree.test b85fd4f0861a40ca366ac195e363be2528dcfadf
 F test/safety.test b69e2b2dd5d52a3f78e216967086884bbc1a09c6
@@ -679,6 +681,7 @@ F test/vtab9.test ea58d2b95d61955f87226381716b2d0b1d4e4f9b
 F test/vtabA.test 0dcd4c81ffb56649f47d1b5fb9c5ae807ccf41f7
 F test/vtabB.test 04df5dc531b9f44d9ca65b9c1b79f12b5922a796
 F test/vtabC.test 1cf7896ab6859bfe3074244b2b0e12de5cbdd766
+F test/vtabD.test 6b74062aa6bcb681294706d1ea80ba2012bd4df2
 F test/vtab_alter.test 3a299749fee97ca3d53bd55717f536e4a2284856
 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
 F test/vtab_shared.test c19b2555b807ef2ee014c882cdda5bc8d84fcf48
@@ -688,10 +691,10 @@ F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94
 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
 F test/where6.test 42c4373595f4409d9c6a9987b4a60000ad664faf
-F test/where7.test 2487cda68faabf5edeb524289913f00f8d64e223
+F test/where7.test 42d5e19c88234bfd110e01dd890a449a8ecb24fa
-F test/where8.test 1b9152a086408ee789166d0a954abc597372f868
+F test/where8.test c53467ff79aa71ffc1f905d652d1e52be3fee661
-F test/where8m.test c1010d61826412ff66abd29bfb32e5d6b37d965c
+F test/where8m.test 5a15785fc19b45103da5a36a1be016b21d1e8c42
-F test/where9.test 12c1e46364fb245ff84253758dd76dacc7bfe619
+F test/where9.test e1756a1c8aa9145ebf9024b4dc80ceca336775fb
 F test/whereA.test 522469ca013ff97c81b5367e730042290889a061
 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/zeroblob.test 792124852ec61458a2eb527b5091791215e0be95
@@ -704,7 +707,7 @@ F tool/lempar.c aeba88b8566ff66f8a67c96b3eb2dd95e7d8908d
 F tool/mkkeywordhash.c 8e57fbe8c4fe2f1800f9190fd361231cb8558407
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e x
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
-F tool/mksqlite3c.tcl 671833bd775e76ebd922b9e82c2508a344562511
+F tool/mksqlite3c.tcl 2b34be291baff0fdfeb37a970b51cd7ff9891c2a
 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87
 F tool/omittest.tcl 27d6f6e3b1e95aeb26a1c140e6eb57771c6d794a
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@@ -718,7 +721,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P e62ac26f72224a4ba6c7dc5c32b7e4370461764d
+P 419e320ae51639794335d19699f8e1189e524e7d
-R 39db01b2d3bd255afa6f732be2b5d532
+R 347ea62f899ea16425c03d5a929dd221
-U drh
+U danielk1977
-Z d037c0a2ce0411a3e53db40f26904a65
+Z 1dd514a4d975c58998a5ffaa96ba7ca4

manifest.uuid

@@ -1 +1 @@
-419e320ae51639794335d19699f8e1189e524e7d
+f61e4cd93682fd98bea2a71d346f9eaa68454390

src/rowhash.c

@@ -0,0 +1,321 @@
+/*
+** 2009 April 15
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains the implementation of the "row-hash" data structure.
+**
+** $Id: rowhash.c,v 1.1 2009/04/21 09:02:47 danielk1977 Exp $
+*/
+#include "sqliteInt.h"
+
+typedef struct RowHashElem RowHashElem;
+typedef struct RowHashBlock RowHashBlock;
+
+/*
+** Size of heap allocations made by this module. This limit is 
+** never exceeded.
+*/
+#define ROWHASH_ALLOCATION 1024
+
+/*
+** Number of elements in the RowHashBlock.aElem[] array. This array is
+** sized to make RowHashBlock very close to (without exceeding)
+** ROWHASH_ALLOCATION bytes in size.
+*/
+#define ROWHASH_ELEM_PER_BLOCK (                                            \
+    (ROWHASH_ALLOCATION - ROUND8(sizeof(struct RowHashBlockData))) /        \
+    sizeof(RowHashElem)                                                     \
+)
+
+/*
+** Number of pointers that fit into a single allocation of 
+** ROWHASH_ALLOCATION bytes.
+*/
+#define ROWHASH_POINTER_PER_PAGE (ROWHASH_ALLOCATION/sizeof(void *))
+
+/*
+** If there are less than this number of elements in the block-list, do not
+** bother building a hash-table. Just do a linear search of the list when
+** querying.
+*/
+#define ROWHASH_LINEAR_SEARCH_LIMIT 10
+
+/*
+** Element stored in the hash-table.
+*/
+struct RowHashElem {
+  i64 iVal;
+  RowHashElem *pNext;
+};
+
+/*
+** The following structure is either exactly ROWHASH_ALLOCATION bytes in
+** size or just slightly less. It stores up to ROWHASH_ELEM_PER_BLOCK 
+** RowHashElem structures.
+*/
+struct RowHashBlock {
+  struct RowHashBlockData {
+    int nElem;
+    RowHashBlock *pNext;
+  } data;
+  RowHashElem aElem[ROWHASH_ELEM_PER_BLOCK];
+};
+
+/*
+** RowHash structure. References to a structure of this type are passed
+** around and used as opaque handles by code in other modules.
+*/
+struct RowHash {
+  /* Variables populated by sqlite3RowhashInsert() */
+  int nEntry;               /* Total number of entries in block-list */
+  RowHashBlock *pBlock;     /* Linked list of entries */
+
+  /* Variables populated by makeHashTable() */
+  int iSet;                 /* Most recent iSet parameter passed to Test() */
+  int iMod;                 /* Number of buckets in hash table */
+  int nLeaf;                /* Number of leaf pages in hash table */
+  int nHeight;              /* Height of tree containing leaf pages */
+  void *pHash;              /* Pointer to root of tree */
+  int nLinearLimit;         /* Linear search limit (used if pHash==0) */
+};
+
+
+/*
+** Allocate a tree of height nHeight with *pnLeaf leaf pages. Set *pp to
+** point to the root of the tree. If the maximum number of leaf pages in a
+** tree of height nHeight is less than *pnLeaf, allocate a tree with the 
+** maximum possible number of leaves for height nHeight. 
+**
+** Before returning, subtract the number of leaves in the tree allocated
+** from *pnLeaf.
+**
+** This routine returns SQLITE_NOMEM if a malloc() fails, or SQLITE_OK
+** otherwise.
+*/
+static int allocTable(void **pp, int nHeight, int *pnLeaf){
+  void **ap = (void **)sqlite3MallocZero(ROWHASH_ALLOCATION);
+  if( !ap ){
+    return SQLITE_NOMEM;
+  }
+  *pp = (void *)ap;
+  if( nHeight==0 ){
+    (*pnLeaf)--;
+  }else{
+    int ii;
+    for(ii=0; ii<ROWHASH_POINTER_PER_PAGE && *pnLeaf>0; ii++){
+      if( allocTable(&ap[ii], nHeight-1, pnLeaf) ){
+        return SQLITE_NOMEM;
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Delete the tree of height nHeight passed as the first argument.
+*/
+static void deleteTable(void **ap, int nHeight){
+  if( ap ){
+    if( nHeight>0 ){
+      int ii;
+      for(ii=0; ii<ROWHASH_POINTER_PER_PAGE; ii++){
+        deleteTable((void **)ap[ii], nHeight-1);
+      }
+    }
+    sqlite3_free(ap);
+  }
+}
+
+/*
+** Delete the hash-table stored in p->pHash. The p->pHash pointer is
+** set to zero before returning. This function is the inverse of 
+** allocHashTable()
+*/
+static void deleteHashTable(RowHash *p){
+  deleteTable(p->pHash, p->nHeight);
+  p->pHash = 0;
+}
+
+/*
+** Allocate the hash table structure based on the current values of
+** p->nLeaf and p->nHeight.
+*/
+static int allocHashTable(RowHash *p){
+  int nLeaf = p->nLeaf;
+  assert( p->pHash==0 );
+  assert( p->nLeaf>0 );
+  return allocTable(&p->pHash, p->nHeight, &nLeaf);
+}
+
+/*
+** Find the hash-bucket associated with value iVal. Return a pointer to it.
+*/
+static void **findHashBucket(RowHash *p, i64 iVal){
+  int aOffset[16];
+  int n = p->nHeight;
+  void **ap = p->pHash;
+  int h = (((u64)iVal) % p->iMod);
+  for(n=0; n<p->nHeight; n++){
+    int h1 = h / ROWHASH_POINTER_PER_PAGE;
+    aOffset[n] = h - (h1 * ROWHASH_POINTER_PER_PAGE);
+    h = h1;
+  }
+  aOffset[n] = h;
+  for(n=p->nHeight; n>0; n--){
+    ap = (void **)ap[aOffset[n]];
+  }
+  return &ap[aOffset[0]];
+}
+
+/*
+** Build a hash table to query with sqlite3RowhashTest() based on the
+** set of values stored in the linked list of RowHashBlock structures.
+*/
+static int makeHashTable(RowHash *p, int iSet){
+  RowHashBlock *pBlock;
+  int iMod;
+  int nLeaf;
+  
+  /* Delete the old hash table. */
+  deleteHashTable(p);
+  assert( p->iSet!=iSet );
+  p->iSet = iSet;
+
+  if( p->nEntry<ROWHASH_LINEAR_SEARCH_LIMIT ){
+    p->nLinearLimit = p->nEntry;
+    return SQLITE_OK;
+  }
+
+  /* Determine how many leaves the hash-table will comprise. */
+  nLeaf = 1 + (p->nEntry / ROWHASH_POINTER_PER_PAGE);
+  iMod = nLeaf*ROWHASH_POINTER_PER_PAGE;
+  p->nLeaf = nLeaf;
+  p->iMod = iMod;
+
+  /* Set nHeight to the height of the tree that contains the leaf pages. If
+  ** RowHash.nHeight is zero, then the whole hash-table fits on a single
+  ** leaf. If RowHash.nHeight is 1, then RowHash.pHash points to an array
+  ** of pointers to leaf pages. If 2, pHash points to an array of pointers
+  ** to arrays of pointers to leaf pages. And so on.
+  */
+  p->nHeight = 0;
+  while( nLeaf>1 ){
+    nLeaf = (nLeaf+ROWHASH_POINTER_PER_PAGE-1) / ROWHASH_POINTER_PER_PAGE;
+    p->nHeight++;
+  }
+
+  /* Allocate the hash-table. */
+  if( allocHashTable(p) ){
+    return SQLITE_NOMEM;
+  }
+
+  /* Insert all values into the hash-table. */
+  for(pBlock=p->pBlock; pBlock; pBlock=pBlock->data.pNext){
+    RowHashElem * const pEnd = &pBlock->aElem[pBlock->data.nElem];
+    RowHashElem *pIter;
+    for(pIter=pBlock->aElem; pIter<pEnd; pIter++){
+      RowHashElem **ppElem = (RowHashElem **)findHashBucket(p, pIter->iVal);
+      pIter->pNext = *ppElem;
+      *ppElem = pIter;
+    }
+  }
+
+  return SQLITE_OK;
+}
+
+/*
+** Test if value iVal is in the hash table. If so, set *pExists to 1
+** before returning. If iVal is not in the hash table, set *pExists to 0.
+**
+** Return SQLITE_OK if all goes as planned. If a malloc() fails, return
+** SQLITE_NOMEM.
+*/
+int sqlite3RowhashTest(RowHash *p, int iSet, i64 iVal, int *pExists){
+  *pExists = 0;
+  if( p ){
+    assert( p->pBlock );
+    if( iSet!=p->iSet && makeHashTable(p, iSet) ){
+      return SQLITE_NOMEM;
+    }
+    if( p->pHash ){
+      RowHashElem *pElem = *(RowHashElem **)findHashBucket(p, iVal);
+      for(; pElem; pElem=pElem->pNext){
+        if( pElem->iVal==iVal ){
+          *pExists = 1;
+          break;
+        }
+      }
+    }else{
+      int ii;
+      RowHashElem *aElem = p->pBlock->aElem;
+      for(ii=0; ii<p->nLinearLimit; ii++){
+        if( aElem[ii].iVal==iVal ){
+          *pExists = 1;
+          break;
+        }
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Insert value iVal into the RowHash object.
+**
+** Return SQLITE_OK if all goes as planned. If a malloc() fails, return
+** SQLITE_NOMEM.
+*/
+int sqlite3RowhashInsert(RowHash **pp, i64 iVal){
+  RowHash *p = *pp;
+  
+  /* If the RowHash structure has not been allocated, allocate it now. */
+  if( !p ){
+    p = (RowHash*)sqlite3MallocZero(sizeof(RowHash));
+    if( !p ){
+      return SQLITE_NOMEM;
+    }
+    *pp = p;
+  }
+
+  /* If the current RowHashBlock is full, or if the first RowHashBlock has
+  ** not yet been allocated, allocate one now. */ 
+  if( !p->pBlock || p->pBlock->data.nElem==ROWHASH_ELEM_PER_BLOCK ){
+    RowHashBlock *pBlock = (RowHashBlock*)sqlite3Malloc(sizeof(RowHashBlock));
+    if( !pBlock ){
+      return SQLITE_NOMEM;
+    }
+    pBlock->data.nElem = 0;
+    pBlock->data.pNext = p->pBlock;
+    p->pBlock = pBlock;
+  }
+
+  /* Add iVal to the current RowHashBlock. */
+  p->pBlock->aElem[p->pBlock->data.nElem].iVal = iVal;
+  p->pBlock->data.nElem++;
+  p->nEntry++;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy the RowHash object passed as the first argument.
+*/
+void sqlite3RowhashDestroy(RowHash *p){
+  if( p ){
+    RowHashBlock *pBlock, *pNext;
+    deleteHashTable(p);
+    for(pBlock=p->pBlock; pBlock; pBlock=pNext){
+      pNext = pBlock->data.pNext;
+      sqlite3_free(pBlock);
+    }
+    sqlite3_free(p);
+  }
+}
+

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.854 2009/04/08 13:51:51 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.855 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -557,6 +557,7 @@ struct BusyHandler {
 typedef struct AggInfo AggInfo;
 typedef struct AuthContext AuthContext;
 typedef struct Bitvec Bitvec;
+typedef struct RowHash RowHash;
 typedef struct RowSet RowSet;
 typedef struct CollSeq CollSeq;
 typedef struct Column Column;
@@ -1751,6 +1752,7 @@ struct WhereLevel {
 #define WHERE_FILL_ROWSET      0x0008  /* Save results in a RowSet object */
 #define WHERE_OMIT_OPEN        0x0010  /* Table cursor are already open */
 #define WHERE_OMIT_CLOSE       0x0020  /* Omit close of table & index cursors */
+#define WHERE_FILL_ROWHASH     0x0040  /* Save results in a RowHash object */
 
 /*
 ** The WHERE clause processing routine has two halves.  The
@@ -1763,7 +1765,8 @@ struct WhereInfo {
   Parse *pParse;       /* Parsing and code generating context */
   u16 wctrlFlags;      /* Flags originally passed to sqlite3WhereBegin() */
   u8 okOnePass;        /* Ok to use one-pass algorithm for UPDATE or DELETE */
-  int regRowSet;                 /* Store rowids in this rowset if >=0 */
+  int regRowSet;                 /* Store rowids in this rowset/rowhash */
+  int iRowidHandler;             /* Address of OP_RowSet or OP_RowHash */
   SrcList *pTabList;             /* List of tables in the join */
   int iTop;                      /* The very beginning of the WHERE loop */
   int iContinue;                 /* Jump here to continue with next record */
@@ -2399,6 +2402,10 @@ void sqlite3RowSetClear(RowSet*);
 void sqlite3RowSetInsert(RowSet*, i64);
 int sqlite3RowSetNext(RowSet*, i64*);
 
+int sqlite3RowhashInsert(RowHash **pp, i64 iVal);
+int sqlite3RowhashTest(RowHash *p, int iSet, i64 iVal, int *pExists);
+void sqlite3RowhashDestroy(RowHash *p);
+
 void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
 
 #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE)

src/test8.c

@@ -13,7 +13,7 @@
 ** is not included in the SQLite library.  It is used for automated
 ** testing of the SQLite library.
 **
-** $Id: test8.c,v 1.76 2009/04/08 15:45:32 drh Exp $
+** $Id: test8.c,v 1.77 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "tcl.h"
@@ -893,16 +893,16 @@ static int echoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
   pIdxInfo->idxNum = hashString(zQuery);
   pIdxInfo->idxStr = zQuery;
   pIdxInfo->needToFreeIdxStr = 1;
-  if (useCost) {
+  if( useCost ){
     pIdxInfo->estimatedCost = cost;
-  } else if( useIdx ){
+  }else if( useIdx ){
     /* Approximation of log2(nRow). */
     for( ii=0; ii<(sizeof(int)*8); ii++ ){
       if( nRow & (1<<ii) ){
         pIdxInfo->estimatedCost = (double)ii;
       }
     }
-  } else {
+  }else{
     pIdxInfo->estimatedCost = (double)nRow;
   }
   return rc;

src/vdbe.c

@@ -43,7 +43,7 @@
 ** in this file for details.  If in doubt, do not deviate from existing
 ** commenting and indentation practices when changing or adding code.
 **
-** $Id: vdbe.c,v 1.832 2009/04/10 12:55:17 danielk1977 Exp $
+** $Id: vdbe.c,v 1.833 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -3468,6 +3468,7 @@ case OP_NotExists: {        /* jump, in3 */
     pC->rowidIsValid = res==0 ?1:0;
     pC->nullRow = 0;
     pC->cacheStatus = CACHE_STALE;
+    pC->deferredMoveto = 0;
     if( res!=0 ){
       pc = pOp->p2 - 1;
       assert( pC->rowidIsValid==0 );
@@ -4601,6 +4602,55 @@ case OP_RowSetRead: {       /* jump, out3 */
   break;
 }
 
+/* Opcode: RowHash P1 P2 P3 P4
+**
+** Register P3 is assumed to hold an integer value. If register P1
+** contains a rowid-hash object and the rowid-hash object contains
+** the value held in P3, jump to register P2. Otherwise, insert the
+** integer in P3 into the rowid-hash container.
+**
+** The rowid-hash is optimized for the case where successive sets
+** of integers, where each set contains no duplicates. Each set
+** of values is identified by a unique P4 value. The first set
+** must have P4==0, the final set P4=-1.
+**
+** This allows optimizations: (a) when P4==0 there is no need to test
+** the row-hash object for P3, as it is guaranteed not to contain it,
+** (b) when P4==-1 there is no need to insert the value, as it will
+** never be tested for, and (c) when a value that is part of set X is
+** inserted, there is no need to search to see if the same value was
+** previously inserted as part of set X (only if it was previously
+** inserted as part of some other set).
+*/
+case OP_RowHash: {                     /* jump, in1, in3 */
+  int iSet = pOp->p4.i;
+  assert( pIn3->flags&MEM_Int );
+
+  /* If there is anything other than a row-hash object in memory cell P1,
+  ** delete it now and initialize P1 with an empty row-hash (a null pointer
+  ** is an acceptable representation of an empty row-hash).
+  */
+  if( (pIn1->flags & MEM_RowHash)==0 ){
+    sqlite3VdbeMemReleaseExternal(pIn1);
+    pIn1->u.pRowHash = 0;
+    pIn1->flags = MEM_RowHash;
+  }
+
+  assert( pOp->p4type==P4_INT32 );
+  if( iSet ){
+    int exists;
+    rc = sqlite3RowhashTest(pIn1->u.pRowHash, pOp->p4.i, pIn3->u.i, &exists);
+    if( exists ){
+      pc = pOp->p2 - 1;
+      break;
+    }
+  }
+  if( iSet>=0 ){
+    rc = sqlite3RowhashInsert(&pIn1->u.pRowHash, pIn3->u.i);
+  }
+  break;
+}
+
 
 #ifndef SQLITE_OMIT_TRIGGER
 /* Opcode: ContextPush * * * 

src/vdbeInt.h

@@ -15,7 +15,7 @@
 ** 6000 lines long) it was split up into several smaller files and
 ** this header information was factored out.
 **
-** $Id: vdbeInt.h,v 1.167 2009/04/10 12:55:17 danielk1977 Exp $
+** $Id: vdbeInt.h,v 1.168 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #ifndef _VDBEINT_H_
 #define _VDBEINT_H_
@@ -115,6 +115,7 @@ struct Mem {
     int nZero;          /* Used when bit MEM_Zero is set in flags */
     FuncDef *pDef;      /* Used only when flags==MEM_Agg */
     RowSet *pRowSet;    /* Used only when flags==MEM_RowSet */
+    RowHash *pRowHash;  /* Used only when flags==MEM_RowHash */
   } u;
   double r;           /* Real value */
   sqlite3 *db;        /* The associated database connection */
@@ -148,6 +149,7 @@ struct Mem {
 #define MEM_Real      0x0008   /* Value is a real number */
 #define MEM_Blob      0x0010   /* Value is a BLOB */
 #define MEM_RowSet    0x0020   /* Value is a RowSet object */
+#define MEM_RowHash   0x0040   /* Value is a RowHash object */
 #define MEM_TypeMask  0x00ff   /* Mask of type bits */
 
 /* Whenever Mem contains a valid string or blob representation, one of

src/vdbeaux.c

@@ -14,7 +14,7 @@
 ** to version 2.8.7, all this code was combined into the vdbe.c source file.
 ** But that file was getting too big so this subroutines were split out.
 **
-** $Id: vdbeaux.c,v 1.451 2009/04/10 15:42:36 shane Exp $
+** $Id: vdbeaux.c,v 1.452 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -1224,6 +1224,9 @@ static void Cleanup(Vdbe *p){
     if( pMem->flags & MEM_RowSet ){
       sqlite3RowSetClear(pMem->u.pRowSet);
     }
+    if( pMem->flags & MEM_RowHash ){
+      sqlite3RowhashDestroy(pMem->u.pRowHash);
+    }
     MemSetTypeFlag(pMem, MEM_Null);
   }
   releaseMemArray(&p->aMem[1], p->nMem);

src/vdbemem.c

@@ -15,7 +15,7 @@
 ** only within the VDBE.  Interface routines refer to a Mem using the
 ** name sqlite_value
 **
-** $Id: vdbemem.c,v 1.140 2009/04/05 12:22:09 drh Exp $
+** $Id: vdbemem.c,v 1.141 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "vdbeInt.h"
@@ -270,6 +270,7 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){
 */
 void sqlite3VdbeMemReleaseExternal(Mem *p){
   assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) );
+  if( p->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_RowHash) ){
     if( p->flags&MEM_Agg ){
       sqlite3VdbeMemFinalize(p, p->u.pDef);
       assert( (p->flags & MEM_Agg)==0 );
@@ -280,6 +281,10 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){
       p->xDel = 0;
     }else if( p->flags&MEM_RowSet ){
       sqlite3RowSetClear(p->u.pRowSet);
+    }else if( p->flags&MEM_RowHash ){
+      sqlite3RowhashDestroy(p->u.pRowHash);
+      p->u.pRowHash = 0;
+    }
   }
 }
 

src/where.c

@@ -16,7 +16,7 @@
 ** so is applicable.  Because this module is responsible for selecting
 ** indices, you might also think of this module as the "query optimizer".
 **
-** $Id: where.c,v 1.382 2009/04/07 13:48:12 drh Exp $
+** $Id: where.c,v 1.383 2009/04/21 09:02:47 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 
@@ -1475,51 +1475,114 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){
 #define TRACE_IDX_OUTPUTS(A)
 #endif
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
 /* 
-** Compute the best index for a virtual table.
+** Required because bestIndex() is called by bestOrClauseIndex() 
-**
-** The best index is computed by the xBestIndex method of the virtual
-** table module.  This routine is really just a wrapper that sets up
-** the sqlite3_index_info structure that is used to communicate with
-** xBestIndex.
-**
-** In a join, this routine might be called multiple times for the
-** same virtual table.  The sqlite3_index_info structure is created
-** and initialized on the first invocation and reused on all subsequent
-** invocations.  The sqlite3_index_info structure is also used when
-** code is generated to access the virtual table.  The whereInfoDelete() 
-** routine takes care of freeing the sqlite3_index_info structure after
-** everybody has finished with it.
 */
-static double bestVirtualIndex(
+static void bestIndex(
+    Parse*, WhereClause*, struct SrcList_item*, Bitmask, ExprList*, WhereCost*);
+
+/*
+** This routine attempts to find an scanning strategy that can be used 
+** to optimize an 'OR' expression that is part of a WHERE clause. 
+**
+** The table associated with FROM clause term pSrc may be either a
+** regular B-Tree table or a virtual table.
+*/
+static void bestOrClauseIndex(
   Parse *pParse,              /* The parsing context */
   WhereClause *pWC,           /* The WHERE clause */
   struct SrcList_item *pSrc,  /* The FROM clause term to search */
   Bitmask notReady,           /* Mask of cursors that are not available */
-  ExprList *pOrderBy,            /* The order by clause */
+  ExprList *pOrderBy,         /* The ORDER BY clause */
-  int orderByUsable,             /* True if we can potential sort */
+  WhereCost *pCost            /* Lowest cost query plan */
-  sqlite3_index_info **ppIdxInfo /* Index information passed to xBestIndex */
 ){
-  Table *pTab = pSrc->pTab;
+#ifndef SQLITE_OMIT_OR_OPTIMIZATION
-  sqlite3_vtab *pVtab = pTab->pVtab;
+  const int iCur = pSrc->iCursor;   /* The cursor of the table to be accessed */
-  sqlite3_index_info *pIdxInfo;
+  const Bitmask maskSrc = getMask(pWC->pMaskSet, iCur);  /* Bitmask for pSrc */
+  WhereTerm * const pWCEnd = &pWC->a[pWC->nTerm];        /* End of pWC->a[] */
+  WhereTerm *pTerm;                 /* A single term of the WHERE clause */
+
+  /* Search the WHERE clause terms for a usable WO_OR term. */
+  for(pTerm=pWC->a; pTerm<pWCEnd; pTerm++){
+    if( pTerm->eOperator==WO_OR 
+     && ((pTerm->prereqAll & ~maskSrc) & notReady)==0
+     && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 
+    ){
+      WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc;
+      WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm];
+      WhereTerm *pOrTerm;
+      int flags = WHERE_MULTI_OR;
+      double rTotal = 0;
+      double nRow = 0;
+
+      for(pOrTerm=pOrWC->a; pOrTerm<pOrWCEnd; pOrTerm++){
+        WhereCost sTermCost;
+        WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", 
+          (pOrTerm - pOrWC->a), (pTerm - pWC->a)
+        ));
+        if( pOrTerm->eOperator==WO_AND ){
+          WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
+          bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
+        }else if( pOrTerm->leftCursor==iCur ){
+          WhereClause tempWC;
+          tempWC.pParse = pWC->pParse;
+          tempWC.pMaskSet = pWC->pMaskSet;
+          tempWC.op = TK_AND;
+          tempWC.a = pOrTerm;
+          tempWC.nTerm = 1;
+          bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
+        }else{
+          continue;
+        }
+        rTotal += sTermCost.rCost;
+        nRow += sTermCost.nRow;
+        if( rTotal>=pCost->rCost ) break;
+      }
+
+      /* If there is an ORDER BY clause, increase the scan cost to account 
+      ** for the cost of the sort. */
+      if( pOrderBy!=0 ){
+        rTotal += nRow*estLog(nRow);
+        WHERETRACE(("... sorting increases OR cost to %.9g\n", rTotal));
+      }
+
+      /* If the cost of scanning using this OR term for optimization is
+      ** less than the current cost stored in pCost, replace the contents
+      ** of pCost. */
+      WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n", rTotal, nRow));
+      if( rTotal<pCost->rCost ){
+        pCost->rCost = rTotal;
+        pCost->nRow = nRow;
+        pCost->plan.wsFlags = flags;
+        pCost->plan.u.pTerm = pTerm;
+      }
+    }
+  }
+#endif /* SQLITE_OMIT_OR_OPTIMIZATION */
+}
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+/*
+** Allocate and populate an sqlite3_index_info structure. It is the 
+** responsibility of the caller to eventually release the structure
+** by passing the pointer returned by this function to sqlite3_free().
+*/
+static sqlite3_index_info *allocateIndexInfo(
+  Parse *pParse, 
+  WhereClause *pWC,
+  struct SrcList_item *pSrc,
+  ExprList *pOrderBy
+){
+  int i, j;
+  int nTerm;
   struct sqlite3_index_constraint *pIdxCons;
   struct sqlite3_index_orderby *pIdxOrderBy;
   struct sqlite3_index_constraint_usage *pUsage;
   WhereTerm *pTerm;
-  int i, j;
   int nOrderBy;
-  int rc;
+  sqlite3_index_info *pIdxInfo;
 
-  /* If the sqlite3_index_info structure has not been previously
+  WHERETRACE(("Recomputing index info for %s...\n", pSrc->pTab->zName));
-  ** allocated and initialized for this virtual table, then allocate
-  ** and initialize it now
-  */
-  pIdxInfo = *ppIdxInfo;
-  if( pIdxInfo==0 ){
-    int nTerm;
-    WHERETRACE(("Recomputing index info for %s...\n", pTab->zName));
 
   /* Count the number of possible WHERE clause constraints referring
   ** to this virtual table */
@@ -1555,9 +1618,8 @@ static double bestVirtualIndex(
   if( pIdxInfo==0 ){
     sqlite3ErrorMsg(pParse, "out of memory");
     /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
-      return (double)0;
+    return 0;
   }
-    *ppIdxInfo = pIdxInfo;
 
   /* Initialize the structure.  The sqlite3_index_info structure contains
   ** many fields that are declared "const" to prevent xBestIndex from
@@ -1600,6 +1662,98 @@ static double bestVirtualIndex(
     pIdxOrderBy[i].iColumn = pExpr->iColumn;
     pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder;
   }
+
+  return pIdxInfo;
+}
+
+/*
+** The table object reference passed as the second argument to this function
+** must represent a virtual table. This function invokes the xBestIndex()
+** method of the virtual table with the sqlite3_index_info pointer passed
+** as the argument.
+**
+** If an error occurs, pParse is populated with an error message and a
+** non-zero value is returned. Otherwise, 0 is returned and the output
+** part of the sqlite3_index_info structure is left populated.
+**
+** Whether or not an error is returned, it is the responsibility of the
+** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates
+** that this is required.
+*/
+static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
+  sqlite3_vtab *pVtab = pTab->pVtab;
+  int i;
+  int rc;
+
+  (void)sqlite3SafetyOff(pParse->db);
+  WHERETRACE(("xBestIndex for %s\n", pTab->zName));
+  TRACE_IDX_INPUTS(p);
+  rc = pVtab->pModule->xBestIndex(pVtab, p);
+  TRACE_IDX_OUTPUTS(p);
+  (void)sqlite3SafetyOn(pParse->db);
+
+  if( rc!=SQLITE_OK ){
+    if( rc==SQLITE_NOMEM ){
+      pParse->db->mallocFailed = 1;
+    }else if( !pVtab->zErrMsg ){
+      sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
+    }else{
+      sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
+    }
+  }
+  sqlite3DbFree(pParse->db, pVtab->zErrMsg);
+  pVtab->zErrMsg = 0;
+
+  for(i=0; i<p->nConstraint; i++){
+    if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){
+      sqlite3ErrorMsg(pParse, 
+          "table %s: xBestIndex returned an invalid plan", pTab->zName);
+    }
+  }
+
+  return pParse->nErr;
+}
+
+
+/*
+** Compute the best index for a virtual table.
+**
+** The best index is computed by the xBestIndex method of the virtual
+** table module.  This routine is really just a wrapper that sets up
+** the sqlite3_index_info structure that is used to communicate with
+** xBestIndex.
+**
+** In a join, this routine might be called multiple times for the
+** same virtual table.  The sqlite3_index_info structure is created
+** and initialized on the first invocation and reused on all subsequent
+** invocations.  The sqlite3_index_info structure is also used when
+** code is generated to access the virtual table.  The whereInfoDelete() 
+** routine takes care of freeing the sqlite3_index_info structure after
+** everybody has finished with it.
+*/
+static void bestVirtualIndex(
+  Parse *pParse,                  /* The parsing context */
+  WhereClause *pWC,               /* The WHERE clause */
+  struct SrcList_item *pSrc,      /* The FROM clause term to search */
+  Bitmask notReady,               /* Mask of cursors that are not available */
+  ExprList *pOrderBy,             /* The order by clause */
+  WhereCost *pCost,               /* Lowest cost query plan */
+  sqlite3_index_info **ppIdxInfo  /* Index information passed to xBestIndex */
+){
+  Table *pTab = pSrc->pTab;
+  sqlite3_index_info *pIdxInfo;
+  struct sqlite3_index_constraint *pIdxCons;
+  struct sqlite3_index_constraint_usage *pUsage;
+  WhereTerm *pTerm;
+  int i, j;
+  int nOrderBy;
+
+  /* If the sqlite3_index_info structure has not been previously
+  ** allocated and initialized, then allocate and initialize it now.
+  */
+  pIdxInfo = *ppIdxInfo;
+  if( pIdxInfo==0 ){
+    *ppIdxInfo = pIdxInfo = allocateIndexInfo(pParse, pWC, pSrc, pOrderBy);
   }
 
   /* At this point, the sqlite3_index_info structure that pIdxInfo points
@@ -1614,14 +1768,7 @@ static double bestVirtualIndex(
   ** sqlite3ViewGetColumnNames() would have picked up the error. 
   */
   assert( pTab->azModuleArg && pTab->azModuleArg[0] );
-  assert( pVtab );
+  assert( pTab->pVtab );
-#if 0
-  if( pTab->pVtab==0 ){
-    sqlite3ErrorMsg(pParse, "undefined module %s for table %s",
-        pTab->azModuleArg[0], pTab->zName);
-    return 0.0;
-  }
-#endif
 
   /* Set the aConstraint[].usable fields and initialize all 
   ** output variables to zero.
@@ -1661,40 +1808,38 @@ static double bestVirtualIndex(
   /* ((double)2) In case of SQLITE_OMIT_FLOATING_POINT... */
   pIdxInfo->estimatedCost = SQLITE_BIG_DBL / ((double)2);
   nOrderBy = pIdxInfo->nOrderBy;
-  if( pIdxInfo->nOrderBy && !orderByUsable ){
+  if( !pOrderBy ){
-    *(int*)&pIdxInfo->nOrderBy = 0;
+    pIdxInfo->nOrderBy = 0;
   }
 
-  (void)sqlite3SafetyOff(pParse->db);
+  if( vtabBestIndex(pParse, pTab, pIdxInfo) ){
-  WHERETRACE(("xBestIndex for %s\n", pTab->zName));
+    return;
-  TRACE_IDX_INPUTS(pIdxInfo);
+  }
-  rc = pVtab->pModule->xBestIndex(pVtab, pIdxInfo);
-  TRACE_IDX_OUTPUTS(pIdxInfo);
-  (void)sqlite3SafetyOn(pParse->db);
 
-  if( rc!=SQLITE_OK ){
+  /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the
-    if( rc==SQLITE_NOMEM ){
+  ** inital value of lowestCost in this loop. If it is, then the
-      pParse->db->mallocFailed = 1;
+  ** (cost<lowestCost) test below will never be true.
-    }else if( !pVtab->zErrMsg ){
+  ** 
-      sqlite3ErrorMsg(pParse, "%s", sqlite3ErrStr(rc));
+  ** Use "(double)2" instead of "2.0" in case OMIT_FLOATING_POINT 
+  ** is defined.
+  */
+  if( (SQLITE_BIG_DBL/((double)2))<pIdxInfo->estimatedCost ){
+    pCost->rCost = (SQLITE_BIG_DBL/((double)2));
   }else{
-      sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
+    pCost->rCost = pIdxInfo->estimatedCost;
   }
+  pCost->plan.wsFlags = WHERE_VIRTUALTABLE;
+  pCost->plan.u.pVtabIdx = pIdxInfo;
+  if( pIdxInfo && pIdxInfo->orderByConsumed ){
+    pCost->plan.wsFlags |= WHERE_ORDERBY;
   }
-  sqlite3DbFree(pParse->db, pVtab->zErrMsg);
+  pCost->plan.nEq = 0;
-  pVtab->zErrMsg = 0;
+  pIdxInfo->nOrderBy = nOrderBy;
 
-  for(i=0; i<pIdxInfo->nConstraint; i++){
+  /* Try to find a more efficient access pattern by using multiple indexes
-    if( !pIdxInfo->aConstraint[i].usable && pUsage[i].argvIndex>0 ){
+  ** to optimize an OR expression within the WHERE clause. 
-      sqlite3ErrorMsg(pParse, 
+  */
-          "table %s: xBestIndex returned an invalid plan", pTab->zName);
+  bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
-      /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */
-      return (double)0;
-    }
-  }
-
-  *(int*)&pIdxInfo->nOrderBy = nOrderBy;
-  return pIdxInfo->estimatedCost;
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
@@ -1726,7 +1871,7 @@ static double bestVirtualIndex(
 ** selected plan may still take advantage of the tables built-in rowid
 ** index.
 */
-static void bestIndex(
+static void bestBtreeIndex(
   Parse *pParse,              /* The parsing context */
   WhereClause *pWC,           /* The WHERE clause */
   struct SrcList_item *pSrc,  /* The FROM clause term to search */
@@ -1744,7 +1889,6 @@ static void bestIndex(
   double cost;                /* Cost of using pProbe */
   double nRow;                /* Estimated number of rows in result set */
   int i;                      /* Loop counter */
-  Bitmask maskSrc;            /* Bitmask for the pSrc table */
 
   WHERETRACE(("bestIndex: tbl=%s notReady=%llx\n", pSrc->pTab->zName,notReady));
   pProbe = pSrc->pTab->pIndex;
@@ -1860,61 +2004,7 @@ static void bestIndex(
     }
   }
 
-#ifndef SQLITE_OMIT_OR_OPTIMIZATION
+  bestOrClauseIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
-  /* Search for an OR-clause that can be used to look up the table.
-  */
-  maskSrc = getMask(pWC->pMaskSet, iCur);
-  for(i=0, pTerm=pWC->a; i<pWC->nTerm; i++, pTerm++){
-    WhereClause tempWC;
-    tempWC = *pWC;
-    if( pTerm->eOperator==WO_OR 
-        && ((pTerm->prereqAll & ~maskSrc) & notReady)==0
-        && (pTerm->u.pOrInfo->indexable & maskSrc)!=0 ){
-      WhereClause *pOrWC = &pTerm->u.pOrInfo->wc;
-      WhereTerm *pOrTerm;
-      int j;
-      int sortable = 0;
-      double rTotal = 0;
-      nRow = 0;
-      for(j=0, pOrTerm=pOrWC->a; j<pOrWC->nTerm; j++, pOrTerm++){
-        WhereCost sTermCost;
-        WHERETRACE(("... Multi-index OR testing for term %d of %d....\n", j,i));
-        if( pOrTerm->eOperator==WO_AND ){
-          WhereClause *pAndWC = &pOrTerm->u.pAndInfo->wc;
-          bestIndex(pParse, pAndWC, pSrc, notReady, 0, &sTermCost);
-        }else if( pOrTerm->leftCursor==iCur ){
-          tempWC.a = pOrTerm;
-          tempWC.nTerm = 1;
-          bestIndex(pParse, &tempWC, pSrc, notReady, 0, &sTermCost);
-        }else{
-          continue;
-        }
-        rTotal += sTermCost.rCost;
-        nRow += sTermCost.nRow;
-        if( rTotal>=pCost->rCost ) break;
-      }
-      if( pOrderBy!=0 ){
-        if( sortableByRowid(iCur, pOrderBy, pWC->pMaskSet, &rev) && !rev ){
-          sortable = 1;
-        }else{
-          rTotal += nRow*estLog(nRow);
-          WHERETRACE(("... sorting increases OR cost to %.9g\n", rTotal));
-        }
-      }
-      WHERETRACE(("... multi-index OR cost=%.9g nrow=%.9g\n",
-                  rTotal, nRow));
-      if( rTotal<pCost->rCost ){
-        pCost->rCost = rTotal;
-        pCost->nRow = nRow;
-        pCost->plan.wsFlags = WHERE_MULTI_OR;
-        pCost->plan.u.pTerm = pTerm;
-        if( sortable ){
-          pCost->plan.wsFlags = WHERE_ORDERBY|WHERE_MULTI_OR;
-        }
-      }
-    }
-  }
-#endif /* SQLITE_OMIT_OR_OPTIMIZATION */
 
   /* If the pSrc table is the right table of a LEFT JOIN then we may not
   ** use an index to satisfy IS NULL constraints on that table.  This is
@@ -2068,6 +2158,31 @@ static void bestIndex(
         pCost->rCost, pCost->plan.wsFlags, pCost->plan.nEq));
 }
 
+/*
+** Find the query plan for accessing table pSrc->pTab. Write the
+** best query plan and its cost into the WhereCost object supplied 
+** as the last parameter. This function may calculate the cost of
+** both real and virtual table scans.
+*/
+static void bestIndex(
+  Parse *pParse,              /* The parsing context */
+  WhereClause *pWC,           /* The WHERE clause */
+  struct SrcList_item *pSrc,  /* The FROM clause term to search */
+  Bitmask notReady,           /* Mask of cursors that are not available */
+  ExprList *pOrderBy,         /* The ORDER BY clause */
+  WhereCost *pCost            /* Lowest cost query plan */
+){
+  if( IsVirtual(pSrc->pTab) ){
+    sqlite3_index_info *p = 0;
+    bestVirtualIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost, &p);
+    if( p->needToFreeIdxStr ){
+      sqlite3_free(p->idxStr);
+    }
+    sqlite3DbFree(pParse->db, p);
+  }else{
+    bestBtreeIndex(pParse, pWC, pSrc, notReady, pOrderBy, pCost);
+  }
+}
 
 /*
 ** Disable a term in the WHERE clause.  Except, do not disable the term
@@ -2264,25 +2379,6 @@ static int codeAllEqualityTerms(
   return regBase;
 }
 
-/*
-** Return TRUE if the WhereClause pWC contains no terms that
-** are not virtual and which have not been coded.
-**
-** To put it another way, return TRUE if no additional WHERE clauses
-** tests are required in order to establish that the current row
-** should go to output and return FALSE if there are some terms of
-** the WHERE clause that need to be validated before outputing the row.
-*/
-static int whereRowReadyForOutput(WhereClause *pWC){
-  WhereTerm *pTerm;
-  int j;
- 
-  for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){
-    if( (pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED))==0 ) return 0;
-  }
-  return 1;
-}
-
 /*
 ** Generate code for the start of the iLevel-th loop in the WHERE clause
 ** implementation described by pWInfo.
@@ -2309,6 +2405,26 @@ static Bitmask codeOneLoopStart(
   int regRowSet;       /* Write rowids to this RowSet if non-negative */
   int codeRowSetEarly; /* True if index fully constrains the search */
 
+  /* Sometimes, this function is required to generate code to do 
+  ** something with the rowid of each row scanned. Specifically:
+  **
+  **   1) If pWInfo->regRowSet is non-zero, then the rowid must be inserted
+  **      into the RowSet object stored in register pWInfo->regRowSet.
+  **
+  **   2) If pWInfo->regRowHash is non-zero, then the rowid must be inserted
+  **      into the RowHash object stored in register pWInfo->regRowHash.
+  **
+  ** Extracting a rowid value from a VDBE cursor is not always a cheap
+  ** operation, especially if the rowid is being extracted from an index
+  ** cursor. If the rowid value is available as a by-product of the code
+  ** generated to create the top of the scan loop, then it can be reused
+  ** for either of the two purposes enumerated above without extracting
+  ** it from a cursor. The following two variables are used to communicate
+  ** the availability of the rowid value to the C-code at the end of this
+  ** function that generates the rowid-handling VDBE code.
+  */
+  int iRowidReg = 0;              /* Rowid is stored in this register */
+  int iReleaseReg = 0;            /* Temp register to free before returning */
 
   pParse = pWInfo->pParse;
   v = pParse->pVdbe;
@@ -2317,7 +2433,8 @@ static Bitmask codeOneLoopStart(
   pTabItem = &pWInfo->pTabList->a[pLevel->iFrom];
   iCur = pTabItem->iCursor;
   bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;
-  omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0;
+  omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 
+           && (wctrlFlags & WHERE_FILL_ROWHASH)==0;
   regRowSet = pWInfo->regRowSet;
   codeRowSetEarly = 0;
 
@@ -2386,11 +2503,6 @@ static Bitmask codeOneLoopStart(
     pLevel->op = OP_VNext;
     pLevel->p1 = iCur;
     pLevel->p2 = sqlite3VdbeCurrentAddr(v);
-    codeRowSetEarly = regRowSet>=0 ? whereRowReadyForOutput(pWC) : 0;
-    if( codeRowSetEarly ){
-      sqlite3VdbeAddOp2(v, OP_VRowid, iCur, iReg);
-      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, iReg);
-    }
     sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2);
   }else
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
@@ -2401,22 +2513,16 @@ static Bitmask codeOneLoopStart(
     **          we reference multiple rows using a "rowid IN (...)"
     **          construct.
     */
-    int r1;
+    iReleaseReg = sqlite3GetTempReg(pParse);
-    int rtmp = sqlite3GetTempReg(pParse);
     pTerm = findTerm(pWC, iCur, -1, notReady, WO_EQ|WO_IN, 0);
     assert( pTerm!=0 );
     assert( pTerm->pExpr!=0 );
     assert( pTerm->leftCursor==iCur );
     assert( omitTable==0 );
-    r1 = codeEqualityTerm(pParse, pTerm, pLevel, rtmp);
+    iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, iReleaseReg);
     addrNxt = pLevel->addrNxt;
-    sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, addrNxt);
+    sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt);
-    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, r1);
+    sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg);
-    codeRowSetEarly = (pWC->nTerm==1 && regRowSet>=0) ?1:0;
-    if( codeRowSetEarly ){
-      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
-    }
-    sqlite3ReleaseTempReg(pParse, rtmp);
     VdbeComment((v, "pk"));
     pLevel->op = OP_Noop;
   }else if( pLevel->plan.wsFlags & WHERE_ROWID_RANGE ){
@@ -2483,19 +2589,12 @@ static Bitmask codeOneLoopStart(
     pLevel->p1 = iCur;
     pLevel->p2 = start;
     pLevel->p5 = (pStart==0 && pEnd==0) ?1:0;
-    codeRowSetEarly = regRowSet>=0 ? whereRowReadyForOutput(pWC) : 0;
-    if( codeRowSetEarly || testOp!=OP_Noop ){
-      int r1 = sqlite3GetTempReg(pParse);
-      sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
     if( testOp!=OP_Noop ){
-        sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, r1);
+      iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse);
+      sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg);
+      sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg);
       sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC | SQLITE_JUMPIFNULL);
     }
-      if( codeRowSetEarly ){
-        sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
-      }
-      sqlite3ReleaseTempReg(pParse, r1);
-    }
   }else if( pLevel->plan.wsFlags & (WHERE_COLUMN_RANGE|WHERE_COLUMN_EQ) ){
     /* Case 3: A scan using an index.
     **
@@ -2681,20 +2780,16 @@ static Bitmask codeOneLoopStart(
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
       sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);
     }
+    sqlite3ReleaseTempReg(pParse, r1);
 
     /* Seek the table cursor, if required */
     disableTerm(pLevel, pRangeStart);
     disableTerm(pLevel, pRangeEnd);
-    codeRowSetEarly = regRowSet>=0 ? whereRowReadyForOutput(pWC) : 0;
+    if( !omitTable ){
-    if( !omitTable || codeRowSetEarly ){
+      iRowidReg = iReleaseReg = sqlite3GetTempReg(pParse);
-      sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, r1);
+      sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg);
-      if( codeRowSetEarly ){
+      sqlite3VdbeAddOp2(v, OP_Seek, iCur, iRowidReg);  /* Deferred seek */
-        sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
-      }else{
-        sqlite3VdbeAddOp2(v, OP_Seek, iCur, r1);  /* Deferred seek */
     }
-    }
-    sqlite3ReleaseTempReg(pParse, r1);
 
     /* Record the instruction used to terminate the loop. Disable 
     ** WHERE clause terms made redundant by the index range scan.
@@ -2717,66 +2812,105 @@ static Bitmask codeOneLoopStart(
     **   SELECT * FROM t1 WHERE a=5 OR b=7 OR (c=11 AND d=13)
     **
     ** In the example, there are three indexed terms connected by OR.
-    ** The top of the loop is constructed by creating a RowSet object
+    ** The top of the loop looks like this:
-    ** and populating it.  Then looping over elements of the rowset.
     **
-    **        Null 1
+    **          Null       1                # Zero the row-hash in reg 1
-    **        # fill RowSet 1 with entries where a=5 using i1
-    **        # fill Rowset 1 with entries where b=7 using i2
-    **        # fill Rowset 1 with entries where c=11 and d=13 i3 and t1
-    **     A: RowSetRead 1, B, 2
-    **        Seek       i, 2
     **
-    ** The bottom of the loop looks like this:
+    ** Then, for each indexed term, the following. The arguments to
+    ** RowHash are such that the rowid of the current row is inserted
+    ** into the row-hash. If it is already present, control skips the
+    ** Gosub opcode and jumps straight to the code generated by WhereEnd().
+    **
+    **        sqlite3WhereBegin(<term>)
+    **          RowHash                     # Insert rowid into rowhash
+    **          Gosub      2 A
+    **        sqlite3WhereEnd()
+    **
+    ** Following the above, code to terminate the loop. Label A, the target
+    ** of the Gosub above, jumps to the instruction right after the Goto.
+    **
+    **          Null       1                # Zero the row-hash in reg 1
+    **          Goto       B                # The loop is finished.
+    **
+    **       A: <loop body>                 # Return data, whatever.
+    **
+    **          Return     2                # Jump back to the Gosub
+    **
+    **       B: <after the loop>
     **
-    **        Goto       0, A
-    **     B:
     */
-    int regOrRowset;       /* Register holding the RowSet object */
+    const int f = WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FILL_ROWHASH;
-    int regNextRowid;      /* Register holding next rowid */
+
     WhereClause *pOrWc;    /* The OR-clause broken out into subterms */
-    WhereTerm *pOrTerm;    /* A single subterm within the OR-clause */
+    WhereTerm *pFinal;     /* Final subterm within the OR-clause. */
     SrcList oneTab;        /* Shortened table list */
 
+    int regReturn = ++pParse->nMem;           /* Register used with OP_Gosub */
+    int regRowhash = ++pParse->nMem;          /* Register for RowHash object */
+    int iLoopBody = sqlite3VdbeMakeLabel(v);  /* Start of loop body */
+    int iRetInit;                             /* Address of regReturn init */
+    int ii;
+   
     pTerm = pLevel->plan.u.pTerm;
     assert( pTerm!=0 );
     assert( pTerm->eOperator==WO_OR );
     assert( (pTerm->wtFlags & TERM_ORINFO)!=0 );
     pOrWc = &pTerm->u.pOrInfo->wc;
-    codeRowSetEarly = (regRowSet>=0 && pWC->nTerm==1) ?1:0;
+    pFinal = &pOrWc->a[pOrWc->nTerm-1];
 
-    if( codeRowSetEarly ){
+    /* Set up a SrcList containing just the table being scanned by this loop. */
-      regOrRowset = regRowSet;
-    }else{
-      regOrRowset = sqlite3GetTempReg(pParse);
-      sqlite3VdbeAddOp2(v, OP_Null, 0, regOrRowset);
-    }
     oneTab.nSrc = 1;
     oneTab.nAlloc = 1;
     oneTab.a[0] = *pTabItem;
-    for(j=0, pOrTerm=pOrWc->a; j<pOrWc->nTerm; j++, pOrTerm++){
+
-      WhereInfo *pSubWInfo;
+    /* Initialize the row-hash register to contain NULL. An SQL NULL is 
-      if( pOrTerm->leftCursor!=iCur && pOrTerm->eOperator!=WO_AND ) continue;
+    ** equivalent to an empty row-hash. 
-      pSubWInfo = sqlite3WhereBegin(pParse, &oneTab, pOrTerm->pExpr, 0,
+    **
-                        WHERE_FILL_ROWSET | WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE,
+    ** Also initialize regReturn to contain the address of the instruction 
-                        regOrRowset);
+    ** immediately following the OP_Return at the bottom of the loop. This
+    ** is required in a few obscure LEFT JOIN cases where control jumps
+    ** over the top of the loop into the body of it. In this case the 
+    ** correct response for the end-of-loop code (the OP_Return) is to 
+    ** fall through to the next instruction, just as an OP_Next does if
+    ** called on an uninitialized cursor.
+    */
+    sqlite3VdbeAddOp2(v, OP_Null, 0, regRowhash);
+    iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn);
+
+    /* iReleaseReg = iRowidReg = sqlite3GetTempReg(pParse); */
+    for(ii=0; ii<pOrWc->nTerm; ii++){
+      WhereTerm *pOrTerm = &pOrWc->a[ii];
+      if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){
+        WhereInfo *pSubWInfo;          /* Info for single OR-term scan */
+
+        /* Loop through table entries that match term pOrTerm. */
+        pSubWInfo = sqlite3WhereBegin(
+            pParse, &oneTab, pOrTerm->pExpr, 0, f, regRowhash);
         if( pSubWInfo ){
+          int iSet = ((ii==pOrWc->nTerm-1)?-1:ii);
+          /* The call to sqlite3WhereBegin has coded an OP_RowHash 
+          ** at instruction iRowHash. Set P2 (the jump target) of this
+          ** instruction to jump past the OP_Gosub coded below. This way,
+          ** if the rowid is already in the hash-table, the body of the
+          ** loop is not executed.
+          */
+          int iRowHash = pSubWInfo->iRowidHandler;
+          sqlite3VdbeChangeP2(v, iRowHash, sqlite3VdbeCurrentAddr(v) + 1);
+          sqlite3VdbeChangeP4(v, iRowHash, (char *)iSet, P4_INT32);
+          sqlite3VdbeAddOp2(v, OP_Gosub, regReturn, iLoopBody);
+
+          /* Finish the loop through table entries that match term pOrTerm. */
           sqlite3WhereEnd(pSubWInfo);
         }
       }
-    sqlite3VdbeResolveLabel(v, addrCont);
-    if( !codeRowSetEarly ){
-      regNextRowid = sqlite3GetTempReg(pParse);
-      addrCont = 
-         sqlite3VdbeAddOp3(v, OP_RowSetRead, regOrRowset,addrBrk,regNextRowid);
-      sqlite3VdbeAddOp2(v, OP_Seek, iCur, regNextRowid);
-      sqlite3ReleaseTempReg(pParse, regNextRowid);
-      /* sqlite3ReleaseTempReg(pParse, regOrRowset); // Preserve the RowSet */
-      pLevel->op = OP_Goto;
-      pLevel->p2 = addrCont;
-    }else{
-      pLevel->op = OP_Noop;
     }
+    sqlite3VdbeChangeP1(v, iRetInit, sqlite3VdbeCurrentAddr(v));
+    sqlite3VdbeAddOp2(v, OP_Null, 0, regRowhash);
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, pLevel->addrBrk);
+    sqlite3VdbeResolveLabel(v, iLoopBody);
+
+    pLevel->op = OP_Return;
+    pLevel->p1 = regReturn;
     disableTerm(pLevel, pTerm);
   }else
 #endif /* SQLITE_OMIT_OR_OPTIMIZATION */
@@ -2793,7 +2927,6 @@ static Bitmask codeOneLoopStart(
     pLevel->p1 = iCur;
     pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk);
     pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP;
-    codeRowSetEarly = 0;
   }
   notReady &= ~getMask(pWC->pMaskSet, iCur);
 
@@ -2839,24 +2972,35 @@ static Bitmask codeOneLoopStart(
     }
   }
 
-  /*
+  /* Do the special rowid handling now. */
-  ** If it was requested to store the results in a rowset and that has
+  if( regRowSet ){
-  ** not already been do, then do so now.
+    assert( regRowSet>0 );
+    if( iRowidReg==0 ){
+      /* The rowid was not available as a side-effect of the code 
+      ** genenerated above. So extract it from the cursor now.
       */
-  if( regRowSet>=0 && !codeRowSetEarly ){
+      assert( iReleaseReg==0 );
-    int r1 = sqlite3GetTempReg(pParse);
+      iReleaseReg = iRowidReg = sqlite3GetTempReg(pParse);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
       if( (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
-      sqlite3VdbeAddOp2(v, OP_VRowid, iCur, r1);
+        sqlite3VdbeAddOp2(v, OP_VRowid, iCur, iRowidReg);
       }else
 #endif
       {
-      sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1);
+        sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg);
       }
-    sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, r1);
-    sqlite3ReleaseTempReg(pParse, r1);
     }
     
+    pWInfo->iRowidHandler = sqlite3VdbeCurrentAddr(v);
+    if( pWInfo->wctrlFlags&WHERE_FILL_ROWSET ){
+      sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, iRowidReg);
+    }else{
+      assert( pWInfo->wctrlFlags&WHERE_FILL_ROWHASH );
+      sqlite3VdbeAddOp3(v, OP_RowHash, regRowSet, 0, iRowidReg);
+    }
+  }
+  sqlite3ReleaseTempReg(pParse, iReleaseReg);
+
   return notReady;
 }
 
@@ -2882,7 +3026,7 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){
     for(i=0; i<pWInfo->nLevel; i++){
       sqlite3_index_info *pInfo = pWInfo->a[i].pIdxInfo;
       if( pInfo ){
-        assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed );
+        /* assert( pInfo->needToFreeIdxStr==0 || db->mallocFailed ); */
         if( pInfo->needToFreeIdxStr ){
           sqlite3_free(pInfo->idxStr);
         }
@@ -3038,10 +3182,11 @@ WhereInfo *sqlite3WhereBegin(
   pWInfo->pParse = pParse;
   pWInfo->pTabList = pTabList;
   pWInfo->iBreak = sqlite3VdbeMakeLabel(v);
-  pWInfo->regRowSet = (wctrlFlags & WHERE_FILL_ROWSET) ? regRowSet : -1;
+  pWInfo->regRowSet = regRowSet;
   pWInfo->pWC = pWC = (WhereClause *)&((u8 *)pWInfo)[nByteWInfo];
   pWInfo->wctrlFlags = wctrlFlags;
   pMaskSet = (WhereMaskSet*)&pWC[1];
+  assert( regRowSet==0 || (wctrlFlags&(WHERE_FILL_ROWSET|WHERE_FILL_ROWHASH)) );
 
   /* Split the WHERE clause into separate subexpressions where each
   ** subexpression is separated by an AND operator.
@@ -3126,7 +3271,8 @@ WhereInfo *sqlite3WhereBegin(
     bestPlan.rCost = SQLITE_BIG_DBL;
     for(j=iFrom, pTabItem=&pTabList->a[j]; j<pTabList->nSrc; j++, pTabItem++){
       int doNotReorder;    /* True if this table should not be reordered */
-      WhereCost sCost;   /* Cost information from bestIndex() */
+      WhereCost sCost;     /* Cost information from best[Virtual]Index() */
+      ExprList *pOrderBy;  /* ORDER BY clause for index to optimize */
 
       doNotReorder =  (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0;
       if( once && doNotReorder ) break;
@@ -3135,34 +3281,17 @@ WhereInfo *sqlite3WhereBegin(
         if( j==iFrom ) iFrom++;
         continue;
       }
+      pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0);
+
       assert( pTabItem->pTab );
 #ifndef SQLITE_OMIT_VIRTUALTABLE
       if( IsVirtual(pTabItem->pTab) ){
-        sqlite3_index_info *pVtabIdx; /* Current virtual index */
+        sqlite3_index_info **pp = &pWInfo->a[j].pIdxInfo;
-        sqlite3_index_info **ppIdxInfo = &pWInfo->a[j].pIdxInfo;
+        bestVirtualIndex(pParse, pWC, pTabItem, notReady, pOrderBy, &sCost, pp);
-        sCost.rCost = bestVirtualIndex(pParse, pWC, pTabItem, notReady,
-                                       ppOrderBy ? *ppOrderBy : 0, i==0,
-                                       ppIdxInfo);
-        sCost.plan.wsFlags = WHERE_VIRTUALTABLE;
-        sCost.plan.u.pVtabIdx = pVtabIdx = *ppIdxInfo;
-        if( pVtabIdx && pVtabIdx->orderByConsumed ){
-          sCost.plan.wsFlags = WHERE_VIRTUALTABLE | WHERE_ORDERBY;
-        }
-        sCost.plan.nEq = 0;
-        /* (double)2 In case of SQLITE_OMIT_FLOATING_POINT... */
-        if( (SQLITE_BIG_DBL/((double)2))<sCost.rCost ){
-          /* The cost is not allowed to be larger than SQLITE_BIG_DBL (the
-          ** inital value of lowestCost in this loop. If it is, then
-          ** the (cost<lowestCost) test below will never be true.
-          */ 
-          /* (double)2 In case of SQLITE_OMIT_FLOATING_POINT... */
-          sCost.rCost = (SQLITE_BIG_DBL/((double)2));
-        }
       }else 
 #endif
       {
-        bestIndex(pParse, pWC, pTabItem, notReady,
+        bestBtreeIndex(pParse, pWC, pTabItem, notReady, pOrderBy, &sCost);
-                  (i==0 && ppOrderBy) ? *ppOrderBy : 0, &sCost);
       }
       if( once==0 || sCost.rCost<bestPlan.rCost ){
         once = 1;
@@ -3207,7 +3336,7 @@ WhereInfo *sqlite3WhereBegin(
     }
   }
   WHERETRACE(("*** Optimizer Finished ***\n"));
-  if( db->mallocFailed ){
+  if( pParse->nErr || db->mallocFailed ){
     goto whereBeginError;
   }
 

test/make-where7.tcl

@@ -103,19 +103,17 @@ for {set i 2} {$i<=$NT+1} {incr i} {
   set seen($w) 1
   set result [lsort -int [array names r]]
   puts "do_test where7-2.$i.1 \173"
-  puts "  count_steps \173"
+  puts "  count_steps_sort \173"
   puts "     SELECT a FROM t2"
   set wc [join $w "\n         OR "]
   puts "      WHERE $wc"
-  puts "      ORDER BY a"
   puts "  \175"
   puts "\175 {$result scan 0 sort 0}"
   puts "do_test where7-2.$i.2 \173"
-  puts "  count_steps \173"
+  puts "  count_steps_sort \173"
   puts "     SELECT a FROM t3"
   set wc [join $w "\n         OR "]
   puts "      WHERE $wc"
-  puts "      ORDER BY a"
   puts "  \175"
   puts "\175 {$result scan 0 sort 0}"
 }

test/rowhash.test

@@ -0,0 +1,53 @@
+# 2009 April 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements regression tests for SQLite library.  The
+# focus of this file is the code in rowhash.c.
+#
+# $Id: rowhash.test,v 1.1 2009/04/21 09:02:47 danielk1977 Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+do_test rowhash-1.1 {
+  execsql {
+    CREATE TABLE t1(id INTEGER PRIMARY KEY, a, b, c);
+    CREATE INDEX i1 ON t1(a);
+    CREATE INDEX i2 ON t1(b);
+    CREATE INDEX i3 ON t1(c);
+  }
+} {}
+
+proc do_keyset_test {name lKey} {
+  db transaction {
+    execsql { DELETE FROM t1 }
+    foreach key $lKey {
+      execsql { INSERT INTO t1 VALUES($key, 'a', 'b', 'c') }
+    }
+  }
+  do_test $name {
+    lsort -integer [execsql {
+      SELECT id FROM t1 WHERE a = 'a' OR b = 'b' OR c = 'c';
+    }]
+  } [lsort -integer $lKey]
+}
+
+do_keyset_test rowhash-2.1 {1 2 3}
+do_keyset_test rowhash-2.2 {0 1 2 3}
+do_keyset_test rowhash-2.3 {62 125 188}
+for {set i 4} {$i < 10} {incr i} {
+  for {set j 0} {$j < 5000} {incr j} {
+    lappend L [expr int(rand()*10000000000)]
+  }
+  do_keyset_test rowhash-2.$i $L
+}
+
+finish_test

test/vtabD.test

@@ -0,0 +1,70 @@
+# 2009 April 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is creating and dropping virtual tables.
+#
+# $Id: vtabD.test,v 1.1 2009/04/21 09:02:47 danielk1977 Exp $
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !vtab||!schema_pragmas { finish_test ; return }
+
+# Register the echo module
+register_echo_module [sqlite3_connection_pointer db]
+
+do_test vtabD-1.1 {
+  execsql {
+    CREATE TABLE t1(a, b);
+    CREATE INDEX i1 ON t1(a);
+    CREATE INDEX i2 ON t1(b);
+    CREATE VIRTUAL TABLE tv1 USING echo(t1);
+  }
+} {}
+do_test vtabD-1.2 {
+  execsql BEGIN
+  for {set i 0} {$i < 100000} {incr i} {
+    execsql { INSERT INTO t1 VALUES($i, $i*$i) }
+  }
+  execsql COMMIT
+} {}
+do_test vtabD-1.3 {
+  execsql { SELECT * FROM tv1 WHERE a = 1 OR b = 4 }
+} {1 1 2 4}
+do_test vtabD-1.4 {
+  execsql { SELECT * FROM tv1 WHERE a = 1 OR b = 1 }
+} {1 1}
+do_test vtabD-1.5 {
+  execsql { SELECT * FROM tv1 WHERE (a > 0 AND a < 5) OR (b > 15 AND b < 65) }
+} {1 1 2 4 3 9 4 16 5 25 6 36 7 49 8 64}
+
+do_test vtabD-1.6 {
+  execsql { SELECT * FROM tv1 WHERE a < 500 OR b = 810000 }
+} [execsql {
+  SELECT * FROM t1 WHERE a < 500
+    UNION ALL
+  SELECT * FROM t1 WHERE b = 810000 AND NOT (a < 500)
+}]
+
+do_test vtabD-1.7 {
+  execsql { SELECT * FROM tv1 WHERE a < 90000 OR b = 8100000000 }
+} [execsql {
+  SELECT * FROM t1 WHERE a < 90000
+    UNION ALL
+  SELECT * FROM t1 WHERE b = 8100000000 AND NOT (a < 90000)
+}]
+
+do_test vtabD-1.8 {
+  execsql { SELECT * FROM tv1 WHERE a = 90001 OR b = 810000 }
+} {90001 8100180001 900 810000}
+
+finish_test
+

test/where8.test

@@ -12,7 +12,7 @@
 # is testing of where.c. More specifically, the focus is the optimization
 # of WHERE clauses that feature the OR operator.
 #
-# $Id: where8.test,v 1.5 2008/12/30 16:13:05 danielk1977 Exp $
+# $Id: where8.test,v 1.6 2009/04/21 09:02:47 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -63,19 +63,19 @@ do_test where8-1.2 {
 
 do_test where8-1.3 { 
   execsql_status2 { SELECT c FROM t1 WHERE a > 8 OR b = 'two' }
-} {II IX X 0 0 6}
+} {IX X II 0 0 6}
 
 do_test where8-1.4 { 
   execsql_status2 { SELECT c FROM t1 WHERE a > 8 OR b GLOB 't*' }
-} {II III IX X 0 0 9}
+} {IX X III II 0 0 9}
 
 do_test where8-1.5 { 
   execsql_status2 { SELECT c FROM t1 WHERE a > 8 OR b GLOB 'f*' }
-} {IV V IX X 0 0 9}
+} {IX X V IV 0 0 9}
 
 do_test where8-1.6 { 
   execsql_status { SELECT c FROM t1 WHERE a = 1 OR b = 'three' ORDER BY rowid }
-} {I III 0 0}
+} {I III 0 1}
 
 do_test where8-1.7 { 
   execsql_status { SELECT c FROM t1 WHERE a = 1 OR b = 'three' ORDER BY a }
@@ -88,13 +88,13 @@ do_test where8-1.8 {
 
 do_test where8-1.9 {
   execsql_status2 { SELECT c FROM t1 WHERE a >= 9 OR b <= 'eight' }
-} {VIII IX X 0 0 6}
+} {IX X VIII 0 0 6}
 
 do_test where8-1.10 {
   execsql_status2 { 
     SELECT c FROM t1 WHERE (a >= 9 AND c != 'X') OR b <= 'eight' 
   }
-} {VIII IX 0 0 7}
+} {IX VIII 0 0 6}
 
 do_test where8-1.11 {
   execsql_status2 { 
@@ -120,7 +120,7 @@ do_test where8-1.13 {
     WHERE a = 2 OR b = 'three' OR a = 4 OR b = 'five' OR a = 6
     ORDER BY rowid
   }
-} {II III IV V VI 0 0 15}
+} {II III IV V VI 0 1 18}
 do_test where8-1.14 {
   execsql_status2 {
     SELECT c FROM t1
@@ -129,7 +129,7 @@ do_test where8-1.14 {
       b = 'seven' OR a = 8 OR b = 'nine' OR a = 10
     ORDER BY rowid
   }
-} {II III IV V VI VII VIII IX X 0 0 26}
+} {II III IV V VI VII VIII IX X 0 1 33}
 
 do_test where8-1.15 {
   execsql_status2 {
@@ -137,8 +137,7 @@ do_test where8-1.15 {
       a BETWEEN 2 AND 4 OR b = 'nine'
     ORDER BY rowid
   }
-} {II III IV IX 0 0 10}
+} {II III IV IX 0 1 12}
-
 
 
 #--------------------------------------------------------------------------
@@ -242,7 +241,7 @@ do_test where8-3.8 {
     WHERE (a = 2 OR b = 'three') AND (d = a OR e = 'sixteen')
     ORDER BY t1.rowid
   }
-} {2 2 2 4 3 3 3 4 0 0}
+} {2 2 2 4 3 3 3 4 0 1}
 
 do_test where8-3.9 {
   # The "OR c = 'IX'" term forces a linear scan.
@@ -252,13 +251,13 @@ do_test where8-3.9 {
     WHERE (a = 2 OR b = 'three' OR c = 'IX') AND (d = a OR e = 'sixteen')
     ORDER BY t1.rowid
   }
-} {2 2 2 4 3 3 3 4 9 4 9 9 9 0}
+} {2 2 2 4 3 3 3 4 9 9 9 4 9 0}
 
 do_test where8-3.10 {
   execsql_status {
     SELECT d FROM t2 WHERE e IS NULL OR e = 'four'
   }
-} {1 2 3 5 10 0 0}
+} {1 3 5 10 2 0 0}
 
 do_test where8-3.11 {
   execsql_status {

test/where8m.test

@@ -12,7 +12,7 @@
 # is testing of where.c. More specifically, the focus is the optimization
 # of WHERE clauses that feature the OR operator.
 #
-# $Id: where8m.test,v 1.1 2008/12/30 12:00:12 danielk1977 Exp $
+# $Id: where8m.test,v 1.2 2009/04/21 09:02:47 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -37,5 +37,22 @@ do_malloc_test where8m-1 -sqlprep {
     a BETWEEN 1 AND 3  AND b < 5 AND b > 2 AND c = 4;
 }
 
+do_malloc_test where8m-2 -tclprep {
+  db eval {
+    BEGIN;
+    CREATE TABLE t1(a, b, c);
+    CREATE INDEX i1 ON t1(a);
+    CREATE INDEX i2 ON t1(b);
+  }
+  for {set i 0} {$i < 1000} {incr i} {
+    set ii [expr $i*$i]
+    set iii [expr $i*$i]
+    db eval { INSERT INTO t1 VALUES($i, $ii, $iii) }
+  }
+  db eval COMMIT
+} -sqlbody {
+  SELECT count(*) FROM t1 WHERE a BETWEEN 5 AND 995 OR b BETWEEN 5 AND 900000;
+}
+
 finish_test
 

test/where9.test

@@ -11,7 +11,7 @@
 # This file implements regression tests for SQLite library.  The
 # focus of this file is testing the multi-index OR clause optimizer.
 #
-# $Id: where9.test,v 1.7 2009/02/24 10:01:52 danielk1977 Exp $
+# $Id: where9.test,v 1.8 2009/04/21 09:02:47 danielk1977 Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -166,7 +166,7 @@ do_test where9-1.2.1 {
         OR d IS NULL
     ORDER BY a
   }
-} {90 91 92 96 97 99 scan 0 sort 0}
+} {90 91 92 96 97 99 scan 0 sort 1}
 do_test where9-1.2.2 {
   count_steps {
     SELECT a FROM t1
@@ -213,7 +213,7 @@ do_test where9-1.3.1 {
         OR (b NOT NULL AND c NOT NULL AND d IS NULL)
     ORDER BY a
   }
-} {90 91 92 97 scan 0 sort 0}
+} {90 91 92 97 scan 0 sort 1}
 do_test where9-1.3.2 {
   count_steps {
     SELECT a FROM t4
@@ -248,20 +248,26 @@ do_test where9-1.4 {
      WHERE (b>=950 AND b<=1010) OR (b IS NULL AND c NOT NULL)
     ORDER BY a
   }
-} {87 88 89 90 91 scan 0 sort 0}
+} {87 88 89 90 91 scan 0 sort 1}
 do_test where9-1.5 {
+  # When this test was originally written, SQLite used a rowset object 
+  # to optimize the "ORDER BY a" clause. Now that it is using a rowhash,
+  # this is not possible. So we have to comment out one term of the OR
+  # expression in order to prevent SQLite from deeming a full-table
+  # scan to be a better strategy than using multiple indexes, which would
+  # defeat the point of the test.
   count_steps {
     SELECT a FROM t1
      WHERE a=83
         OR b=913
         OR c=28028
         OR (d>=82 AND d<83)
-        OR (e>2802 AND e<2803) 
+/*      OR (e>2802 AND e<2803)  */
         OR f='fghijklmn'
         OR g='hgfedcb'
     ORDER BY a
   }
-} {5 31 57 82 83 84 85 86 87 scan 0 sort 0}
+} {5 31 57 82 83 84 85 86 87 scan 0 sort 1}
 do_test where9-1.6 {
   count_steps {
     SELECT a FROM t1
@@ -323,7 +329,7 @@ do_test where9-2.5 {
      WHERE t1.a=80 OR t1.b=880 OR (t1.c=27027 AND round(t1.d)==80)
     ORDER BY 1
   }
-} {80 2 80 28 80 54 80 80 scan 0 sort 0}
+} {80 80 80 2 80 28 80 54 scan 0 sort 1}
 do_test where9-2.6 {
   count_steps {
     SELECT t1.a, coalesce(t2.a,9999)
@@ -331,7 +337,7 @@ do_test where9-2.6 {
      WHERE t1.a=80 OR t1.b=880 OR (t1.c=27027 AND round(t1.d)==80)
     ORDER BY 1
   }
-} {80 9999 scan 0 sort 0}
+} {80 9999 scan 0 sort 1}
 do_test where9-2.7 {
   count_steps {
     SELECT t3.x, t1.a, coalesce(t2.a,9999)
@@ -347,7 +353,7 @@ do_test where9-2.8 {
       FROM t3 JOIN
            t1 LEFT JOIN t2 ON (t1.c=t2.c AND t1.d=t2.d) OR (t1.f)=t2.f
      WHERE t1.a=t3.y OR t1.b=t3.y*11 OR (t1.c=27027 AND round(t1.d)==80)
-    ORDER BY 1, 2
+    ORDER BY 1, 2, 3
   }
 } {1 80 2 1 80 28 1 80 54 1 80 80 2 80 2 2 80 28 2 80 54 2 80 80 scan 1 sort 1}
 

tool/mksqlite3c.tcl

@@ -239,6 +239,7 @@ foreach file {
    os_unix.c
    os_win.c
 
+   rowhash.c
    bitvec.c
    pcache.c
    pcache1.c