Merge all changes from trunk.

FossilOrigin-Name: f2e15b1974e55373b5819e3b2326653f890f409f

ext/fts3/fts3Int.h

@@ -39,6 +39,18 @@ extern const sqlite3_api_routines *sqlite3_api;
 #include "fts3_tokenizer.h"
 #include "fts3_hash.h"
 
+/*
+** This constant determines the maximum depth of an FTS expression tree
+** that the library will create and use. FTS uses recursion to perform 
+** various operations on the query tree, so the disadvantage of a large
+** limit is that it may allow very large queries to use large amounts
+** of stack space (perhaps causing a stack overflow).
+*/
+#ifndef SQLITE_FTS3_MAX_EXPR_DEPTH
+# define SQLITE_FTS3_MAX_EXPR_DEPTH 12
+#endif
+
+
 /*
 ** This constant controls how often segments are merged. Once there are
 ** FTS3_MERGE_COUNT segments of level N, they are merged into a single

ext/fts3/fts3_expr.c

@@ -1000,17 +1000,16 @@ int sqlite3Fts3ExprParse(
   Fts3Expr **ppExpr,                  /* OUT: Parsed query structure */
   char **pzErr                        /* OUT: Error message (sqlite3_malloc) */
 ){
-  static const int MAX_EXPR_DEPTH = 12;
   int rc = fts3ExprParseUnbalanced(
       pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr
   );
   
   /* Rebalance the expression. And check that its depth does not exceed
-  ** MAX_EXPR_DEPTH.  */
+  ** SQLITE_FTS3_MAX_EXPR_DEPTH.  */
   if( rc==SQLITE_OK && *ppExpr ){
-    rc = fts3ExprBalance(ppExpr, MAX_EXPR_DEPTH);
+    rc = fts3ExprBalance(ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
     if( rc==SQLITE_OK ){
-      rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH);
+      rc = fts3ExprCheckDepth(*ppExpr, SQLITE_FTS3_MAX_EXPR_DEPTH);
     }
   }
 
@@ -1019,7 +1018,8 @@ int sqlite3Fts3ExprParse(
     *ppExpr = 0;
     if( rc==SQLITE_TOOBIG ){
       *pzErr = sqlite3_mprintf(
-          "FTS expression tree is too large (maximum depth %d)", MAX_EXPR_DEPTH
+          "FTS expression tree is too large (maximum depth %d)", 
+          SQLITE_FTS3_MAX_EXPR_DEPTH
       );
       rc = SQLITE_ERROR;
     }else if( rc==SQLITE_ERROR ){

ext/fts3/fts3_unicode2.c

@@ -101,28 +101,27 @@ int sqlite3FtsUnicodeIsalnum(int c){
     0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803,
     0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07,
     0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02,
-    0x037FFC02, 0x03E3FC01, 0x03EC7801, 0x03ECA401, 0x03EEC810,
+    0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802,
-    0x03F4F802, 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023,
+    0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013,
-    0x03F95013, 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807,
+    0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06,
-    0x03FCEC06, 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405,
+    0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003,
-    0x04040003, 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E,
+    0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01,
-    0x040E7C01, 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01,
+    0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403,
-    0x04280403, 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01,
+    0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009,
-    0x04294009, 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016,
+    0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003,
-    0x04420003, 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004,
+    0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003,
-    0x04460003, 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004,
+    0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E,
-    0x05BD442E, 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5,
+    0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046,
-    0x07480046, 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01,
+    0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401,
-    0x075C5401, 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401,
+    0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401,
-    0x075EA401, 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064,
+    0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F,
-    0x07C2800F, 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F,
+    0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C,
-    0x07C4C03C, 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009,
+    0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002,
-    0x07C94002, 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014,
+    0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025,
-    0x07CE8025, 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001,
+    0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6,
-    0x07D108B6, 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018,
+    0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46,
-    0x07D7EC46, 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401,
+    0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060,
-    0x38008060, 0x380400F0, 0x3C000001, 0x3FFFF401, 0x40000001,
+    0x380400F0,
-    0x43FFF401,
   };
   static const unsigned int aAscii[4] = {
     0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001,

ext/fts3/unicode/mkunicode.tcl

@@ -239,7 +239,10 @@ proc an_load_unicodedata_text {zName} {
     foreach $lField $fields {}
 
     set iCode [expr "0x$code"]
-    set bAlnum [expr {[lsearch {L N} [string range $general_category 0 0]]>=0}]
+    set bAlnum [expr {
+         [lsearch {L N} [string range $general_category 0 0]] >= 0
+      || $general_category=="Co"
+    }]
 
     if { !$bAlnum } { lappend lRet $iCode }
   }
@@ -360,7 +363,7 @@ proc print_isalnum {zFunc lRange} {
     }
     assert( aEntry[0]<key );
     assert( key>=aEntry[iRes] );
-    return (c >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
+    return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF)));
   }
   return 1;}
   puts "\}"
@@ -729,7 +732,7 @@ proc print_fileheader {} {
 */
   }]
   puts ""
-  puts "#if !defined(SQLITE_DISABLE_FTS3_UNICODE)"
+  puts "#if defined(SQLITE_ENABLE_FTS4_UNICODE61)"
   puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)"
   puts ""
   puts "#include <assert.h>"
@@ -805,4 +808,4 @@ if {$::generate_test_code} {
 }
 
 puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */"
-puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */"
+puts "#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */"

manifest

@@ -1,5 +1,5 @@
-C "make\stest"\snow\spassing.
+C Merge\sall\schanges\sfrom\strunk.
-D 2013-06-12T17:08:06.018
+D 2013-06-12T17:17:45.907
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -80,9 +80,9 @@ F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
 F ext/fts3/fts3.c 760fecd100598dc22f9733d472b7cf2e9375c620
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
-F ext/fts3/fts3Int.h 0b167bed9e63151635620a4f639bc62ac6012cba
+F ext/fts3/fts3Int.h cb4df04cf886d9920a71df9e8faaa5aae2fa48c6
 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd
-F ext/fts3/fts3_expr.c 193d6fc156d744ab548a2ed06c31869e54dac739
+F ext/fts3/fts3_expr.c f8eb1046063ba342c7114eba175cabb31c4a64e7
 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914
 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf
 F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5
@@ -95,14 +95,14 @@ F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16
 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9
-F ext/fts3/fts3_unicode2.c a863f05f758af36777dffc2facc898bc73fec896
+F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d
 F ext/fts3/fts3_write.c 6a1fc0e922e76b68e594bf7bc33bac72af9dc47b
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675
+F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368
 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43
 F ext/icu/icu.c 7538f98eab2854cf17fa5f7797bffa6c76e3863b
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -165,7 +165,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c 7fba377c29573adfc6091832e27ee1fcbefb51d0
 F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca
 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2
-F src/build.c 838cbdcbf18de2fd5723ad5864a78cc806f6c75b
+F src/build.c eadda12ec0cae14dcdc13e97fd9a3fd2f02e9e8f
 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267
@@ -289,7 +289,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73
-F src/where.c 58522e12fcb559ac138d1cf51348b6861345968e
+F src/where.c 97f5d9c3a9dfa802d91f7723cd1d782a3ce78e7a
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -365,7 +365,7 @@ F test/check.test 2eb93611139a7dfaed3be80067c7dc5ceb5fb287
 F test/close.test 340bd24cc58b16c6bc01967402755027c37eb815
 F test/closure01.test dbb28f1ea9eeaf0a53ec5bc0fed352e479def8c7
 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91
-F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce
+F test/collate1.test b709989e6e6ff6e1d2bd64231c2c1d8146846c9e
 F test/collate2.test 285cef1b58ce2b3cf074a386f763ce753c81c97f
 F test/collate3.test 79558a286362cb9ed603c6fa543f1cda7f563f0f
 F test/collate4.test 031f7265c13308b724ba3c49f41cc04612bd92b1
@@ -551,7 +551,7 @@ F test/fts4merge.test c424309743fdd203f8e56a1f1cd7872cd66cc0ee
 F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891
 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7
 F test/fts4merge4.test c19c85ca1faa7b6d536832b49c12e1867235f584
-F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40
+F test/fts4unicode.test c8ac44217bf6c17812b03eaafa6c06995ad304c2
 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d
 F test/func.test b0fc34fdc36897769651975a2b0a606312753643
 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@@ -1049,7 +1049,7 @@ F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31
 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361
 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688
 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd
-F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x
+F tool/build-all-msvc.bat c55f64ca200308fb5fa5c1ee751ea95a13977b5a x
 F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367
 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2
 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b
@@ -1095,7 +1095,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c 4d0393bdbe7230adb712e925863744dd2b7ffc5b
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 25e2cde105a19293bdb9c001b48624e5d7f8c4e5
+P addd7f466d6ff55f82d907286650c26b06e9397b 24fc9d4438a5615dd20af5419456166df83a72ea
-R 74299348f4ebeefec70a5406f9e251d7
+R 400236daf5743c1455f090bd73116cba
 U drh
-Z ad22135d0c0bd81f6331e3b3327b23e1
+Z eafccaa2bb28a08ba2c2409a2846f8df

manifest.uuid

@@ -1 +1 @@
-addd7f466d6ff55f82d907286650c26b06e9397b
+f2e15b1974e55373b5819e3b2326653f890f409f

src/build.c

@@ -1276,6 +1276,7 @@ void sqlite3AddCollateType(Parse *pParse, Token *pToken){
 
   if( sqlite3LocateCollSeq(pParse, zColl) ){
     Index *pIdx;
+    sqlite3DbFree(db, p->aCol[i].zColl);
     p->aCol[i].zColl = zColl;
   
     /* If the column is declared as "<name> PRIMARY KEY COLLATE <type>",

src/where.c

@@ -77,7 +77,7 @@ struct WhereLevel {
   u8 iFrom;             /* Which entry in the FROM clause */
   u8 op, p5;            /* Opcode and P5 of the opcode that ends the loop */
   int p1, p2;           /* Operands of the opcode used to ends the loop */
-  union {               /* Information that depends on plan.wsFlags */
+  union {               /* Information that depends on pWLoop->wsFlags */
     struct {
       int nIn;              /* Number of entries in aInLoop[] */
       struct InLoop {
@@ -85,7 +85,7 @@ struct WhereLevel {
         int addrInTop;         /* Top of the IN loop */
         u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
       } *aInLoop;           /* Information about each nested IN operator */
-    } in;                 /* Used when plan.wsFlags&WHERE_IN_ABLE */
+    } in;                 /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */
     Index *pCovidx;       /* Possible covering index for WHERE_MULTI_OR */
   } u;
   struct WhereLoop *pWLoop;  /* The selected WhereLoop object */
@@ -2814,8 +2814,6 @@ static int codeEqualityTerm(
       && pLoop->u.btree.pIndex->aSortOrder[iEq]
     ){
       testcase( iEq==0 );
-      testcase( iEq==pLevel->plan.u.pIdx->nColumn-1 );
-      testcase( iEq>0 && iEq+1<pLevel->plan.u.pIdx->nColumn );
       testcase( bRev );
       bRev = !bRev;
     }
@@ -3555,8 +3553,8 @@ static Bitmask codeOneLoopStart(
     ** If it is, jump to the next iteration of the loop.
     */
     r1 = sqlite3GetTempReg(pParse);
-    testcase( pLevel->plan.wsFlags & WHERE_BTM_LIMIT );
+    testcase( pLoop->wsFlags & WHERE_BTM_LIMIT );
-    testcase( pLevel->plan.wsFlags & WHERE_TOP_LIMIT );
+    testcase( pLoop->wsFlags & WHERE_TOP_LIMIT );
     if( (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 ){
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, nEq, r1);
       sqlite3VdbeAddOp2(v, OP_IsNull, r1, addrCont);

test/collate1.test

@@ -305,4 +305,33 @@ do_test collate1-4.5 {
   }
 } {}
 
+# A problem reported on the mailing list:  A CREATE TABLE statement
+# is allowed to have two or more COLLATE clauses on the same column.
+# That probably ought to be an error, but we allow it for backwards
+# compatibility.  Just make sure it works and doesn't leak memory.
+#
+do_test collate1-5.1 {
+  execsql {
+    CREATE TABLE c5(
+      id INTEGER PRIMARY KEY,
+      a TEXT COLLATE binary COLLATE nocase COLLATE rtrim,
+      b TEXT COLLATE nocase COLLATE binary,
+      c TEXT COLLATE rtrim COLLATE binary COLLATE rtrim COLLATE nocase
+    );
+    INSERT INTO c5 VALUES(1, 'abc','abc','abc');
+    INSERT INTO c5 VALUES(2, 'abc   ','ABC','ABC');
+    SELECT id FROM c5 WHERE a='abc' ORDER BY id;
+  }
+} {1 2}
+do_test collate1-5.2 {
+  execsql {
+    SELECT id FROM c5 WHERE b='abc' ORDER BY id;
+  }
+} {1}
+do_test collate1-5.3 {
+  execsql {
+    SELECT id FROM c5 WHERE c='abc' ORDER BY id;
+  }
+} {1 2}
+
 finish_test

test/fts4unicode.test

@@ -384,5 +384,23 @@ foreach T $tokenizers {
   do_isspace_test 6.$T.19 $T   {8287 12288}
 }
 
+#-------------------------------------------------------------------------
+# Test that the private use ranges are treated as alphanumeric.
+#
+breakpoint
+foreach {tn1 c} {
+  1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
+} {
+  foreach {tn2 config res} {
+    1 ""             "0 hello*world hello*world"
+    2 "separators=*" "0 hello hello 1 world world"
+  } {
+    set config [string map [list * $c] $config]
+    set input  [string map [list * $c] "hello*world"]
+    set output [string map [list * $c] $res]
+    do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
+  }
+}
+
 
 finish_test

tool/build-all-msvc.bat

@@ -374,7 +374,7 @@ FOR %%P IN (%PLATFORMS%) DO (
           REM       need to remove the build output for the files we are
           REM       specifically wanting to build for each platform.
           REM
-          %__ECHO% DEL /Q sqlite3.dll sqlite3.lib sqlite3.pdb
+          %__ECHO% DEL /Q *.lo sqlite3.dll sqlite3.lib sqlite3.pdb
         )
 
         REM