Allow expressions with subtypes to be read from indexes unless they are

being used as direct or indirect parameters to SQLITE_SUBTYPE functions.

FossilOrigin-Name: 39a56a23fec24dd713905457b6d4ed7c148f88e325a26c376f1e6daf147c69c8

manifest

@@ -1,5 +1,5 @@
-C Add\sthe\sundocumented\stest/debug\sfunction\sparseuri(),\suseful\sfor\sfuzzing.\nOnly\sappears\swhen\scompiling\swith\sSQLITE_DEBUG.
+C Allow\sexpressions\swith\ssubtypes\sto\sbe\sread\sfrom\sindexes\sunless\sthey\sare\nbeing\sused\sas\sdirect\sor\sindirect\sparameters\sto\sSQLITE_SUBTYPE\sfunctions.
-D 2024-10-07T12:48:21.105
+D 2024-10-08T10:10:42.057
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -719,10 +719,10 @@ F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a
 F src/dbpage.c db1be8adaf1f839ad733c08baeac5c22aa912f7b535865c0c061382602081360
 F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c
 F src/delete.c 03a77ba20e54f0f42ebd8eddf15411ed6bdb06a2c472ac4b6b336521bf7cea42
-F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239
+F src/expr.c a9d9f5fdfbdd3b2c94d7af1b11f181464b8a641736cf32cb92fa3c5e7ecb30df
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c 928ed2517e8732113d2b9821aa37af639688d752f4ea9ac6e0e393d713eeb76f
-F src/func.c ed6baeeb414ef18ce729793587dae8bd30f11e6aacec8675bd33727e0bcb3765
+F src/func.c 1d093b93b8f097665721e59a1c404d7db4dc591e1a777a7a1022dfbda21e108b
 F src/global.c a19e4b1ca1335f560e9560e590fc13081e21f670643367f99cb9e8f9dc7d615b
 F src/hash.c 9ee4269fb1d6632a6fecfb9479c93a1f29271bddbbaf215dd60420bcb80c7220
 F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51
@@ -767,14 +767,14 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c 3ba0ad907b7773ed642f66cea8a2c9c8edc18841aa1050b6218dbb3479e86225
 F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
-F src/resolve.c 9750a281f7ba073b4e6da2be1a6c4071f5d841a7746c5fb3f70d6d793b6675ea
+F src/resolve.c c8a5372b97b2a2e972a280676f06ddb5b74e885d3b1f5ce383f839907b57ef68
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
 F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe
 F src/shell.c.in 981efe98f98a983c1d0193d18528eb2d765207c0c82b67b610be60f17995a43e
 F src/sqlite.h.in 1def838497ad53c81486649ce79821925d1ac20a9843af317a344d507efe116e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
-F src/sqliteInt.h 989dca8b25ca11f5c52e5a457cc500042c43b0b3e5fea9a12d9020d0350722cd
+F src/sqliteInt.h ad02397dc4d22b77f9a331412d46e4c1e49459dd386fba8373fa148998e1e7d0
 F src/sqliteLimit.h 6878ab64bdeb8c24a1d762d45635e34b96da21132179023338c93f820eee6728
 F src/status.c cb11f8589a6912af2da3bb1ec509a94dd8ef27df4d4c1a97e0bcf2309ece972b
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -854,7 +854,7 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c a0d42bfdef935e1389737152394d08e59e7c48697f40a9fc2e0552cb19dc731f
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c 461d41017d900d4248a268df96d2d30506c4dcc2257f4167c4f46072003ce2cf
+F src/where.c 12fe24880901997372b88fd7ca9a21457404ad35201712c02cc57978578abb10
 F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192
 F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf
 F src/whereexpr.c 562ce89d7f1c24a54c5124576e04928600061c87d83a30e63dcbaadf20eb0653
@@ -1324,6 +1324,7 @@ F test/indexA.test 11d84f6995e6e5b9d8315953fb1b6d29772ee7c7803ee9112715e7e4dd3e4
 F test/indexedby.test f21eca4f7a6ffe14c8500a7ad6cd53166666c99e5ccd311842a28bc94a195fe0
 F test/indexexpr1.test 24fa85a12da384dd1d56f7b24e593c51a8a54b4c5e2e8bbb9e5fdf1099427faf
 F test/indexexpr2.test 1c382e81ef996d8ae8b834a74f2a9013dddf59214c32201d7c8a656d739f999a
+F test/indexexpr3.test 9d893bf440937ebcc1e59c7c9c1505c40c918346a3ddde76a69078f3c733c45d
 F test/indexfault.test 98d78a8ff1f5335628b62f886a1cb7c7dac1ef6d48fa39c51ec871c87dce9811
 F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7
 F test/insert.test 4e3f0de67aac3c5be1f4aaedbcea11638f1b5cdc9a3115be14d19aa9db7623c6
@@ -2215,9 +2216,9 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P bce52ce2a6e7f3d3d1b2807d1ea95243d9b655e557c1bb6f0b8a9a6cefb1aed6 aa9bd711cc1b0136098388976d22adc0a2fc89f50fe2273ed80ee3e4e50c98b6
+P 011fab70cb3d194b27742ebb236b05be582230567cf78e3e6cac6911de86922f f150c3c5b898975b1f83d61fa589753449a48f8a0007e8e167dbd702528197c5
-R 2751128e1e0fdc619c86fd365893c4ad
+R 4f5fad7a5b3f0800af84dc5d2cb9639f
-T +closed aa9bd711cc1b0136098388976d22adc0a2fc89f50fe2273ed80ee3e4e50c98b6
+T +closed f150c3c5b898975b1f83d61fa589753449a48f8a0007e8e167dbd702528197c5
 U drh
-Z 13ff87e9c834e664927aecce64813015
+Z d4cafc45e7a2f93bf4c10b241262518d
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-011fab70cb3d194b27742ebb236b05be582230567cf78e3e6cac6911de86922f
+39a56a23fec24dd713905457b6d4ed7c148f88e325a26c376f1e6daf147c69c8

src/expr.c

@@ -4554,6 +4554,59 @@ static int exprCodeInlineFunction(
   return target;
 }
 
+/*
+** Expression Node callback for sqlite3ExprCanReturnSubtype().
+**
+** Only a function call is able to return a subtype.  So if the node
+** is not a function call, return WRC_Prune immediately.
+**
+** A function call is able to return a subtype if it has the
+** SQLITE_RESULT_SUBTYPE property.
+**
+** Assume that every function is able to pass-through a subtype from
+** one of its argument (using sqlite3_result_value()).  Most functions
+** are not this way, but we don't have a mechanism to distinguish those
+** that are from those that are not, so assume they all work this way.
+** That means that if one of its arguments is another function and that
+** other function is able to return a subtype, then this function is
+** able to return a subtype.
+*/
+static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){
+  int n;
+  FuncDef *pDef;
+  sqlite3 *db;
+  if( pExpr->op!=TK_FUNCTION ){
+    return WRC_Prune;
+  }
+  assert( ExprUseXList(pExpr) );
+  db = pWalker->pParse->db;
+  n = ALWAYS(pExpr->x.pList) ? pExpr->x.pList->nExpr : 0;
+  pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
+  if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
+    pWalker->eCode = 1;
+    return WRC_Prune;
+  }
+  return WRC_Continue;
+}
+
+/*
+** Return TRUE if expression pExpr is able to return a subtype.
+**
+** A TRUE return does not guarantee that a subtype will be returned.
+** It only indicates that a subtype return is possible.  False positives
+** are acceptable as they only disable an optimization.  False negatives,
+** on the other hand, can lead to incorrect answers.
+*/
+static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){
+  Walker w;
+  memset(&w, 0, sizeof(w));
+  w.pParse = pParse;
+  w.xExprCallback = exprNodeCanReturnSubtype;
+  sqlite3WalkExpr(&w, pExpr);
+  return w.eCode;
+}
+
+
 /*
 ** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr.
 ** If it is, then resolve the expression by reading from the index and
@@ -4586,6 +4639,17 @@ static SQLITE_NOINLINE int sqlite3IndexedExprLookup(
       continue;
     }
 
+
+    /* Functions that might set a subtype should not be replaced by the
+    ** value taken from an expression index if they are themselves an
+    ** argument to another scalar function or aggregate. 
+    ** https://sqlite.org/forum/forumpost/68d284c86b082c3e */
+    if( ExprHasProperty(pExpr, EP_SubtArg)
+     && sqlite3ExprCanReturnSubtype(pParse, pExpr) 
+    ){
+      continue;
+    }
+
     v = pParse->pVdbe;
     assert( v!=0 );
     if( p->bMaybeNullRow ){

src/func.c

@@ -2706,7 +2706,8 @@ void sqlite3RegisterBuiltinFunctions(void){
     WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0,
                                  SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER ),
     FUNCTION2(typeof,            1, 0, 0, typeofFunc,  SQLITE_FUNC_TYPEOF),
-    FUNCTION2(subtype,           1, 0, 0, subtypeFunc, SQLITE_FUNC_TYPEOF),
+    FUNCTION2(subtype,           1, 0, 0, subtypeFunc,
+                                           SQLITE_FUNC_TYPEOF|SQLITE_SUBTYPE),
     FUNCTION2(length,            1, 0, 0, lengthFunc,  SQLITE_FUNC_LENGTH),
     FUNCTION2(octet_length,      1, 0, 0, bytelengthFunc,SQLITE_FUNC_BYTELEN),
     FUNCTION(instr,              2, 0, 0, instrFunc        ),
@@ -2812,7 +2813,7 @@ void sqlite3RegisterBuiltinFunctions(void){
     MFUNCTION(sqrt,              1, sqrt,      math1Func   ),
     MFUNCTION(radians,           1, degToRad,  math1Func   ),
     MFUNCTION(degrees,           1, radToDeg,  math1Func   ),
-    FUNCTION(pi,                 0, 0, 0,      piFunc      ),
+    MFUNCTION(pi,                0, 0,         piFunc      ),
 #endif /* SQLITE_ENABLE_MATH_FUNCTIONS */
     FUNCTION(sign,               1, 0, 0,      signFunc    ),
     INLINE_FUNC(coalesce,       -1, INLINEFUNC_coalesce, 0 ),

src/resolve.c

@@ -1183,6 +1183,24 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
           }
         }
 #endif
+
+        /* If the function may call sqlite3_value_subtype(), then set the 
+        ** EP_SubtArg flag on all of its argument expressions. This prevents
+        ** where.c from replacing the expression with a value read from an
+        ** index on the same expression, which will not have the correct 
+        ** subtype. Also set the flag if the function expression itself is
+        ** an EP_SubtArg expression. In this case subtypes are required as 
+        ** the function may return a value with a subtype back to its 
+        ** caller using sqlite3_result_value().  */
+        if( (pDef->funcFlags & SQLITE_SUBTYPE) 
+         || ExprHasProperty(pExpr, EP_SubtArg) 
+        ){
+          int ii;
+          for(ii=0; ii<n; ii++){
+            ExprSetProperty(pList->a[ii].pExpr, EP_SubtArg);
+          }
+        }
+
         if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){
           /* For the purposes of the EP_ConstFunc flag, date and time
           ** functions and other functions that change slowly are considered

src/sqliteInt.h

@@ -3104,7 +3104,7 @@ struct Expr {
 #define EP_IsTrue   0x10000000 /* Always has boolean value of TRUE */
 #define EP_IsFalse  0x20000000 /* Always has boolean value of FALSE */
 #define EP_FromDDL  0x40000000 /* Originates from sqlite_schema */
-               /*   0x80000000 // Available */
+#define EP_SubtArg  0x80000000 /* Is argument to SQLITE_SUBTYPE function */
 
 /* The EP_Propagate mask is a set of properties that automatically propagate
 ** upwards into parent nodes.

src/where.c

@@ -6302,58 +6302,6 @@ static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful(
   }
 }
 
-/*
-** Expression Node callback for sqlite3ExprCanReturnSubtype().
-**
-** Only a function call is able to return a subtype.  So if the node
-** is not a function call, return WRC_Prune immediately.
-**
-** A function call is able to return a subtype if it has the
-** SQLITE_RESULT_SUBTYPE property.
-**
-** Assume that every function is able to pass-through a subtype from
-** one of its argument (using sqlite3_result_value()).  Most functions
-** are not this way, but we don't have a mechanism to distinguish those
-** that are from those that are not, so assume they all work this way.
-** That means that if one of its arguments is another function and that
-** other function is able to return a subtype, then this function is
-** able to return a subtype.
-*/
-static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){
-  int n;
-  FuncDef *pDef;
-  sqlite3 *db;
-  if( pExpr->op!=TK_FUNCTION ){
-    return WRC_Prune;
-  }
-  assert( ExprUseXList(pExpr) );
-  db = pWalker->pParse->db;
-  n = pExpr->x.pList ? pExpr->x.pList->nExpr : 0;
-  pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0);
-  if( pDef==0 || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){
-    pWalker->eCode = 1;
-    return WRC_Prune;
-  }
-  return WRC_Continue;
-}
-
-/*
-** Return TRUE if expression pExpr is able to return a subtype.
-**
-** A TRUE return does not guarantee that a subtype will be returned.
-** It only indicates that a subtype return is possible.  False positives
-** are acceptable as they only disable an optimization.  False negatives,
-** on the other hand, can lead to incorrect answers.
-*/
-static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){
-  Walker w;
-  memset(&w, 0, sizeof(w));
-  w.pParse = pParse;
-  w.xExprCallback = exprNodeCanReturnSubtype;
-  sqlite3WalkExpr(&w, pExpr);
-  return w.eCode;
-}
-
 /*
 ** The index pIdx is used by a query and contains one or more expressions.
 ** In other words pIdx is an index on an expression.  iIdxCur is the cursor
@@ -6387,12 +6335,6 @@ static SQLITE_NOINLINE void whereAddIndexedExpr(
       continue;
     }
     if( sqlite3ExprIsConstant(0,pExpr) ) continue;
-    if( pExpr->op==TK_FUNCTION && sqlite3ExprCanReturnSubtype(pParse,pExpr) ){
-      /* Functions that might set a subtype should not be replaced by the
-      ** value taken from an expression index since the index omits the
-      ** subtype.  https://sqlite.org/forum/forumpost/68d284c86b082c3e */
-      continue;
-    }
     p = sqlite3DbMallocRaw(pParse->db,  sizeof(IndexedExpr));
     if( p==0 ) break;
     p->pIENext = pParse->pIdxEpr;

test/indexexpr3.test

@@ -0,0 +1,83 @@
+# 2024-10-05
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing indexes on expressions.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix indexexpr3
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a, j);
+  INSERT INTO t1 VALUES(1, '{x:"one"}');
+  INSERT INTO t1 VALUES(2, '{x:"two"}');
+  INSERT INTO t1 VALUES(3, '{x:"three"}');
+
+  CREATE INDEX i1 ON t1( json_extract(j, '$.x') );
+  CREATE INDEX i2 ON t1( a, json_extract(j, '$.x') );
+}
+
+proc do_hasfunction_test {tn sql res} {
+  set nFunction 0
+  db eval "EXPLAIN $sql" x {
+    if {$x(opcode)=="Function"} {
+      incr nFunction 
+    }
+  }
+
+  do_execsql_test $tn "
+    SELECT $nFunction;
+    $sql
+  " $res
+}
+
+do_hasfunction_test 1.1 {
+  SELECT json_extract(j, '$.x') FROM t1 ORDER BY 1;
+} {
+  0 one three two
+}
+
+do_hasfunction_test 1.2 {
+  SELECT json_extract(j, '$.x') FROM t1 WHERE a=2
+} {
+  0 two
+}
+
+do_hasfunction_test 1.3 {
+  SELECT coalesce(json_extract(j, '$.x'), 'five') FROM t1 WHERE a=2
+} {
+  0 two
+}
+
+do_hasfunction_test 1.4 {
+  SELECT json_extract(j, '$.x') || '.two' FROM t1 WHERE a=2
+} {
+  0 two.two
+}
+
+do_hasfunction_test 1.5 {
+  SELECT json_insert( '{}', '$.y', json_extract(j, '$.x') ) FROM t1 WHERE a=2
+} {
+  2 {{"y":"two"}}
+}
+
+do_hasfunction_test 1.6 {
+  SELECT json_insert( '{}', '$.y', coalesce( json_extract(j, '$.x'), 'five' ) )
+  FROM t1 WHERE a=2
+} {
+  2 {{"y":"two"}}
+}
+
+
+finish_test
+