From e318f10ce25f2c36ce3bcbca71688c1060546dd8 Mon Sep 17 00:00:00 2001 From: drh <> Date: Tue, 23 Jan 2024 13:21:40 +0000 Subject: [PATCH] If a BLOB looks like JSON when cast to text, then treat it as if it really were JSON. This replicates a long-standing bug in the JSON processing routines, and thereby avoids breaking legacy. FossilOrigin-Name: d79a37690ce7ebb91df203170d73511da44546328043c2b3fe1786b2f0087093 --- manifest | 13 +++---- manifest.uuid | 2 +- src/json.c | 45 ++++++++++++------------- test/json107.test | 86 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 31 deletions(-) create mode 100644 test/json107.test diff --git a/manifest b/manifest index ddd60a6df8..3a5eb3c4a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\s-DSQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE\scompile-time\soption\scauses\sblob\ninputs\sto\sJSON\sfunctions\sthat\sare\snot\sJSONB\sto\sbe\sprocessed\sas\sif\sthey\nwhere\stext,\simmulating\shistorical\sbugging\sbehavior\swhich\ssome\sapplications\nhave\scome\sto\srely\supon.\s\sSee\n[forum:/forumpost/012136abd5292b8d|forum\sthread\s012136abd5292b8d]\sfor\ndiscussion. -D 2024-01-22T14:16:10.354 +C If\sa\sBLOB\slooks\slike\sJSON\swhen\scast\sto\stext,\sthen\streat\sit\sas\sif\sit\sreally\nwere\sJSON.\s\sThis\sreplicates\sa\slong-standing\sbug\sin\sthe\sJSON\sprocessing\nroutines,\sand\sthereby\savoids\sbreaking\slegacy. +D 2024-01-23T13:21:40.846 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -698,7 +698,7 @@ F src/hash.h 3340ab6e1d13e725571d7cee6d3e3135f0779a7d8e76a9ce0a85971fa3953c51 F src/hwtime.h f9c2dfb84dce7acf95ce6d289e46f5f9d3d1afd328e53da8f8e9008e3b3caae6 F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 3f0a94082d978bbdd33c38fefea15346c6c6bffb70bc645a71dc0f1f87dd3276 -F src/json.c 99f2825b44327811bdf47808e57077388b83bcc6a253c1269d13fd1b3237138d +F src/json.c 14563eed12fdf0753eb8df2a72480a626335ab4467bc4dfdf15f9652f11ae1b8 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36 F src/main.c 438b95162acfa17b7d218f586f5bde11d6ae82bcf030c9611fc537556870ad6b @@ -1342,6 +1342,7 @@ F test/json103.test 53df87f83a4e5fa0c0a56eb29ff6c94055c6eb919f33316d62161a888011 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1 F test/json105.test 043838b56e68f3252a0dcf5be1689016f6f3f05056f8dcfcdc9d074f4d932988 F test/json106.test 1d46a9294e2ced35c7f87cebbcb9626d01abab04f1969d7ded7b6f6a1d9be0f2 +F test/json107.test 59054e815c8f6b67d634d44ace421cf975828fb5651c4460aa66015c8e19d562 F test/json501.test ab168a12eb6eb14d479f8c1cdae3ac062fd5a4679f17f976e96f1af518408330 F test/json502.test 84634d3dbb521d2814e43624025b760c6198456c8197bbec6c977c0236648f5b F test/jsonb01.test cace70765b36a36aec9a85a41ea65667d3bbf647d4400ddc3ac76f8fe7d94f90 @@ -2159,8 +2160,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 4dc00f577632c7b13135505007804d42a12f81a9ebd58d4c380727256b054b7e -R 59af26cba445671768ccc033868954ad +P 65572223583d43e1d61ec029641f9d3ff340e68ecfba8342c8d1b0a91a680f2a +R d924d06eebab83ae665aafd4e67b28da U drh -Z 9f8b0dd9fdc4264b889c936f6659e28d +Z 70b17e73c0bc74c502f8785eed06d844 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 25bb88a0e7..916c951931 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -65572223583d43e1d61ec029641f9d3ff340e68ecfba8342c8d1b0a91a680f2a \ No newline at end of file +d79a37690ce7ebb91df203170d73511da44546328043c2b3fe1786b2f0087093 \ No newline at end of file diff --git a/src/json.c b/src/json.c index b9a88822e0..69995bbdfb 100644 --- a/src/json.c +++ b/src/json.c @@ -3311,20 +3311,18 @@ rebuild_from_cache: } return p; } -#if defined(SQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE) - /* If the input is a BLOB that is not JSONB, fall through into trying - ** to process that BLOB as if it where text. This goes against all - ** historical documentation about how the SQLite JSON functions are - ** suppose to work. Nevertheless, many SQLite implementations prior to - ** version 3.45.0 contained a bug such that they did behave this way - ** and some applications came to depend upon this buggy behavior. The - ** SQLITE_JSON_BLOB_INPUT_BUG_COMPATIBLE compile-time option provides - ** a mechanism for those applications to continue working even after - ** the bug was fixed. See - ** https://sqlite.org/forum/forumpost/012136abd5292b8d */ -#else - goto json_pfa_malformed; -#endif + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, espeically when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ } p->zJson = (char*)sqlite3_value_text(pArg); p->nJson = sqlite3_value_bytes(pArg); @@ -4300,12 +4298,12 @@ static void jsonValidFunc( return; } case SQLITE_BLOB: { - if( (flags & 0x0c)!=0 && jsonFuncArgMightBeBinary(argv[0]) ){ + if( jsonFuncArgMightBeBinary(argv[0]) ){ if( flags & 0x04 ){ /* Superficial checking only - accomplished by the ** jsonFuncArgMightBeBinary() call above. */ res = 1; - }else{ + }else if( flags & 0x08 ){ /* Strict checking. Check by translating BLOB->TEXT->BLOB. If ** no errors occur, call that a "strict check". */ JsonParse px; @@ -4316,8 +4314,11 @@ static void jsonValidFunc( iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); res = iErr==0; } + break; } - break; + /* Fall through into interpreting the input as text. See note + ** above at tag-20240124-a. */ + /* no break */ deliberate_fall_through } default: { JsonParse px; @@ -5053,13 +5054,9 @@ static int jsonEachFilter( memset(&p->sParse, 0, sizeof(p->sParse)); p->sParse.nJPRef = 1; p->sParse.db = p->db; - if( sqlite3_value_type(argv[0])==SQLITE_BLOB ){ - if( jsonFuncArgMightBeBinary(argv[0]) ){ - p->sParse.nBlob = sqlite3_value_bytes(argv[0]); - p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); - }else{ - goto json_each_malformed_input; - } + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); }else{ p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); p->sParse.nJson = sqlite3_value_bytes(argv[0]); diff --git a/test/json107.test b/test/json107.test new file mode 100644 index 0000000000..779b557fba --- /dev/null +++ b/test/json107.test @@ -0,0 +1,86 @@ +# 2024-01-23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Legacy JSON bug: If the input is a BLOB that when cast into TEXT looks +# like valid JSON, then treat it as valid JSON. +# +# The original intent of the JSON functions was to raise an error on any +# BLOB input. That intent was clearly documented, but the code failed to +# to implement it. Subsequently, many applications began to depend on the +# incorrect behavior, especially apps that used readfile() to read JSON +# content, since readfile() returns a BLOB. So we need to support the +# bug moving forward. +# +# The tests in this fail verify that the original buggy behavior is +# preserved. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix json107 + +if {[db one {PRAGMA encoding}]!="UTF-8"} { + # These tests only work for a UTF-8 encoding. + finish_test + return +} + +do_execsql_test 1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB) ); +} 1 +do_execsql_test 1.1.1 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 1); +} 1 +do_execsql_test 1.1.2 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 2); +} 1 +do_execsql_test 1.1.4 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 4); +} 0 +do_execsql_test 1.1.8 { + SELECT json_valid( CAST('{"a":1}' AS BLOB), 8); +} 0 + +do_execsql_test 1.2.1 { + SELECT CAST('{"a":123}' AS blob) -> 'a'; +} 123 +do_execsql_test 1.2.2 { + SELECT CAST('{"a":123}' AS blob) ->> 'a'; +} 123 +do_execsql_test 1.2.3 { + SELECT json_extract(CAST('{"a":123}' AS blob), '$.a'); +} 123 +do_execsql_test 1.3 { + SELECT json_insert(CAST('{"a":123}' AS blob),'$.b',456); +} {{{"a":123,"b":456}}} +do_execsql_test 1.4 { + SELECT json_remove(CAST('{"a":123,"b":456}' AS blob),'$.a'); +} {{{"b":456}}} +do_execsql_test 1.5 { + SELECT json_set(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.6 { + SELECT json_replace(CAST('{"a":123,"b":456}' AS blob),'$.a',789); +} {{{"a":789,"b":456}}} +do_execsql_test 1.7 { + SELECT json_type(CAST('{"a":123,"b":456}' AS blob)); +} object +do_execsql_test 1.8 { + SELECT json(CAST('{"a":123,"b":456}' AS blob)); +} {{{"a":123,"b":456}}} + +ifcapable vtab { + do_execsql_test 2.1 { + SELECT key, value FROM json_tree( CAST('{"a":123,"b":456}' AS blob) ) + WHERE atom; + } {a 123 b 456} +} +finish_test