mirrors/postgres
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix some minor spec-compliance issues in jsonpath lexer.

Browse Source 
Although the SQL/JSON tech report makes reference to ECMAScript which
allows both single- and double-quoted strings, all the rest of the
report speaks only of double-quoted string literals in jsonpaths.
That's more compatible with JSON itself; moreover single-quoted strings
are hard to use inside a jsonpath that is itself a single-quoted SQL
literal.  So guess that the intent is to allow only double-quoted
literals, and remove lexer support for single-quoted literals.
It'll be less painful to add this again later if we're wrong, than to
remove a shipped feature.

Also, adjust the lexer so that unrecognized backslash sequences are
treated as just meaning the escaped character, not as errors.  This
change has much better support in the standards, as JSON, JavaScript
and ECMAScript all make it plain that that's what's supposed to
happen.

Back-patch to v12.

Discussion: https://postgr.es/m/CAPpHfdvDci4iqNF9fhRkTqhe-5_8HmzeLt56drH%2B_Rv2rNRqfg@mail.gmail.com
This commit is contained in:
Tom Lane 2019-09-20 14:22:58 -04:00
parent 96b6c82c9d
commit e56cad84d5
6 changed files with 38 additions and 235 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
src/backend/utils/adt/jsonpath_scan.l
View File

@ -59,25 +59,24 @@ fprintf_to_ereport(const char *fmt, const char *msg)
%option noyyfree %option noyyfree
/* /*
* We use exclusive states for quoted, signle-quoted and non-quoted strings, * We use exclusive states for quoted and non-quoted strings,
* quoted variable names and C-tyle comments. * quoted variable names and C-style comments.
* Exclusive states: * Exclusive states:
* <xq> - quoted strings * <xq> - quoted strings
* <xnq> - non-quoted strings * <xnq> - non-quoted strings
* <xvq> - quoted variable names * <xvq> - quoted variable names
* <xsq> - single-quoted strings
* <xc> - C-style comment * <xc> - C-style comment
*/ */
%x xq %x xq
%x xnq %x xnq
%x xvq %x xvq
%x xsq
%x xc %x xc
special [\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/] special [\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/]
any [^\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/\\\"\' \t\n\r\f]
blank [ \t\n\r\f] blank [ \t\n\r\f]
/* "other" means anything that's not special, blank, or '\' or '"' */
other [^\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/\\\" \t\n\r\f]
digit [0-9] digit [0-9]
integer (0|[1-9]{digit}*) integer (0|[1-9]{digit}*)
@ -95,7 +94,7 @@ hex_fail \\x{hex_dig}{0,1}
%% %%
<xnq>{any}+ { <xnq>{other}+ {
addstring(false, yytext, yyleng); addstring(false, yytext, yyleng);
} }
@ -105,13 +104,12 @@ hex_fail \\x{hex_dig}{0,1}
return checkKeyword(); return checkKeyword();
} }
<xnq>\/\* { <xnq>\/\* {
yylval->str = scanstring; yylval->str = scanstring;
BEGIN xc; BEGIN xc;
} }
<xnq>({special}|\"|\') { <xnq>({special}|\") {
yylval->str = scanstring; yylval->str = scanstring;
yyless(0); yyless(0);
BEGIN INITIAL; BEGIN INITIAL;
@ -124,39 +122,37 @@ hex_fail \\x{hex_dig}{0,1}
return checkKeyword(); return checkKeyword();
} }
<xnq,xq,xvq,xsq>\\[\"\'\\] { addchar(false, yytext[1]); } <xnq,xq,xvq>\\b { addchar(false, '\b'); }
<xnq,xq,xvq,xsq>\\b { addchar(false, '\b'); } <xnq,xq,xvq>\\f { addchar(false, '\f'); }
<xnq,xq,xvq,xsq>\\f { addchar(false, '\f'); } <xnq,xq,xvq>\\n { addchar(false, '\n'); }
<xnq,xq,xvq,xsq>\\n { addchar(false, '\n'); } <xnq,xq,xvq>\\r { addchar(false, '\r'); }
<xnq,xq,xvq,xsq>\\r { addchar(false, '\r'); } <xnq,xq,xvq>\\t { addchar(false, '\t'); }
<xnq,xq,xvq,xsq>\\t { addchar(false, '\t'); } <xnq,xq,xvq>\\v { addchar(false, '\v'); }
<xnq,xq,xvq,xsq>\\v { addchar(false, '\v'); } <xnq,xq,xvq>{unicode}+ { parseUnicode(yytext, yyleng); }
<xnq,xq,xvq,xsq>{unicode}+ { parseUnicode(yytext, yyleng); } <xnq,xq,xvq>{hex_char} { parseHexChar(yytext); }
<xnq,xq,xvq,xsq>{hex_char} { parseHexChar(yytext); } <xnq,xq,xvq>{unicode}*{unicodefail} { yyerror(NULL, "invalid unicode sequence"); }
<xnq,xq,xvq,xsq>{unicode}*{unicodefail} { yyerror(NULL, "invalid unicode sequence"); } <xnq,xq,xvq>{hex_fail} { yyerror(NULL, "invalid hex character sequence"); }
<xnq,xq,xvq,xsq>{hex_fail} { yyerror(NULL, "invalid hex character sequence"); } <xnq,xq,xvq>{unicode}+\\ {
<xnq,xq,xvq,xsq>{unicode}+\\ {
/* throw back the \\, and treat as unicode */ /* throw back the \\, and treat as unicode */
yyless(yyleng - 1); yyless(yyleng - 1);
parseUnicode(yytext, yyleng); parseUnicode(yytext, yyleng);
} }
<xnq,xq,xvq,xsq>\\. { yyerror(NULL, "escape sequence is invalid"); } <xnq,xq,xvq>\\. { addchar(false, yytext[1]); }
<xnq,xq,xvq,xsq>\\ { yyerror(NULL, "unexpected end after backslash"); } <xnq,xq,xvq>\\ { yyerror(NULL, "unexpected end after backslash"); }
<xq,xvq,xsq><<EOF>> { yyerror(NULL, "unexpected end of quoted string"); } <xq,xvq><<EOF>> { yyerror(NULL, "unexpected end of quoted string"); }
<xq>\" { <xq>\" {
yylval->str = scanstring; yylval->str = scanstring;
@ -170,16 +166,8 @@ hex_fail \\x{hex_dig}{0,1}
return VARIABLE_P; return VARIABLE_P;
} }
<xsq>\' {
yylval->str = scanstring;
BEGIN INITIAL;
return STRING_P;
}
<xq,xvq>[^\\\"]+ { addstring(false, yytext, yyleng); } <xq,xvq>[^\\\"]+ { addstring(false, yytext, yyleng); }
<xsq>[^\\\']+ { addstring(false, yytext, yyleng); }
<xc>\*\/ { BEGIN INITIAL; } <xc>\*\/ { BEGIN INITIAL; }
<xc>[^\*]+ { } <xc>[^\*]+ { }
@ -210,7 +198,7 @@ hex_fail \\x{hex_dig}{0,1}
\> { return GREATER_P; } \> { return GREATER_P; }
\${any}+ { \${other}+ {
addstring(true, yytext + 1, yyleng - 1); addstring(true, yytext + 1, yyleng - 1);
addchar(false, '\0'); addchar(false, '\0');
yylval->str = scanstring; yylval->str = scanstring;
@ -263,27 +251,22 @@ hex_fail \\x{hex_dig}{0,1}
({realfail1}|{realfail2}) { yyerror(NULL, "invalid floating point number"); } ({realfail1}|{realfail2}) { yyerror(NULL, "invalid floating point number"); }
{any}+ {
addstring(true, yytext, yyleng);
BEGIN xnq;
}
\" { \" {
addchar(true, '\0'); addchar(true, '\0');
BEGIN xq; BEGIN xq;
} }
\' {
addchar(true, '\0');
BEGIN xsq;
}
\\ { \\ {
yyless(0); yyless(0);
addchar(true, '\0'); addchar(true, '\0');
BEGIN xnq; BEGIN xnq;
} }
{other}+ {
addstring(true, yytext, yyleng);
BEGIN xnq;
}
<<EOF>> { yyterminate(); } <<EOF>> { yyterminate(); }
%% %%

18
src/test/regress/expected/jsonpath.out
View File

@ -171,30 +171,24 @@ select '"\b\f\r\n\t\v\"\''\\"'::jsonpath;
"\b\f\r\n\t\u000b\"'\\" "\b\f\r\n\t\u000b\"'\\"
(1 row) (1 row)
select '''\b\f\r\n\t\v\"\''\\'''::jsonpath;
jsonpath
-------------------------
"\b\f\r\n\t\u000b\"'\\"
(1 row)
select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath; select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath;
jsonpath jsonpath
---------- ----------
"PgSQL" "PgSQL"
(1 row) (1 row)
select '''\x50\u0067\u{53}\u{051}\u{00004C}'''::jsonpath;
jsonpath
----------
"PgSQL"
(1 row)
select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath; select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath;
jsonpath jsonpath
--------------------- ---------------------
$."fooPgSQL\t\"bar" $."fooPgSQL\t\"bar"
(1 row) (1 row)
select '"\z"'::jsonpath; -- unrecognized escape is just the literal char
jsonpath
----------
"z"
(1 row)
select '$.g ? ($.a == 1)'::jsonpath; select '$.g ? ($.a == 1)'::jsonpath;
jsonpath jsonpath
-------------------- --------------------

78
src/test/regress/expected/jsonpath_encoding.out
View File

@ -81,84 +81,6 @@ select '"null \\u0000 escape"'::jsonpath as not_an_escape;
"null \\u0000 escape" "null \\u0000 escape"
(1 row) (1 row)
-- checks for single-quoted values
-- basic unicode input
SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u00\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u000g\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
ERROR: invalid Unicode escape value at or near "E'\'\u0000"
LINE 1: SELECT E'\'\u0000\''::jsonpath;
^
SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
jsonpath
----------
"ꯍ"
(1 row)
-- handling of unicode surrogate pairs
select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
correct_in_utf8
-----------------
"😄🐶"
(1 row)
select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83d\ud83d"
LINE 1: select E'\'\ud83d\ud83d\''::jsonpath;
^
select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
LINE 1: select E'\'\ude04\ud83d\''::jsonpath;
^
select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83dX"
LINE 1: select E'\'\ud83dX\''::jsonpath;
^
select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
LINE 1: select E'\'\ude04X\''::jsonpath;
^
--handling of simple unicode escapes
select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
correct_in_utf8
------------------------
"the Copyright © sign"
(1 row)
select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
correct_everywhere
----------------------
"dollar $ character"
(1 row)
select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
not_an_escape
----------------------
"dollar $ character"
(1 row)
select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
ERROR: invalid Unicode escape value at or near "E'\'null \u0000"
LINE 1: select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
^
select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
ERROR: unsupported Unicode escape sequence
LINE 1: select E'\'null \\u0000 escape\''::jsonpath as not_an_escape...
^
DETAIL: \u0000 cannot be converted to text.
-- checks for quoted key names -- checks for quoted key names
-- basic unicode input -- basic unicode input
SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape

72
src/test/regress/expected/jsonpath_encoding_1.out
View File

@ -78,78 +78,6 @@ select '"null \\u0000 escape"'::jsonpath as not_an_escape;
"null \\u0000 escape" "null \\u0000 escape"
(1 row) (1 row)
-- checks for single-quoted values
-- basic unicode input
SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u00\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
ERROR: invalid Unicode escape
LINE 1: SELECT E'\'\u000g\''::jsonpath;
^
HINT: Unicode escapes must be \uXXXX or \UXXXXXXXX.
SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
ERROR: invalid Unicode escape value at or near "E'\'\u0000"
LINE 1: SELECT E'\'\u0000\''::jsonpath;
^
SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'\uaBcD"
LINE 1: SELECT E'\'\uaBcD\''::jsonpath;
^
-- handling of unicode surrogate pairs
select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'\ud83d\ude04"
LINE 1: select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_...
^
select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83d\ud83d"
LINE 1: select E'\'\ud83d\ud83d\''::jsonpath;
^
select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
LINE 1: select E'\'\ude04\ud83d\''::jsonpath;
^
select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
ERROR: invalid Unicode surrogate pair at or near "E'\'\ud83dX"
LINE 1: select E'\'\ud83dX\''::jsonpath;
^
select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
ERROR: invalid Unicode surrogate pair at or near "E'\'\ude04"
LINE 1: select E'\'\ude04X\''::jsonpath;
^
--handling of simple unicode escapes
select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
ERROR: Unicode escape values cannot be used for code point values above 007F when the server encoding is not UTF8 at or near "E'\'the Copyright \u00a9"
LINE 1: select E'\'the Copyright \u00a9 sign\''::jsonpath as correct...
^
select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
correct_everywhere
----------------------
"dollar $ character"
(1 row)
select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
not_an_escape
----------------------
"dollar $ character"
(1 row)
select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
ERROR: invalid Unicode escape value at or near "E'\'null \u0000"
LINE 1: select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
^
select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
ERROR: unsupported Unicode escape sequence
LINE 1: select E'\'null \\u0000 escape\''::jsonpath as not_an_escape...
^
DETAIL: \u0000 cannot be converted to text.
-- checks for quoted key names -- checks for quoted key names
-- basic unicode input -- basic unicode input
SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape SELECT '$."\u"'::jsonpath; -- ERROR, incomplete escape

3
src/test/regress/sql/jsonpath.sql
View File

@ -30,10 +30,9 @@ select '$.a/+-1'::jsonpath;
select '1 * 2 + 4 % -3 != false'::jsonpath; select '1 * 2 + 4 % -3 != false'::jsonpath;
select '"\b\f\r\n\t\v\"\''\\"'::jsonpath; select '"\b\f\r\n\t\v\"\''\\"'::jsonpath;
select '''\b\f\r\n\t\v\"\''\\'''::jsonpath;
select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath; select '"\x50\u0067\u{53}\u{051}\u{00004C}"'::jsonpath;
select '''\x50\u0067\u{53}\u{051}\u{00004C}'''::jsonpath;
select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath; select '$.foo\x50\u0067\u{53}\u{051}\u{00004C}\t\"bar'::jsonpath;
select '"\z"'::jsonpath; -- unrecognized escape is just the literal char
select '$.g ? ($.a == 1)'::jsonpath; select '$.g ? ($.a == 1)'::jsonpath;
select '$.g ? (@ == 1)'::jsonpath; select '$.g ? (@ == 1)'::jsonpath;

23
src/test/regress/sql/jsonpath_encoding.sql
View File

@ -24,29 +24,6 @@ select '"dollar \\u0024 character"'::jsonpath as not_an_escape;
select '"null \u0000 escape"'::jsonpath as not_unescaped; select '"null \u0000 escape"'::jsonpath as not_unescaped;
select '"null \\u0000 escape"'::jsonpath as not_an_escape; select '"null \\u0000 escape"'::jsonpath as not_an_escape;
-- checks for single-quoted values
-- basic unicode input
SELECT E'\'\u\''::jsonpath; -- ERROR, incomplete escape
SELECT E'\'\u00\''::jsonpath; -- ERROR, incomplete escape
SELECT E'\'\u000g\''::jsonpath; -- ERROR, g is not a hex digit
SELECT E'\'\u0000\''::jsonpath; -- OK, legal escape
SELECT E'\'\uaBcD\''::jsonpath; -- OK, uppercase and lower case both OK
-- handling of unicode surrogate pairs
select E'\'\ud83d\ude04\ud83d\udc36\''::jsonpath as correct_in_utf8;
select E'\'\ud83d\ud83d\''::jsonpath; -- 2 high surrogates in a row
select E'\'\ude04\ud83d\''::jsonpath; -- surrogates in wrong order
select E'\'\ud83dX\''::jsonpath; -- orphan high surrogate
select E'\'\ude04X\''::jsonpath; -- orphan low surrogate
--handling of simple unicode escapes
select E'\'the Copyright \u00a9 sign\''::jsonpath as correct_in_utf8;
select E'\'dollar \u0024 character\''::jsonpath as correct_everywhere;
select E'\'dollar \\u0024 character\''::jsonpath as not_an_escape;
select E'\'null \u0000 escape\''::jsonpath as not_unescaped;
select E'\'null \\u0000 escape\''::jsonpath as not_an_escape;
-- checks for quoted key names -- checks for quoted key names
-- basic unicode input -- basic unicode input