SQL JSON path enhanced numeric literals

Add support for non-decimal integer literals and underscores in
numeric literals to SQL JSON path language.  This follows the rules of
ECMAScript, as referred to by the SQL standard.

Internally, all the numeric literal parsing of jsonpath goes through
numeric_in, which already supports all this, so this patch is just a
bit of lexer work and some tests and documentation.

Reviewed-by: Dean Rasheed <dean.a.rasheed@gmail.com>
Discussion: https://www.postgresql.org/message-id/flat/b11b25bb-6ec1-d42f-cedd-311eae59e1fb@enterprisedb.com

diff --git a/doc/src/sgml/json.sgml b/doc/src/sgml/json.sgml
index 4a5e71dfc267b98e5619a6339cb0ac0cdacf28ca..b6c2ddbf55a0e7be7633a85e9f24e425b8ff5ebf 100644 (file)
--- a/doc/src/sgml/json.sgml
+++ b/doc/src/sgml/json.sgml
@@ -779,6 +779,18 @@ UPDATE table_name SET jsonb_field[1]['a'] = '1';
    </listitem>
   </itemizedlist>
 
+  <para>
+   Numeric literals in SQL/JSON path expressions follow JavaScript rules,
+   which are different from both SQL and JSON in some minor details.  For
+   example, SQL/JSON path allows <literal>.1</literal> and
+   <literal>1.</literal>, which are invalid in JSON.  Non-decimal integer
+   literals and underscore separators are supported, for example,
+   <literal>1_000_000</literal>, <literal>0x1EEE_FFFF</literal>,
+   <literal>0o273</literal>, <literal>0b100101</literal>.  In SQL/JSON path
+   (and in JavaScript, but not in SQL proper), there must not be an underscore
+   separator directly after the radix prefix.
+  </para>
+
   <para>
    An SQL/JSON path expression is typically written in an SQL query as an
    SQL character string literal, so it must be enclosed in single quotes,

diff --git a/src/backend/catalog/sql_features.txt b/src/backend/catalog/sql_features.txt
index 75a09f14e089599f7f71c56cb376e3d8e3d721fb..0fb9ab75335d19baaef00f6040ed458e089f648e 100644 (file)
--- a/src/backend/catalog/sql_features.txt
+++ b/src/backend/catalog/sql_features.txt
@@ -553,6 +553,7 @@ T836    SQL/JSON path language: starts with predicate           YES
 T837   SQL/JSON path language: regex_like predicate            YES 
 T838   JSON_TABLE: PLAN DEFAULT clause         NO  
 T839   Formatted cast of datetimes to/from character strings           NO  
+T840   Hex integer literals in SQL/JSON path language          YES SQL:202x draft
 M001   Datalinks           NO  
 M002   Datalinks via SQL/CLI           NO  
 M003   Datalinks via Embedded SQL          NO  

diff --git a/src/backend/utils/adt/jsonpath_scan.l b/src/backend/utils/adt/jsonpath_scan.l
index e08b1c7cd7c96f0ccc104ce2b7b99f67df86196a..0916fc102751bf9eaa2f942428869db6851fa887 100644 (file)
--- a/src/backend/utils/adt/jsonpath_scan.l
+++ b/src/backend/utils/adt/jsonpath_scan.l
@@ -90,21 +90,32 @@ blank       [ \t\n\r\f]
 /* "other" means anything that's not special, blank, or '\' or '"' */
 other      [^\?\%\$\.\[\]\{\}\(\)\|\&\!\=\<\>\@\#\,\*:\-\+\/\\\" \t\n\r\f]
 
-digit      [0-9]
-integer        (0|[1-9]{digit}*)
-decimal        ({integer}\.{digit}*|\.{digit}+)
-real       ({integer}|{decimal})[Ee][-+]?{digit}+
-realfail   ({integer}|{decimal})[Ee][-+]
-
-integer_junk   {integer}{other}
+decdigit   [0-9]
+hexdigit   [0-9A-Fa-f]
+octdigit   [0-7]
+bindigit   [0-1]
+
+/* DecimalInteger in ECMAScript; must not start with 0 unless it's exactly 0 */
+decinteger (0|[1-9](_?{decdigit})*)
+/* DecimalDigits in ECMAScript; only used as part of other rules */
+decdigits  {decdigit}(_?{decdigit})*
+/* Non-decimal integers; in ECMAScript, these must not have underscore after prefix */
+hexinteger 0[xX]{hexdigit}(_?{hexdigit})*
+octinteger 0[oO]{octdigit}(_?{octdigit})*
+bininteger 0[bB]{bindigit}(_?{bindigit})*
+
+decimal        ({decinteger}\.{decdigits}?|\.{decdigits})
+real       ({decinteger}|{decimal})[Ee][-+]?{decdigits}
+realfail   ({decinteger}|{decimal})[Ee][-+]
+
+decinteger_junk    {decinteger}{other}
 decimal_junk   {decimal}{other}
 real_junk      {real}{other}
 
-hex_dig        [0-9A-Fa-f]
-unicode        \\u({hex_dig}{4}|\{{hex_dig}{1,6}\})
-unicodefail    \\u({hex_dig}{0,3}|\{{hex_dig}{0,6})
-hex_char   \\x{hex_dig}{2}
-hex_fail   \\x{hex_dig}{0,1}
+unicode        \\u({hexdigit}{4}|\{{hexdigit}{1,6}\})
+unicodefail    \\u({hexdigit}{0,3}|\{{hexdigit}{0,6})
+hex_char   \\x{hexdigit}{2}
+hex_fail   \\x{hexdigit}{0,1}
 
 %%
 
@@ -274,7 +285,28 @@ hex_fail   \\x{hex_dig}{0,1}
                                    return NUMERIC_P;
                                }
 
-{integer}                      {
+{decinteger}                   {
+                                   addstring(true, yytext, yyleng);
+                                   addchar(false, '\0');
+                                   yylval->str = scanstring;
+                                   return INT_P;
+                               }
+
+{hexinteger}                   {
+                                   addstring(true, yytext, yyleng);
+                                   addchar(false, '\0');
+                                   yylval->str = scanstring;
+                                   return INT_P;
+                               }
+
+{octinteger}                   {
+                                   addstring(true, yytext, yyleng);
+                                   addchar(false, '\0');
+                                   yylval->str = scanstring;
+                                   return INT_P;
+                               }
+
+{bininteger}                   {
                                    addstring(true, yytext, yyleng);
                                    addchar(false, '\0');
                                    yylval->str = scanstring;
@@ -287,7 +319,7 @@ hex_fail    \\x{hex_dig}{0,1}
                                        "invalid numeric literal");
                                    yyterminate();
                                }
-{integer_junk}                 {
+{decinteger_junk}              {
                                    jsonpath_yyerror(
                                        NULL, escontext,
                                        "trailing junk after numeric literal");

diff --git a/src/test/regress/expected/jsonpath.out b/src/test/regress/expected/jsonpath.out
index f866fb474f414ffbf41495ebe546fa33f69e6acb..eeffb38c1b690cbd8c122af5885e1c1624646e7f 100644 (file)
--- a/src/test/regress/expected/jsonpath.out
+++ b/src/test/regress/expected/jsonpath.out
@@ -836,6 +836,7 @@ select '$ ? (@.a < +10.1e+1)'::jsonpath;
  $?(@."a" < 101)
 (1 row)
 
+-- numeric literals
 select '0'::jsonpath;
  jsonpath 
 ----------
@@ -846,6 +847,10 @@ select '00'::jsonpath;
 ERROR:  trailing junk after numeric literal at or near "00" of jsonpath input
 LINE 1: select '00'::jsonpath;
                ^
+select '0755'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0755'::jsonpath;
+               ^
 select '0.0'::jsonpath;
  jsonpath 
 ----------
@@ -1032,6 +1037,163 @@ select '1?(2>3)'::jsonpath;
  (1)?(2 > 3)
 (1 row)
 
+-- nondecimal
+select '0b100101'::jsonpath;
+ jsonpath 
+----------
+ 37
+(1 row)
+
+select '0o273'::jsonpath;
+ jsonpath 
+----------
+ 187
+(1 row)
+
+select '0x42F'::jsonpath;
+ jsonpath 
+----------
+ 1071
+(1 row)
+
+-- error cases
+select '0b'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "0b" of jsonpath input
+LINE 1: select '0b'::jsonpath;
+               ^
+select '1b'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1b" of jsonpath input
+LINE 1: select '1b'::jsonpath;
+               ^
+select '0b0x'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0b0x'::jsonpath;
+               ^
+select '0o'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "0o" of jsonpath input
+LINE 1: select '0o'::jsonpath;
+               ^
+select '1o'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1o" of jsonpath input
+LINE 1: select '1o'::jsonpath;
+               ^
+select '0o0x'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0o0x'::jsonpath;
+               ^
+select '0x'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "0x" of jsonpath input
+LINE 1: select '0x'::jsonpath;
+               ^
+select '1x'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1x" of jsonpath input
+LINE 1: select '1x'::jsonpath;
+               ^
+select '0x0y'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0x0y'::jsonpath;
+               ^
+-- underscores
+select '1_000_000'::jsonpath;
+ jsonpath 
+----------
+ 1000000
+(1 row)
+
+select '1_2_3'::jsonpath;
+ jsonpath 
+----------
+ 123
+(1 row)
+
+select '0x1EEE_FFFF'::jsonpath;
+ jsonpath  
+-----------
+ 518979583
+(1 row)
+
+select '0o2_73'::jsonpath;
+ jsonpath 
+----------
+ 187
+(1 row)
+
+select '0b10_0101'::jsonpath;
+ jsonpath 
+----------
+ 37
+(1 row)
+
+select '1_000.000_005'::jsonpath;
+  jsonpath   
+-------------
+ 1000.000005
+(1 row)
+
+select '1_000.'::jsonpath;
+ jsonpath 
+----------
+ 1000
+(1 row)
+
+select '.000_005'::jsonpath;
+ jsonpath 
+----------
+ 0.000005
+(1 row)
+
+select '1_000.5e0_1'::jsonpath;
+ jsonpath 
+----------
+ 10005
+(1 row)
+
+-- error cases
+select '_100'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '_100'::jsonpath;
+               ^
+select '100_'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "100_" of jsonpath input
+LINE 1: select '100_'::jsonpath;
+               ^
+select '100__000'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '100__000'::jsonpath;
+               ^
+select '_1_000.5'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '_1_000.5'::jsonpath;
+               ^
+select '1_000_.5'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1_000_" of jsonpath input
+LINE 1: select '1_000_.5'::jsonpath;
+               ^
+select '1_000._5'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1_000._" of jsonpath input
+LINE 1: select '1_000._5'::jsonpath;
+               ^
+select '1_000.5_'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1_000.5_" of jsonpath input
+LINE 1: select '1_000.5_'::jsonpath;
+               ^
+select '1_000.5e_1'::jsonpath;
+ERROR:  trailing junk after numeric literal at or near "1_000.5e" of jsonpath input
+LINE 1: select '1_000.5e_1'::jsonpath;
+               ^
+-- underscore after prefix not allowed in JavaScript (but allowed in SQL)
+select '0b_10_0101'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0b_10_0101'::jsonpath;
+               ^
+select '0o_273'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0o_273'::jsonpath;
+               ^
+select '0x_42F'::jsonpath;
+ERROR:  syntax error at end of jsonpath input
+LINE 1: select '0x_42F'::jsonpath;
+               ^
 -- test non-error-throwing API
 SELECT str as jsonpath,
        pg_input_is_valid(str,'jsonpath') as ok,

diff --git a/src/test/regress/sql/jsonpath.sql b/src/test/regress/sql/jsonpath.sql
index f2cd63272945a5cdf7209b02645c2e04a8ee201b..56e0bef57fd3db7690a5aa39a6d5418ec82c3ce3 100644 (file)
--- a/src/test/regress/sql/jsonpath.sql
+++ b/src/test/regress/sql/jsonpath.sql
@@ -152,8 +152,11 @@ select '$ ? (@.a < 10.1e+1)'::jsonpath;
 select '$ ? (@.a < -10.1e+1)'::jsonpath;
 select '$ ? (@.a < +10.1e+1)'::jsonpath;
 
+-- numeric literals
+
 select '0'::jsonpath;
 select '00'::jsonpath;
+select '0755'::jsonpath;
 select '0.0'::jsonpath;
 select '0.000'::jsonpath;
 select '0.000e1'::jsonpath;
@@ -188,6 +191,53 @@ select '(1.).e'::jsonpath;
 select '(1.).e3'::jsonpath;
 select '1?(2>3)'::jsonpath;
 
+-- nondecimal
+select '0b100101'::jsonpath;
+select '0o273'::jsonpath;
+select '0x42F'::jsonpath;
+
+-- error cases
+select '0b'::jsonpath;
+select '1b'::jsonpath;
+select '0b0x'::jsonpath;
+
+select '0o'::jsonpath;
+select '1o'::jsonpath;
+select '0o0x'::jsonpath;
+
+select '0x'::jsonpath;
+select '1x'::jsonpath;
+select '0x0y'::jsonpath;
+
+-- underscores
+select '1_000_000'::jsonpath;
+select '1_2_3'::jsonpath;
+select '0x1EEE_FFFF'::jsonpath;
+select '0o2_73'::jsonpath;
+select '0b10_0101'::jsonpath;
+
+select '1_000.000_005'::jsonpath;
+select '1_000.'::jsonpath;
+select '.000_005'::jsonpath;
+select '1_000.5e0_1'::jsonpath;
+
+-- error cases
+select '_100'::jsonpath;
+select '100_'::jsonpath;
+select '100__000'::jsonpath;
+
+select '_1_000.5'::jsonpath;
+select '1_000_.5'::jsonpath;
+select '1_000._5'::jsonpath;
+select '1_000.5_'::jsonpath;
+select '1_000.5e_1'::jsonpath;
+
+-- underscore after prefix not allowed in JavaScript (but allowed in SQL)
+select '0b_10_0101'::jsonpath;
+select '0o_273'::jsonpath;
+select '0x_42F'::jsonpath;
+
+
 -- test non-error-throwing API
 
 SELECT str as jsonpath,