[SPARK-58366][SQL] Support JSON_TABLE table-valued function

common/utils/src/main/resources/error/error-conditions.json

@@ -1926,6 +1926,11 @@
           "Input schema <schema> must be a struct, an array, a map or a variant."
         ]
       },
+      "INVALID_JSON_TABLE_PATH" : {
+        "message" : [
+          "The <location> of JSON_TABLE has an invalid or unsupported JSON path <path>. Only simple, wildcard-free paths are supported (a trailing '[*]' is allowed on the row path)."
+        ]
+      },
       "INVALID_MAP_KEY_TYPE" : {
         "message" : [
           "The key of map cannot be/contain <keyType>."
@@ -5017,6 +5022,11 @@
           "CREATE TEMPORARY TABLE ... USING ... is a deprecated syntax. To overcome the issue, please use CREATE TEMPORARY VIEW instead."
         ]
       },
+      "DUPLICATE_JSON_TABLE_COLUMN" : {
+        "message" : [
+          "JSON_TABLE has duplicate column name <columnName>. Column names in the COLUMNS clause must be unique."
+        ]
+      },
       "EMPTY_IN_PREDICATE" : {
         "message" : [
           "IN predicate requires at least one value. Empty IN clauses like 'IN ()' are not allowed. Consider using 'WHERE FALSE' if you need an always-false condition, or provide at least one value in the IN list."

docs/sql-ref-ansi-compliance.md

@@ -539,6 +539,7 @@ Below is a list of all the keywords in Spark SQL.
 |ELSEIF|non-reserved|non-reserved|non-reserved|
 |END|reserved|non-reserved|reserved|
 |ENFORCED|non-reserved|non-reserved|non-reserved|
+|ERROR|non-reserved|non-reserved|non-reserved|
 |ESCAPE|reserved|non-reserved|reserved|
 |ESCAPED|non-reserved|non-reserved|non-reserved|
 |EVOLUTION|non-reserved|non-reserved|non-reserved|
@@ -616,6 +617,7 @@ Below is a list of all the keywords in Spark SQL.
 |ITERATE|non-reserved|non-reserved|non-reserved|
 |JOIN|reserved|strict-non-reserved|reserved|
 |JSON|non-reserved|non-reserved|non-reserved|
+|JSON_TABLE|non-reserved|non-reserved|reserved|
 |KEY|non-reserved|non-reserved|non-reserved|
 |KEYS|non-reserved|non-reserved|non-reserved|
 |LANGUAGE|non-reserved|non-reserved|reserved|

docs/sql-ref-syntax-qry-select-json-table.md

@@ -0,0 +1,130 @@
+---
+layout: global
+title: JSON_TABLE
+displayTitle: JSON_TABLE
+license: |
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+---
+
+### Description
+
+The `JSON_TABLE` table-valued function shreds a JSON document into a relational table. A
+_row path_ selects a sequence of JSON items, and a `COLUMNS` clause projects a value out of each
+item into a typed column. This is the SQL-standard way (SQL:2016) to turn JSON into rows and
+columns, and is commonly used to migrate queries from other systems such as Oracle, DB2, and
+MySQL.
+
+Only the flat (non-nested) form is currently supported. `NESTED PATH` columns are not yet
+supported.
+
+### Syntax
+
+```sql
+JSON_TABLE ( json_expr, row_path COLUMNS ( column_definition [ , ... ] ) [ error_clause ] ) [ table_alias ]
+
+column_definition
+    { column_name FOR ORDINALITY
+    | column_name data_type [ PATH json_path ]
+    | column_name data_type EXISTS [ PATH json_path ] }
+
+error_clause
+    { NULL | ERROR } ON ERROR
+```
+
+### Parameters
+
+* **json_expr**
+
+    An expression that evaluates to a `STRING` containing the JSON document.
+
+* **row_path**
+
+    A SQL/JSON path literal that selects the row source. A path ending in `[*]` (for example
+    `'$.items[*]'`) selects each element of the matched array as a separate row. Any other path
+    (for example `'$'`) selects a single value as one row. If the path matches nothing, no rows
+    are produced.
+
+* **column_name FOR ORDINALITY**
+
+    Declares a `BIGINT` column that is a 1-based sequential counter of the generated rows.
+
+* **column_name data_type [ PATH json_path ]**
+
+    A value column. The value at `json_path` (relative to a row item) is extracted and cast to
+    `data_type`. If `PATH` is omitted, the path defaults to the column name read as a single
+    object key: a simple identifier maps like `$.name`, while a name containing special characters
+    such as a dot is treated as one literal key (for example a column named `a.b` reads the key
+    `"a.b"`, equivalent to `$['a.b']`, not the nested path `a` -> `b`). If the path matches nothing,
+    the column is `NULL`.
+
+* **column_name data_type EXISTS [ PATH json_path ]**
+
+    An existence column. Evaluates to a truthy value when `json_path` matches and a falsy value
+    otherwise, cast to `data_type` (for example `BOOLEAN`).
+
+* **{ NULL | ERROR } ON ERROR**
+
+    Controls behavior when `json_expr` is `NULL` or not well-formed JSON. `NULL ON ERROR` (the
+    default) produces no rows. `ERROR ON ERROR` raises an error.
+
+* **table_alias**
+
+    An optional alias for the output, optionally followed by a column alias list.
+
+### Examples
+
+```sql
+-- Expand a JSON array into rows with typed columns and an ordinality counter
+SELECT t.* FROM JSON_TABLE(
+  '{"items":[{"id":1,"n":"a"},{"id":2,"n":"b"}]}',
+  '$.items[*]'
+  COLUMNS (
+    seq  FOR ORDINALITY,
+    id   INT    PATH '$.id',
+    name STRING PATH '$.n'
+  )
+) AS t;
++---+---+----+
+|seq| id|name|
++---+---+----+
+|  1|  1|   a|
+|  2|  2|   b|
++---+---+----+
+
+-- Implicit column path derived from the column name, and an EXISTS column
+SELECT * FROM JSON_TABLE(
+  '{"rows":[{"id":10,"opt":1},{"id":20}]}',
+  '$.rows[*]'
+  COLUMNS (id INT, hasOpt BOOLEAN EXISTS PATH '$.opt')
+) AS t;
++---+------+
+| id|hasOpt|
++---+------+
+| 10|  true|
+| 20| false|
++---+------+
+
+-- Join JSON_TABLE output against a base table using LATERAL
+SELECT d.id, t.k
+FROM docs d,
+LATERAL JSON_TABLE(d.doc, '$.tags[*]' COLUMNS (k STRING PATH '$.k')) AS t;
+```
+
+### Related Statements
+
+* [SELECT](sql-ref-syntax-qry-select.html)
+* [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+* [LATERAL VIEW Clause](sql-ref-syntax-qry-select-lateral-view.html)

docs/sql-ref-syntax-qry-select.md

@@ -93,6 +93,7 @@ SELECT [ hints , ... ] [ ALL | DISTINCT ] { [ [ named_expression | regex_column_
      * [Pivot relation](sql-ref-syntax-qry-select-pivot.html)
      * [Unpivot relation](sql-ref-syntax-qry-select-unpivot.html)
      * [Table-value function](sql-ref-syntax-qry-select-tvf.html)
+     * [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html)
      * [Inline table](sql-ref-syntax-qry-select-inline-table.html)
      * [UNNEST relation](sql-ref-syntax-qry-select-unnest.html)
      * [ [LATERAL](sql-ref-syntax-qry-select-lateral-subquery.html) ] ( Subquery )

docs/sql-ref-syntax.md

@@ -83,6 +83,7 @@ ability to generate logical and physical plan for a given query using
    * [SORT BY Clause](sql-ref-syntax-qry-select-sortby.html)
    * [TABLESAMPLE](sql-ref-syntax-qry-select-sampling.html)
    * [Table-valued Function](sql-ref-syntax-qry-select-tvf.html)
+   * [JSON_TABLE](sql-ref-syntax-qry-select-json-table.html)
    * [WHERE Clause](sql-ref-syntax-qry-select-where.html)
    * [Aggregate Function](sql-ref-syntax-qry-select-aggregate.html)
    * [Window Function](sql-ref-syntax-qry-select-window.html)

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseLexer.g4

@@ -256,6 +256,7 @@ ELSE: 'ELSE';
 ELSEIF: 'ELSEIF';
 END: 'END';
 ENFORCED: 'ENFORCED';
+ERROR: 'ERROR';
 ESCAPE: 'ESCAPE';
 ESCAPED: 'ESCAPED';
 EVOLUTION: 'EVOLUTION';
@@ -333,6 +334,7 @@ ITEMS: 'ITEMS';
 ITERATE: 'ITERATE';
 JOIN: 'JOIN';
 JSON: 'JSON';
+JSON_TABLE: 'JSON_TABLE';
 KEY: 'KEY';
 KEYS: 'KEYS';
 LANGUAGE: 'LANGUAGE';

sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4

@@ -1209,6 +1209,7 @@ relationPrimary
        watermarkClause? tableAlias                          #aliasedRelation
     | inlineTable                                           #inlineTableDefault2
     | unnest                                                #unnestTable
+    | jsonTable                                             #jsonTableRelation
     | tableFunctionCallWithTrailingClauses                  #tableValuedFunction
     ;
 
@@ -1220,6 +1221,27 @@ unnest
       (WITH ORDINALITY)? tableAlias
     ;
 
+// The ANSI SQL:2016 JSON_TABLE table-valued function. Because the COLUMNS clause is not a normal
+// function-argument list, it has a dedicated production rather than going through
+// tableFunctionCall. Only the flat (non-NESTED) subset is currently supported.
+jsonTable
+    : JSON_TABLE LEFT_PAREN jsonExpr=expression COMMA rowPath=stringLit
+        COLUMNS LEFT_PAREN jsonTableColumn (COMMA jsonTableColumn)* RIGHT_PAREN
+        jsonTableOnErrorClause?
+      RIGHT_PAREN tableAlias
+    ;
+
+jsonTableColumn
+    : colName=errorCapturingIdentifier FOR ORDINALITY                           #jsonTableOrdinalityColumn
+    | colName=errorCapturingIdentifier dataType
+        EXISTS (PATH path=stringLit)?                                           #jsonTableExistsColumn
+    | colName=errorCapturingIdentifier dataType (PATH path=stringLit)?          #jsonTableValueColumn
+    ;
+
+jsonTableOnErrorClause
+    : (NULL | ERROR) ON ERROR
+    ;
+
 optionsClause
     : WITH options=propertyList
     ;
@@ -2150,6 +2172,7 @@ ansiNonReserved
     | DROP
     | ELSEIF
     | ENFORCED
+    | ERROR
     | ESCAPED
     | EVOLUTION
     | EXACT
@@ -2208,6 +2231,7 @@ ansiNonReserved
     | ITEMS
     | ITERATE
     | JSON
+    | JSON_TABLE
     | KEY
     | KEYS
     | LANGUAGE
@@ -2570,6 +2594,7 @@ nonReserved
     | ELSEIF
     | END
     | ENFORCED
+    | ERROR
     | ESCAPE
     | ESCAPED
     | EVOLUTION
@@ -2642,6 +2667,7 @@ nonReserved
     | ITEMS
     | ITERATE
     | JSON
+    | JSON_TABLE
     | KEY
     | KEYS
     | LANGUAGE

sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala

@@ -138,6 +138,13 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase {
       ctx)
   }
 
+  def duplicateJsonTableColumnError(columnName: String, ctx: JsonTableContext): Throwable = {
+    new ParseException(
+      errorClass = "INVALID_SQL_SYNTAX.DUPLICATE_JSON_TABLE_COLUMN",
+      messageParameters = Map("columnName" -> toSQLId(columnName)),
+      ctx)
+  }
+
   def clausesWithPipeOperatorsUnsupportedError(
       ctx: QueryOrganizationContext,
       clauses: String): Throwable = {