Spark: TABLE t, VALUES ... AS t(a, b), DESC TABLE t (#13 onto the v0.63.0 branch) (#14)

* Spark: TABLE t as a statement, VALUES ... AS t(a, b) as a query body (#13)

* Spark: TABLE t as a statement, VALUES ... AS t(a, b) as a query body

Two dialect hooks, on for Spark, Databricks and Hive. supports_table_statement
lets a statement start with TABLE, which parse_query already reads as
SetExpr::Table. supports_values_alias reads a table alias with a column
list after a VALUES body and rewrites the body as SELECT * FROM (VALUES
...) AS t(a, b), since a bare SetExpr::Values has nowhere to carry the
alias; a reserved word after VALUES (UNION, ORDER, LIMIT) is not an alias.

* parse_as_table: take a second name part only after a period

It read three tokens unconditionally, so TABLE t ORDER BY a swallowed
ORDER BY and then failed on what followed; the statement form is the
first place a clause can follow the name.

* Spark: read TABLE t as SELECT * FROM t in every query-body position

The string-only SetExpr::Table drops identifier quoting and leaves
consumers to special-case it at the top level. Under
supports_table_statement, build a SELECT * over the parsed ObjectName
instead, so TABLE works on set-operation sides, in CTEs, INSERT and
CTAS, and keeps its quote style.

* Spark: DESC TABLE t describes the table, not the query TABLE t

With TABLE able to start a query, DESCRIBE TABLE t parsed as a describe
of that query, and DESCRIBE TABLE EXTENDED t took EXTENDED for the table
name. For DESC/DESCRIBE under supports_table_statement, a TABLE followed
by a name now goes straight to the describe-table form, which accepts
EXTENDED/FORMATTED after TABLE as Spark writes it and prints TABLE
first. DESCRIBE table (a table named table) and EXPLAIN are unchanged.

* Spark: read a VALUES alias on each side of a set operation

The alias was read after the whole query body, by which point a set
operator had been consumed, so VALUES ... AS t(a) UNION ... failed.
Read it where the VALUES operand is parsed, the way TABLE is.

Author: moshap-firebolt

src/ast/mod.rs

@@ -5184,12 +5184,12 @@ impl fmt::Display for Statement {
             } => {
                 write!(f, "{describe_alias} ")?;
 
-                if let Some(format) = hive_format {
-                    write!(f, "{format} ")?;
-                }
                 if *has_table_keyword {
                     write!(f, "TABLE ")?;
                 }
+                if let Some(format) = hive_format {
+                    write!(f, "{format} ")?;
+                }
 
                 write!(f, "{table_name}")
             }

src/dialect/databricks.rs

@@ -54,6 +54,16 @@ impl Dialect for DatabricksDialect {
         true
     }
 
+    /// `TABLE t` as a statement
+    fn supports_table_statement(&self) -> bool {
+        true
+    }
+
+    /// `VALUES (1, 2) AS t(a, b)` as a query body
+    fn supports_values_alias(&self) -> bool {
+        true
+    }
+
     /// `1Y`, `1S`, `1L`, `1.0F`, `1.0D`, `1.0BD`
     fn supports_typed_numeric_literal_suffix(&self) -> bool {
         true

src/dialect/hive.rs

@@ -52,6 +52,16 @@ impl Dialect for HiveDialect {
         true
     }
 
+    /// `TABLE t` as a statement
+    fn supports_table_statement(&self) -> bool {
+        true
+    }
+
+    /// `VALUES (1, 2) AS t(a, b)` as a query body
+    fn supports_values_alias(&self) -> bool {
+        true
+    }
+
     /// `1Y`, `1S`, `1L`, `1.0F`, `1.0D`, `1.0BD`
     fn supports_typed_numeric_literal_suffix(&self) -> bool {
         true

src/dialect/mod.rs

@@ -508,6 +508,19 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if a statement may be a bare `TABLE t`, meaning
+    /// `SELECT * FROM t`, as in Spark SQL.
+    fn supports_table_statement(&self) -> bool {
+        false
+    }
+
+    /// Returns true if a `VALUES` query body may carry a table alias with a
+    /// column list, `VALUES (1, 2) AS t(a, b)`, as in Spark SQL. It parses to
+    /// `SELECT * FROM (VALUES (1, 2)) AS t(a, b)`.
+    fn supports_values_alias(&self) -> bool {
+        false
+    }
+
     /// Returns true if the dialect supports numbers containing underscores, e.g. `10_000_000`
     fn supports_numeric_literal_underscores(&self) -> bool {
         false

src/dialect/spark.rs

@@ -55,6 +55,16 @@ impl Dialect for SparkSqlDialect {
         true
     }
 
+    /// `TABLE t` as a statement
+    fn supports_table_statement(&self) -> bool {
+        true
+    }
+
+    /// `VALUES (1, 2) AS t(a, b)` as a query body
+    fn supports_values_alias(&self) -> bool {
+        true
+    }
+
     /// `1Y`, `1S`, `1L`, `1.0F`, `1.0D`, `1.0BD`
     fn supports_typed_numeric_literal_suffix(&self) -> bool {
         true

src/parser/mod.rs

@@ -650,6 +650,10 @@ impl<'a> Parser<'a> {
                     self.prev_token();
                     self.parse_query().map(Into::into)
                 }
+                Keyword::TABLE if self.dialect.supports_table_statement() => {
+                    self.prev_token();
+                    self.parse_query().map(Into::into)
+                }
                 Keyword::TRUNCATE => self.parse_truncate().map(Into::into),
                 Keyword::ATTACH => {
                     if dialect_of!(self is DuckDbDialect) {
@@ -14585,7 +14589,22 @@ impl<'a> Parser<'a> {
             }
         }
 
-        match self.maybe_parse(|parser| parser.parse_statement())? {
+        // Spark's `DESC [TABLE] [EXTENDED | FORMATTED] name`: where `TABLE` can
+        // start a query, `DESC TABLE t` would otherwise describe the query `TABLE t`.
+        // `DESCRIBE table` and `DESCRIBE table.x` still name a table called `table`.
+        let describe_table = describe_alias != DescribeAlias::Explain
+            && self.dialect.supports_table_statement()
+            && self.peek_keyword(Keyword::TABLE)
+            && !matches!(
+                self.peek_nth_token_ref(1).token,
+                Token::EOF | Token::SemiColon | Token::Period
+            );
+        let statement = if describe_table {
+            None
+        } else {
+            self.maybe_parse(|parser| parser.parse_statement())?
+        };
+        match statement {
             Some(Statement::Explain { .. }) | Some(Statement::ExplainTable { .. }) => Err(
                 ParserError::ParserError("Explain must be root of the plan".to_string()),
             ),
@@ -14600,19 +14619,18 @@ impl<'a> Parser<'a> {
                 options,
             }),
             _ => {
-                let hive_format =
-                    match self.parse_one_of_keywords(&[Keyword::EXTENDED, Keyword::FORMATTED]) {
-                        Some(Keyword::EXTENDED) => Some(HiveDescribeFormat::Extended),
-                        Some(Keyword::FORMATTED) => Some(HiveDescribeFormat::Formatted),
-                        _ => None,
-                    };
+                let mut hive_format = self.parse_hive_describe_format();
 
-                let has_table_keyword = if self.dialect.describe_requires_table_keyword() {
-                    // only allow to use TABLE keyword for DESC|DESCRIBE statement
-                    self.parse_keyword(Keyword::TABLE)
-                } else {
-                    false
-                };
+                let has_table_keyword =
+                    if self.dialect.describe_requires_table_keyword() || describe_table {
+                        // only allow to use TABLE keyword for DESC|DESCRIBE statement
+                        self.parse_keyword(Keyword::TABLE)
+                    } else {
+                        false
+                    };
+                if has_table_keyword && hive_format.is_none() {
+                    hive_format = self.parse_hive_describe_format();
+                }
 
                 let table_name = self.parse_object_name(false)?;
                 Ok(Statement::ExplainTable {
@@ -14625,6 +14643,14 @@ impl<'a> Parser<'a> {
         }
     }
 
+    fn parse_hive_describe_format(&mut self) -> Option<HiveDescribeFormat> {
+        match self.parse_one_of_keywords(&[Keyword::EXTENDED, Keyword::FORMATTED]) {
+            Some(Keyword::EXTENDED) => Some(HiveDescribeFormat::Extended),
+            Some(Keyword::FORMATTED) => Some(HiveDescribeFormat::Formatted),
+            _ => None,
+        }
+    }
+
     /// Parse a query expression, i.e. a `SELECT` statement optionally
     /// preceded with some `WITH` CTE declarations and optionally followed
     /// by `ORDER BY`. Unlike some other parse_... methods, this one doesn't
@@ -15209,12 +15235,40 @@ impl<'a> Parser<'a> {
             SetExpr::Query(subquery)
         } else if self.parse_keyword(Keyword::VALUES) {
             let is_mysql = dialect_of!(self is MySqlDialect);
-            SetExpr::Values(self.parse_values(is_mysql, false)?)
+            let values = SetExpr::Values(self.parse_values(is_mysql, false)?);
+            // The alias belongs to this operand, so read it before any set operator.
+            let alias = if self.dialect.supports_values_alias() {
+                self.maybe_parse_table_alias()?
+            } else {
+                None
+            };
+            match alias {
+                Some(alias) => Self::values_with_alias(Box::new(values), alias),
+                None => values,
+            }
         } else if self.parse_keyword(Keyword::VALUE) {
             let is_mysql = dialect_of!(self is MySqlDialect);
             SetExpr::Values(self.parse_values(is_mysql, true)?)
         } else if self.parse_keyword(Keyword::TABLE) {
-            SetExpr::Table(Box::new(self.parse_as_table()?))
+            if self.dialect.supports_table_statement() {
+                // `TABLE t` is `SELECT * FROM t`; keeping the ObjectName
+                // preserves quoting, which the string-only `Table` drops.
+                let name = self.parse_object_name(false)?;
+                Self::select_star_from(TableFactor::Table {
+                    name,
+                    alias: None,
+                    args: None,
+                    with_hints: vec![],
+                    version: None,
+                    partitions: vec![],
+                    with_ordinality: false,
+                    json_path: None,
+                    sample: None,
+                    index_hints: vec![],
+                })
+            } else {
+                SetExpr::Table(Box::new(self.parse_as_table()?))
+            }
         } else {
             return self.expected_ref(
                 "SELECT, VALUES, or a subquery in the query body",
@@ -15796,11 +15850,77 @@ impl<'a> Parser<'a> {
         Ok(clauses)
     }
 
+    /// `VALUES ... AS t(a, b)` as `SELECT * FROM (VALUES ...) AS t(a, b)`:
+    /// the alias has nowhere to go on a bare `SetExpr::Values`.
+    fn values_with_alias(values: Box<SetExpr>, alias: TableAlias) -> SetExpr {
+        let subquery = Box::new(Query {
+            with: None,
+            body: values,
+            order_by: None,
+            limit_clause: None,
+            fetch: None,
+            locks: vec![],
+            for_clause: None,
+            settings: None,
+            format_clause: None,
+            pipe_operators: vec![],
+        });
+        Self::select_star_from(TableFactor::Derived {
+            lateral: false,
+            subquery,
+            alias: Some(alias),
+            sample: None,
+        })
+    }
+
+    fn select_star_from(relation: TableFactor) -> SetExpr {
+        SetExpr::Select(Box::new(Select {
+            select_token: AttachedToken::empty(),
+            optimizer_hints: vec![],
+            distinct: None,
+            select_modifiers: None,
+            top: None,
+            top_before_distinct: false,
+            projection: vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())],
+            exclude: None,
+            into: None,
+            from: vec![TableWithJoins {
+                relation,
+                joins: vec![],
+            }],
+            lateral_views: vec![],
+            prewhere: None,
+            selection: None,
+            group_by: GroupByExpr::Expressions(vec![], vec![]),
+            cluster_by: vec![],
+            distribute_by: vec![],
+            sort_by: vec![],
+            having: None,
+            named_window: vec![],
+            window_before_qualify: false,
+            qualify: None,
+            value_table_mode: None,
+            connect_by: vec![],
+            flavor: SelectFlavor::Standard,
+        }))
+    }
+
     /// Parse `CREATE TABLE x AS TABLE y`
     pub fn parse_as_table(&mut self) -> Result<Table, ParserError> {
         let token1 = self.next_token();
-        let token2 = self.next_token();
-        let token3 = self.next_token();
+        // Only a `.` continues the name; anything else (ORDER BY, a pipe
+        // operator, the end of the statement) belongs to the caller.
+        let qualified = self.peek_token_ref().token == Token::Period;
+        let token2 = if qualified {
+            self.next_token()
+        } else {
+            TokenWithSpan::wrap(Token::EOF)
+        };
+        let token3 = if qualified {
+            self.next_token()
+        } else {
+            TokenWithSpan::wrap(Token::EOF)
+        };
 
         let table_name;
         let schema_name;

tests/sqlparser_common.rs

@@ -4843,6 +4843,8 @@ fn parse_create_table_as() {
 
 #[test]
 fn parse_create_table_as_table() {
+    // Dialects with `supports_table_statement` read `TABLE t` as `SELECT * FROM t`.
+    let dialects = all_dialects_where(|d| !d.supports_table_statement());
     let sql1 = "CREATE TABLE new_table AS TABLE old_table";
 
     let expected_query1 = Box::new(Query {
@@ -4861,7 +4863,7 @@ fn parse_create_table_as_table() {
         pipe_operators: vec![],
     });
 
-    match verified_stmt(sql1) {
+    match dialects.verified_stmt(sql1) {
         Statement::CreateTable(CreateTable { query, name, .. }) => {
             assert_eq!(name, ObjectName::from(vec![Ident::new("new_table")]));
             assert_eq!(query.unwrap(), expected_query1);
@@ -4887,7 +4889,7 @@ fn parse_create_table_as_table() {
         pipe_operators: vec![],
     });
 
-    match verified_stmt(sql2) {
+    match dialects.verified_stmt(sql2) {
         Statement::CreateTable(CreateTable { query, name, .. }) => {
             assert_eq!(name, ObjectName::from(vec![Ident::new("new_table")]));
             assert_eq!(query.unwrap(), expected_query2);