Repository navigation
Fix parsing of reserved keyword select items after a comma - #2969
Jaden-Varkey wants to merge 3 commits into
Conversation
With projection trailing commas enabled (e.g. GenericDialect), a reserved keyword after a comma was always treated as the start of a clause, so `SELECT a, offset FROM t` failed to parse. Fixes apache#2049
|
No dialect label was applied because the title does not name a SQL dialect. If this change targets one or more dialects, edit the title to |
Codecov Report✅ All modified and coverable lines are covered by tests. Additional details and impacted files@@ Coverage Diff @@
## main #2969 +/- ##
=======================================
Coverage 83.28% 83.29%
=======================================
Files 42 42
Lines 35355 35364 +9
Branches 35355 35364 +9
=======================================
+ Hits 29446 29455 +9
Misses 2458 2458
Partials 3451 3451 ☔ View full report in Codecov by Harness. 🚀 New features to boost your workflow:
|
| |kw, parser| { | ||
| Self::is_reserved_for_column_alias(kw, parser) | ||
| && !Self::is_keyword_projection_item(parser) | ||
| }, | ||
| ) | ||
| } | ||
|
|
||
| /// Called after a reserved keyword that follows a comma in a projection, e.g. | ||
| /// `SELECT a, offset FROM t`. Returns true if the next token means the keyword | ||
| /// is a select item rather than a clause after a trailing comma. | ||
| fn is_keyword_projection_item(parser: &mut Parser) -> bool { | ||
| parser.peek_token_ref().token == Token::Comma | ||
| || parser | ||
| .peek_one_of_keywords(&[Keyword::FROM, Keyword::AS]) | ||
| .is_some() | ||
| } |
There was a problem hiding this comment.
You should limit the column reading to LIMIT and OFFSET. As written it covers every keyword in RESERVED_FOR_COLUMN_ALIAS.
Since you are using claude to write this code, you sure make it fuzz and test extensively the syntax you want to add.
| |kw, parser| { | |
| Self::is_reserved_for_column_alias(kw, parser) | |
| && !Self::is_keyword_projection_item(parser) | |
| }, | |
| ) | |
| } | |
| /// Called after a reserved keyword that follows a comma in a projection, e.g. | |
| /// `SELECT a, offset FROM t`. Returns true if the next token means the keyword | |
| /// is a select item rather than a clause after a trailing comma. | |
| fn is_keyword_projection_item(parser: &mut Parser) -> bool { | |
| parser.peek_token_ref().token == Token::Comma | |
| || parser | |
| .peek_one_of_keywords(&[Keyword::FROM, Keyword::AS]) | |
| .is_some() | |
| } | |
| |kw, parser| { | |
| Self::is_reserved_for_column_alias(kw, parser) | |
| && !(matches!(kw, Keyword::LIMIT | Keyword::OFFSET) | |
| && parser.peek_select_item_end(0)) | |
| }, | |
| ) | |
| } | |
| /// Returns true if the `n`th next token ends a select item, so that a `LIMIT` | |
| /// or `OFFSET` before it names a column, as in `SELECT offset, limit FROM t`. | |
| fn peek_select_item_end(&self, n: usize) -> bool { | |
| match &self.peek_nth_token_ref(n).token { | |
| Token::Comma => true, | |
| Token::Word(w) => matches!(w.keyword, Keyword::FROM | Keyword::AS), | |
| _ => false, | |
| } | |
| } |
| for sql in [ | ||
| "SELECT 'a' AS a, offset FROM t", | ||
| "SELECT a, offset FROM t", | ||
| "SELECT a, limit FROM t", | ||
| "SELECT a, offset, b FROM t", | ||
| "SELECT a, offset AS o FROM t", | ||
| "SELECT * FROM (SELECT a, offset FROM t)", | ||
| ] { | ||
| dialects.verified_stmt(sql); | ||
| } | ||
|
|
||
| dialects.one_statement_parses_to("SELECT a, FROM t", "SELECT a FROM t"); | ||
| dialects.one_statement_parses_to("SELECT a, LIMIT 1", "SELECT a LIMIT 1"); | ||
| } |
There was a problem hiding this comment.
You should cover the first-item queries and a keyword that must stay reserved. The two first-item cases and the union case fail on the current head.
| for sql in [ | |
| "SELECT 'a' AS a, offset FROM t", | |
| "SELECT a, offset FROM t", | |
| "SELECT a, limit FROM t", | |
| "SELECT a, offset, b FROM t", | |
| "SELECT a, offset AS o FROM t", | |
| "SELECT * FROM (SELECT a, offset FROM t)", | |
| ] { | |
| dialects.verified_stmt(sql); | |
| } | |
| dialects.one_statement_parses_to("SELECT a, FROM t", "SELECT a FROM t"); | |
| dialects.one_statement_parses_to("SELECT a, LIMIT 1", "SELECT a LIMIT 1"); | |
| } | |
| for sql in [ | |
| "SELECT offset FROM t", | |
| "SELECT offset, 'a' AS a FROM t", | |
| "SELECT 'a' AS a, offset FROM t", | |
| "SELECT a, offset FROM t", | |
| "SELECT a, limit FROM t", | |
| "SELECT a, offset, b FROM t", | |
| "SELECT a, offset AS o FROM t", | |
| "SELECT * FROM (SELECT a, offset FROM t)", | |
| ] { | |
| dialects.verified_stmt(sql); | |
| } | |
| dialects.one_statement_parses_to("SELECT a, FROM t", "SELECT a FROM t"); | |
| dialects.one_statement_parses_to("SELECT a, LIMIT 1", "SELECT a LIMIT 1"); | |
| assert_eq!( | |
| dialects | |
| .parse_sql_statements("SELECT a, union, b FROM t") | |
| .unwrap_err(), | |
| ParserError::ParserError( | |
| "Expected: SELECT, VALUES, or a subquery in the query body, found: ,".to_string() | |
| ) | |
| ); | |
| } |
Also parse a leading LIMIT/OFFSET column in dialects with empty projections.
| Token::Comma => true, | ||
| Token::Word(w) => matches!(w.keyword, Keyword::FROM | Keyword::AS), |
There was a problem hiding this comment.
You should treat every token that ends a select item as the end, not only ,, FROM and AS.
| Token::Comma => true, | |
| Token::Word(w) => matches!(w.keyword, Keyword::FROM | Keyword::AS), | |
| Token::Comma | Token::EOF | Token::SemiColon | Token::RParen => true, | |
| Token::Word(w) => matches!( | |
| w.keyword, | |
| Keyword::AS | |
| | Keyword::INTO | |
| | Keyword::FROM | |
| | Keyword::WHERE | |
| | Keyword::GROUP | |
| | Keyword::HAVING | |
| | Keyword::WINDOW | |
| | Keyword::ORDER | |
| | Keyword::UNION | |
| | Keyword::INTERSECT | |
| | Keyword::EXCEPT | |
| ), |
There was a problem hiding this comment.
You should cover a keyword column followed by something other than FROM. These five fail on the current head, the first with Expected: an expression, found: EOF.
| "SELECT * FROM (SELECT a, offset FROM t)", | |
| "SELECT a, offset", | |
| "SELECT * FROM (SELECT a, offset) AS s", | |
| "SELECT a, offset WHERE b", | |
| "SELECT a, offset INTO u FROM t", | |
| "SELECT a, offset UNION SELECT 1, 2", |
| for sql in [ | ||
| "SELECT offset FROM t", | ||
| "SELECT limit, offset FROM t", | ||
| "SELECT offset AS o FROM t", |
There was a problem hiding this comment.
You should cover the same in the first position of an empty-projection dialect. Both fail on the current head, the first with Expected: an expression, found: EOF.
| "SELECT offset AS o FROM t", | |
| "SELECT offset AS o FROM t", | |
| "SELECT offset", | |
| "SELECT offset WHERE b", |
|
Thanks. I applied the suggestions from both reviews in 61d34f8 and 72996fe. The first-item cases needed one more change. Generic and Postgres support empty projections, so To check it, I parsed every keyword in 27 select-list positions on 13 dialects, on On 61d34f8 I also ran |
Fixes #2049 (and downstream apache/datafusion#17802).
Dialects: none
LIMITandOFFSETused as column names failed to parse in dialects with projection trailing commas or empty projections (e.g.GenericDialect):After a comma, and at the start of an empty-projection select list,
LIMIT/OFFSETare now read as select items when followed by a token that ends a select item (,,AS,FROM,WHERE,INTO,UNION,),;, end of input, etc.). Real trailing commas and empty projections (SELECT a, FROM t,SELECT a, LIMIT 1,SELECT LIMIT 1) are unchanged. Other reserved keywords keep their current behavior.Tests:
parse_projection_trailing_comma_keyword_columninsqlparser_common.rs, plus a fuzz seed.AI-assisted (Claude).