Skip to content

Commit 6d0d61e

Browse files
author
Jules Ivanic
committed
Let a dialect declare which reserved keywords it accepts as names, starting with DuckDB
Reserved keywords are global: a keyword reserved for one dialect's syntax (ClickHouse's PREWHERE and SETTINGS, Exasol's FILE and SCRIPT, MySQL's XOR, ...) is rejected as a name in every dialect, although DuckDB, for instance, does not know these words and uses them as plain column and table names. - `Dialect#getUnreservedKeywords()`: the keywords reserved by default that a dialect accepts as unquoted names. New `Dialect.DUCKDB` (dollar-quoted string tags, adjacent string literals on a new line) declares 24: the 23 that DuckDB 1.5.5's `duckdb_keywords()` does not know, plus its unreserved CSV. MySQL's SELECT modifiers (DISTINCTROW, STRAIGHT_JOIN, SQL_CACHE, ...) stay reserved: the syntactic lookahead into a SELECT takes them as modifiers before any name. - `RelObjectName()` accepts them through `DialectUnreservedWord()` when the configured dialect does, and the identifier checks (`isAliasAhead()`, `canEndExpression()`, ...) consult the same set; the XOR operator, INCLUDES/EXCLUDES conditions and the CONNECT_BY_ROOT operator step aside for them. - Without a dialect, or with any other one, nothing changes. - `TaggedDollarStringTest` treats every dialect enabling dollar-quoted string tags, now including DuckDB, like PostgreSQL.
1 parent 166f0a2 commit 6d0d61e

4 files changed

Lines changed: 152 additions & 11 deletions

File tree

‎src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import java.util.Collections;
1717
import java.util.Arrays;
1818
import java.util.EnumSet;
19+
import java.util.HashSet;
1920
import java.util.List;
2021
import java.util.Set;
2122

@@ -47,7 +48,24 @@ public enum Dialect {
4748
AdjacentStringLiterals.WHITESPACE,
4849
Feature.allowDoubleQuotedStrings,
4950
Feature.allowBackslashEscapeCharacter), SNOWFLAKE(
50-
Feature.allowBackslashEscapeCharacter), INFORMIX, SPANNER, DORIS, COCKROACHDB, TERADATA;
51+
Feature.allowBackslashEscapeCharacter), INFORMIX, SPANNER, DORIS, COCKROACHDB, TERADATA, DUCKDB(
52+
AdjacentStringLiterals.NEWLINE,
53+
Feature.allowDollarQuotedStringTags);
54+
55+
/**
56+
* Keywords reserved by default that DuckDB does not know, so that DuckDB queries use them
57+
* as plain names: DuckDB 1.5.5's {@code duckdb_keywords()} has no such keyword, except CSV,
58+
* an unreserved one. MySQL's SELECT modifiers (DISTINCTROW, STRAIGHT_JOIN, SQL_CACHE,
59+
* SQL_CALC_FOUND_ROWS, SQL_NO_CACHE) stay reserved: the syntactic lookahead into a SELECT
60+
* takes them as modifiers before any name.
61+
*/
62+
private static final Set<String> DUCKDB_UNRESERVED_KEYWORDS = Collections.unmodifiableSet(
63+
new HashSet<>(Arrays.asList("ABSENT", "CASEWHEN", "CONNECT", "CONNECT_BY_ROOT",
64+
"CSV",
65+
"EXCLUDES", "EXTEND", "FBV", "FILE", "FINAL", "IIF", "INCLUDES", "MINUS",
66+
"NOCYCLE", "OPTIMIZE", "OUTPUT", "PREFERRING", "PREWHERE", "PUBLIC",
67+
"SCRIPT",
68+
"SETTINGS", "TOP", "VERIFY", "XOR")));
5169

5270
private final Set<Feature> lexerFeatures;
5371
private final AdjacentStringLiterals adjacentStringLiterals;
@@ -70,6 +88,14 @@ public Set<Feature> getLexerFeatures() {
7088
public AdjacentStringLiterals getAdjacentStringLiterals() {
7189
return adjacentStringLiterals;
7290
}
91+
92+
/**
93+
* Keywords reserved by default (see {@code keywords.rst}) that this dialect accepts as
94+
* unquoted names, in upper case.
95+
*/
96+
public Set<String> getUnreservedKeywords() {
97+
return this == DUCKDB ? DUCKDB_UNRESERVED_KEYWORDS : Collections.emptySet();
98+
}
7399
}
74100

75101
public P withSquareBracketQuotation() {

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 50 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -429,6 +429,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
429429
return true;
430430
default:
431431
return (t.kind >= MIN_NON_RESERVED_WORD && t.kind <= MAX_NON_RESERVED_WORD)
432+
|| isDialectUnreservedKeyword(t)
432433
|| ")".equals(t.image) || "]".equals(t.image)
433434
|| "?".equals(t.image);
434435
}
@@ -619,7 +620,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
619620
boolean ternary = "?".equals(t.image) && isTernaryAhead();
620621
int prec;
621622
if (op == K_AND || op == OP_DOUBLEAND) prec = 4;
622-
else if (op == K_XOR || op == K_OR) prec = 2;
623+
else if (op == K_XOR && !isDialectUnreservedKeyword(t) || op == K_OR) prec = 2;
623624
else if (ternary) prec = 2;
624625
else break;
625626
if (prec < minPrec) break;
@@ -797,7 +798,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
797798
* alternatives accepted there (base identifiers, DATA_TYPE, non-reserved keywords
798799
* and the reserved-keyword-as-identifier list).
799800
*/
800-
private static boolean isIdentifierishNameToken(Token t) {
801+
private boolean isIdentifierishNameToken(Token t) {
801802
switch (t.kind) {
802803
case S_IDENTIFIER: case S_QUOTED_IDENTIFIER: case DATA_TYPE:
803804
case K_DATETIMELITERAL: case K_DATE_LITERAL:
@@ -810,7 +811,8 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
810811
case K_TOP: case K_VALUE: case K_VALUES:
811812
return true;
812813
default:
813-
return t.kind >= MIN_NON_RESERVED_WORD && t.kind <= MAX_NON_RESERVED_WORD;
814+
return t.kind >= MIN_NON_RESERVED_WORD && t.kind <= MAX_NON_RESERVED_WORD
815+
|| isDialectUnreservedKeyword(t);
814816
}
815817
}
816818

@@ -1175,7 +1177,8 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
11751177
private boolean isOptionHintWordAhead() {
11761178
int kind = getToken(1).kind;
11771179
if (kind == S_IDENTIFIER
1178-
|| (kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD)) {
1180+
|| (kind >= MIN_NON_RESERVED_WORD && kind <= MAX_NON_RESERVED_WORD)
1181+
|| isDialectUnreservedKeyword(getToken(1))) {
11791182
return true;
11801183
}
11811184
switch (kind) {
@@ -1505,7 +1508,8 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
15051508
|| next.kind == OPENING_CURLY_BRACKET && "-".equals(getToken(2).image)
15061509
|| next.kind == S_IDENTIFIER || next.kind == S_QUOTED_IDENTIFIER
15071510
|| next.kind == DATA_TYPE
1508-
|| next.kind >= MIN_NON_RESERVED_WORD && next.kind <= MAX_NON_RESERVED_WORD;
1511+
|| next.kind >= MIN_NON_RESERVED_WORD && next.kind <= MAX_NON_RESERVED_WORD
1512+
|| isDialectUnreservedKeyword(next);
15091513
}
15101514

15111515
private boolean isAliasAhead() {
@@ -1532,6 +1536,9 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
15321536
return false;
15331537
}
15341538

1539+
// Reserved keywords the dialect treats as plain names
1540+
if (isDialectUnreservedKeyword(t)) return true;
1541+
15351542
// For reserved keywords in alias position, skip the structural-
15361543
// keyword whitelist and go directly to follower disambiguation.
15371544
// Only check keywords that are actually in RelObjectName's token
@@ -1611,6 +1618,17 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
16111618
}
16121619
}
16131620

1621+
/**
1622+
* Whether {@code token} is a keyword reserved by default that the configured dialect
1623+
* accepts as a plain name, see {@link Dialect#getUnreservedKeywords()}.
1624+
*/
1625+
private boolean isDialectUnreservedKeyword(Token token) {
1626+
String dialect = getAsString(Feature.dialect);
1627+
return dialect != null && token.kind > MAX_NON_RESERVED_WORD && token.image != null
1628+
&& Dialect.valueOf(dialect).getUnreservedKeywords()
1629+
.contains(token.image.toUpperCase(Locale.ROOT));
1630+
}
1631+
16141632
private boolean isKeywordAhead(String keyword) {
16151633
Token token = getToken(1);
16161634
return token.kind != EOF && token.image != null && keyword.equalsIgnoreCase(token.image);
@@ -2185,6 +2203,7 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
21852203
// PostgreSQL JSON operator, it is consumed by prattExpressionRest instead
21862204
if ("?".equals(t.image) && isTernaryAhead()) return false;
21872205
if (isComparisonOperator(t)) return true;
2206+
if (isDialectUnreservedKeyword(t)) return false;
21882207
switch (t.kind) {
21892208
// Each suffix's start token:
21902209
case K_OVERLAPS: // OVERLAPS
@@ -6838,6 +6857,10 @@ String RelObjectName() :
68386857
&& getToken(1).kind <= MAX_NON_RESERVED_WORD })
68396858
result = NonReservedWord()
68406859

6860+
/* ── Reserved keywords the dialect treats as plain names ────────── */
6861+
| LOOKAHEAD({ isDialectUnreservedKeyword(getToken(1)) })
6862+
tk = DialectUnreservedWord()
6863+
68416864
/* ── Reserved keywords usable as identifiers (context-guarded) ─── */
68426865
| LOOKAHEAD({ isReservedKeywordAsIdentifier() })
68436866
( tk=<K_ALL> | tk=<K_ANY> | tk=<K_CASEWHEN> | tk=<K_CONNECT>
@@ -6865,6 +6888,23 @@ String RelObjectName() :
68656888
* was called: dotted-name continuations, index/constraint names, type names,
68666889
* named-parameter identifier chains, and join table references.
68676890
*/
6891+
/**
6892+
* The keywords reserved by default that some dialect accepts as plain names, see
6893+
* Dialect#getUnreservedKeywords(). Only reached behind isDialectUnreservedKeyword().
6894+
*/
6895+
Token DialectUnreservedWord() :
6896+
{ Token tk; }
6897+
{
6898+
(
6899+
tk=<K_ABSENT> | tk=<K_CASEWHEN> | tk=<K_CONNECT> | tk=<K_CONNECT_BY_ROOT> | tk=<K_CSV>
6900+
| tk=<K_EXCLUDES> | tk=<K_EXTEND> | tk=<K_FBV> | tk=<K_FILE> | tk=<K_FINAL> | tk=<K_IIF>
6901+
| tk=<K_INCLUDES> | tk=<K_MINUS> | tk=<K_NOCYCLE> | tk=<K_OPTIMIZE> | tk=<K_OUTPUT>
6902+
| tk=<K_PREFERRING> | tk=<K_PREWHERE> | tk=<K_PUBLIC> | tk=<K_SCRIPT> | tk=<K_SETTINGS>
6903+
| tk=<K_TOP> | tk=<K_VERIFY> | tk=<K_XOR>
6904+
)
6905+
{ return tk; }
6906+
}
6907+
68686908
String RelObjectNameExt() :
68696909
{ Token tk = null; String result = null; }
68706910
{
@@ -9693,14 +9733,14 @@ OracleHierarchicalExpression OracleHierarchicalQueryClause():
96939733
(
96949734
(
96959735
<K_START> <K_WITH> expr=XorExpression() {result.setStartExpression(expr);}
9696-
<K_CONNECT> <K_BY> [ <K_NOCYCLE> { result.setNoCycle(true); } ] expr=XorExpression()
9736+
<K_CONNECT> <K_BY> [ LOOKAHEAD({ getToken(1).kind == K_NOCYCLE }) <K_NOCYCLE> { result.setNoCycle(true); } ] expr=XorExpression()
96979737
{
96989738
result.setConnectExpression(expr);
96999739
}
97009740
)
97019741
|
97029742
(
9703-
<K_CONNECT> <K_BY> [ <K_NOCYCLE> { result.setNoCycle(true); } ] expr=XorExpression()
9743+
<K_CONNECT> <K_BY> [ LOOKAHEAD({ getToken(1).kind == K_NOCYCLE }) <K_NOCYCLE> { result.setNoCycle(true); } ] expr=XorExpression()
97049744
{
97059745
result.setConnectExpression(expr);
97069746
result.setConnectFirst(true);
@@ -10394,7 +10434,7 @@ Expression XorExpression():
1039410434
{
1039510435
result=AndChain()
1039610436
(
10397-
LOOKAHEAD(2)
10437+
LOOKAHEAD(2, { getToken(1).kind != K_XOR || !isDialectUnreservedKeyword(getToken(1)) })
1039810438
(
1039910439
<K_OR> right=AndChain() { result = new OrExpression(result, right); }
1040010440
|
@@ -11428,7 +11468,8 @@ Expression PrimaryExpression() #PrimaryExpression:
1142811468

1142911469
| LOOKAHEAD(2, {!interrupted}) retval = NextValExpression()
1143011470

11431-
| retval=ConnectByRootOperator()
11471+
| LOOKAHEAD({ getToken(1).kind == K_CONNECT_BY_ROOT && !isDialectUnreservedKeyword(getToken(1)) })
11472+
retval=ConnectByRootOperator()
1143211473

1143311474
| retval=ConnectByPriorOperator()
1143411475

‎src/test/java/net/sf/jsqlparser/expression/TaggedDollarStringTest.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import net.sf.jsqlparser.parser.CCJSqlParser;
2323
import net.sf.jsqlparser.parser.CCJSqlParserConstants;
2424
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
25+
import net.sf.jsqlparser.parser.feature.Feature;
2526
import net.sf.jsqlparser.parser.StreamProvider;
2627
import net.sf.jsqlparser.parser.Token;
2728
import net.sf.jsqlparser.schema.Column;
@@ -94,7 +95,9 @@ static Stream<Consumer<CCJSqlParser>> identifierConfigurations() {
9495
},
9596
parser -> parser.withDialect(Dialect.POSTGRESQL)
9697
.withDollarQuotedStringTags(false)),
97-
Stream.of(Dialect.values()).filter(dialect -> dialect != Dialect.POSTGRESQL)
98+
Stream.of(Dialect.values())
99+
.filter(dialect -> !dialect.getLexerFeatures()
100+
.contains(Feature.allowDollarQuotedStringTags))
98101
.map(dialect -> parser -> parser.withDialect(dialect)));
99102
}
100103

‎src/test/java/net/sf/jsqlparser/statement/DialectKeywordsTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*-
2+
* #%L
3+
* JSQLParser library
4+
* %%
5+
* Copyright (C) 2004 - 2026 JSQLParser
6+
* %%
7+
* Dual licensed under GNU LGPL 2.1 or Apache License 2.0
8+
* #L%
9+
*/
10+
package net.sf.jsqlparser.statement;
11+
12+
import net.sf.jsqlparser.JSQLParserException;
13+
import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect;
14+
import net.sf.jsqlparser.parser.CCJSqlParser;
15+
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
16+
import org.junit.jupiter.api.Test;
17+
import org.junit.jupiter.params.ParameterizedTest;
18+
import org.junit.jupiter.params.provider.MethodSource;
19+
20+
import java.util.function.Consumer;
21+
import java.util.stream.Stream;
22+
23+
import static net.sf.jsqlparser.test.TestUtils.assertSqlCanBeParsedAndDeparsed;
24+
import static org.junit.jupiter.api.Assertions.assertThrows;
25+
26+
/**
27+
* Verifies that the keywords a dialect does not reserve can be used as unquoted names with that
28+
* dialect, and stay reserved without it.
29+
*/
30+
public class DialectKeywordsTest {
31+
private static final Consumer<CCJSqlParser> DUCKDB =
32+
parser -> parser.withDialect(Dialect.DUCKDB);
33+
34+
public static Stream<String> duckDbUnreservedKeywords() {
35+
return Dialect.DUCKDB.getUnreservedKeywords().stream().sorted();
36+
}
37+
38+
@ParameterizedTest(name = "Keyword {0}")
39+
@MethodSource("duckDbUnreservedKeywords")
40+
public void testDuckDbUnreservedKeywordAsName(String keyword) throws JSQLParserException {
41+
assertSqlCanBeParsedAndDeparsed(
42+
String.format("SELECT %1$s.%1$s AS %1$s FROM %1$s.%1$s AS %1$s", keyword), true,
43+
DUCKDB);
44+
assertSqlCanBeParsedAndDeparsed(
45+
String.format("SELECT %1$s %1$s FROM %1$s %1$s WHERE %1$s.%1$s = 1", keyword), true,
46+
DUCKDB);
47+
assertSqlCanBeParsedAndDeparsed(
48+
String.format("WITH %1$s AS (SELECT 1 AS %1$s) SELECT %1$s FROM %1$s", keyword),
49+
true, DUCKDB);
50+
}
51+
52+
@Test
53+
public void testDuckDbUnreservedKeywordsStayReservedWithoutTheDialect() {
54+
assertThrows(JSQLParserException.class,
55+
() -> CCJSqlParserUtil.parse("SELECT settings, file FROM script"));
56+
}
57+
58+
@Test
59+
public void testOtherDialectsKeepTheirClauses() throws JSQLParserException {
60+
assertSqlCanBeParsedAndDeparsed("SELECT * FROM t PREWHERE a = 1 WHERE b = 2", true);
61+
assertSqlCanBeParsedAndDeparsed("SELECT 1 FROM t MINUS SELECT 2 FROM u", true);
62+
assertSqlCanBeParsedAndDeparsed("SELECT TOP 5 a FROM t", true);
63+
}
64+
65+
@Test
66+
public void testDuckDbSyntaxUsingItsOwnUnreservedKeywords() throws JSQLParserException {
67+
assertSqlCanBeParsedAndDeparsed("UPDATE t SET a = 1", true, DUCKDB);
68+
assertSqlCanBeParsedAndDeparsed("SELECT a FROM t USING SAMPLE 10%", true, DUCKDB);
69+
assertSqlCanBeParsedAndDeparsed("COPY t TO 'out.csv' (FORMAT csv)", true, DUCKDB);
70+
}
71+
}

0 commit comments

Comments
 (0)