Skip to content

Commit c05ef22

Browse files
fudianchnfudianchn
authored
fix: preserve consumed tokens in unsupported statement recovery (#2752)
* fix: preserve complete tokens during unsupported statement recovery Signed-off-by: 付典 <fudianchn@gmail.com> * fix: separate opaque capture from syntax error recovery Signed-off-by: 付典 <fudianchn@gmail.com> --------- Signed-off-by: 付典 <fudianchn@gmail.com>
1 parent 9856d0e commit c05ef22

6 files changed

Lines changed: 528 additions & 32 deletions

File tree

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 144 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,65 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
140140
// then-branch closes the ternary and must not be taken as the JSON path operator
141141
private int ternaryThenBranchDepth = 0;
142142

143+
// Keep this routing check aligned with SingleStatement, including aliases and dialect gates.
144+
// It does not validate syntax that the grammar captures as opaque.
145+
private boolean isStatementStartAhead() {
146+
Token next = getToken(1);
147+
switch (next.kind) {
148+
case K_SELECT: case K_WITH: case K_VALUES: case K_VALUE: case K_FROM:
149+
case K_PIVOT: case K_UNPIVOT: case OPENING_BRACKET:
150+
case K_INSERT: case K_UPDATE: case K_DELETE: case K_MERGE:
151+
case K_UPSERT: case K_REPLACE: case K_ALTER: case K_RENAME:
152+
case K_CREATE: case K_DROP: case K_ANALYZE: case K_TRUNCATE:
153+
case K_EXEC: case K_EXECUTE: case K_CALL: case K_SET: case K_RESET:
154+
case K_SHOW: case K_REFRESH: case K_USE: case K_BEGIN: case K_START:
155+
case K_SAVEPOINT: case K_ROLLBACK: case K_COMMIT: case K_COMMENT:
156+
case K_DESCRIBE: case K_DESC: case K_EXPLAIN: case K_SUMMARIZE: case K_DECLARE: case K_GRANT:
157+
case K_PRAGMA: case K_INSTALL: case K_LOAD: case K_FORCE:
158+
case K_ATTACH: case K_DETACH: case K_CONNECT: case K_DISCONNECT:
159+
case K_PREPARE: case K_DEALLOCATE: case K_COPY: case K_ASSERT:
160+
case K_PURGE: case K_SESSION: case K_BRANCH: case K_LOCK: case K_IF: case K_TABLE:
161+
return true;
162+
case K_DO:
163+
return Dialect.POSTGRESQL.name().equals(getAsString(Feature.dialect));
164+
case K_IMPORT: case K_EXPORT:
165+
return Dialect.EXASOL.name().equals(getAsString(Feature.dialect))
166+
|| next.kind == K_EXPORT && getToken(2).kind == K_DATA;
167+
default:
168+
return "NOTIFY".equalsIgnoreCase(next.image)
169+
|| "REVOKE".equalsIgnoreCase(next.image)
170+
|| "RELEASE".equalsIgnoreCase(next.image);
171+
}
172+
}
173+
174+
private boolean isInformixIsolationLevel(Token token) {
175+
return "DIRTY".equalsIgnoreCase(token.image) || "COMMITTED".equalsIgnoreCase(token.image)
176+
|| "CURSOR".equalsIgnoreCase(token.image) || token.kind == K_REPEATABLE;
177+
}
178+
179+
private boolean isOpaqueStatementAhead() {
180+
int kind = getToken(1).kind;
181+
return kind != EOF && kind != ST_SEMICOLON && kind != K_END && !isStatementStartAhead();
182+
}
183+
184+
private void captureOpaqueStatements(Statements statements) throws ParseException {
185+
while (getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead()) {
186+
statements.add(UnsupportedStatement());
187+
while (getToken(1).kind == ST_SEMICOLON) {
188+
getNextToken();
189+
}
190+
}
191+
}
192+
193+
private void requireStatementBoundary() throws ParseException {
194+
int kind = getToken(1).kind;
195+
if (kind != EOF && kind != ST_SEMICOLON
196+
&& (kind == K_FROM || kind == K_PIVOT || kind == K_UNPIVOT || !isStatementStartAhead())) {
197+
throw new ParseException("Unexpected token after a supported statement: " + getToken(1).image);
198+
}
199+
}
200+
201+
143202
// depth of DuckDB MAP keys: the following ":" separates a key from its value
144203
// and must not be taken as the JSON path operator
145204
private int mapKeyDepth = 0;
@@ -3303,11 +3362,11 @@ Statement Statement() #Statement:
33033362
( <ST_SEMICOLON> )* <EOF>
33043363
)
33053364
|
3306-
LOOKAHEAD( { stm==null && getAsBoolean(Feature.allowUnsupportedStatements) } ) stm = UnsupportedStatement()
3365+
LOOKAHEAD( { stm==null && getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead() } )
3366+
stm = UnsupportedStatement() ( <ST_SEMICOLON> )* <EOF>
33073367
} catch (ParseException ex) {
3308-
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3309-
stm = new UnsupportedStatement( stm.toString(), error_skipto(ST_SEMICOLON) );
3310-
} else if ( errorRecovery ) {
3368+
if ( errorRecovery ) {
3369+
ifElseStatement = null;
33113370
parseErrors.add(ex);
33123371
error_skipto(ST_SEMICOLON);
33133372
stm = null;
@@ -3392,6 +3451,16 @@ Statement SingleStatement() :
33923451
&& !"=".equals(getToken(3).image) && !".".equals(getToken(3).image) })
33933452
stm = SetIdentityInsert()
33943453
|
3454+
LOOKAHEAD({ getAsBoolean(Feature.allowUnsupportedStatements)
3455+
&& getToken(1).kind == K_SET && getToken(2).kind == S_IDENTIFIER
3456+
&& !"=".equals(getToken(3).image) && !".".equals(getToken(3).image)
3457+
&& !":=".equals(getToken(3).image)
3458+
&& ("IDENTITY_INSERT".equalsIgnoreCase(getToken(2).image)
3459+
|| "ISOLATION".equalsIgnoreCase(getToken(2).image)
3460+
&& (isInformixIsolationLevel(getToken(3))
3461+
|| getToken(3).kind == K_TO && isInformixIsolationLevel(getToken(4)))) })
3462+
stm = OpaqueSetStatement()
3463+
|
33953464
stm = Set()
33963465
|
33973466
stm = Reset()
@@ -3614,11 +3683,28 @@ Statements Statements() #Statements: {
36143683
Expression condition;
36153684
}
36163685
{
3686+
{
3687+
// Opaque roots can be first; capture them before known grammar lookahead.
3688+
while (getToken(1).kind == ST_SEMICOLON) {
3689+
getNextToken();
3690+
}
3691+
captureOpaqueStatements(stmts);
3692+
if (getToken(1).kind == EOF) {
3693+
getNextToken();
3694+
return stmts;
3695+
}
3696+
}
36173697
(
36183698
(
36193699
( <ST_SEMICOLON> )*
3700+
{
3701+
captureOpaqueStatements(stmts);
3702+
if (getToken(1).kind == EOF) {
3703+
getNextToken();
3704+
return stmts;
3705+
}
3706+
}
36203707

3621-
// todo: allow also first statement to be an `UnsupportedStatement`
36223708
try {
36233709
(
36243710
<K_IF> condition=Condition()
@@ -3628,7 +3714,7 @@ Statements Statements() #Statements: {
36283714
<K_ELSE> ( stm2 = SingleStatement() | stm2 = Block() ) { ifElseStatement.setElseStatement(stm2); }
36293715
]
36303716

3631-
{ stmts.add( ifElseStatement ); }
3717+
{ requireStatementBoundary(); stmts.add( ifElseStatement ); }
36323718

36333719
[
36343720
LOOKAHEAD(2) <ST_SEMICOLON>
@@ -3651,12 +3737,7 @@ Statements Statements() #Statements: {
36513737
}
36523738

36533739
} catch (ParseException ex) {
3654-
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3655-
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "", error_skipto(ST_SEMICOLON) );
3656-
if (!unsupportedStatement.isEmpty()) {
3657-
stmts.add( unsupportedStatement );
3658-
}
3659-
} else if ( errorRecovery ) {
3740+
if ( errorRecovery ) {
36603741
parseErrors.add(ex);
36613742
error_skipto(ST_SEMICOLON);
36623743
stmts.add( null );
@@ -3669,6 +3750,13 @@ Statements Statements() #Statements: {
36693750

36703751
( LOOKAHEAD(2)
36713752
( <ST_SEMICOLON> )*
3753+
{
3754+
captureOpaqueStatements(stmts);
3755+
if (getToken(1).kind == EOF) {
3756+
getNextToken();
3757+
return stmts;
3758+
}
3759+
}
36723760
try {
36733761
(
36743762
<K_IF> condition=Condition()
@@ -3678,20 +3766,15 @@ Statements Statements() #Statements: {
36783766
<K_ELSE> ( stm2 = SingleStatement() | stm2 = Block() ) { ifElseStatement.setElseStatement(stm2); }
36793767
]
36803768

3681-
{ stmts.add( ifElseStatement ); }
3769+
{ requireStatementBoundary(); stmts.add( ifElseStatement ); }
36823770
)
36833771
|
36843772
(
36853773
stm = SingleStatement()
36863774
| stm = Block()
3687-
) { stmts.add(stm); stm=null; }
3775+
) { requireStatementBoundary(); stmts.add(stm); stm=null; }
36883776
} catch (ParseException ex) {
3689-
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3690-
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "" , error_skipto(ST_SEMICOLON) );
3691-
if (!unsupportedStatement.isEmpty()) {
3692-
stmts.add( unsupportedStatement );
3693-
}
3694-
} else if ( errorRecovery ) {
3777+
if ( errorRecovery ) {
36953778
parseErrors.add(ex);
36963779
error_skipto(ST_SEMICOLON);
36973780
stmts.add( null );
@@ -3709,7 +3792,7 @@ Statements Statements() #Statements: {
37093792
)*
37103793

37113794
[
3712-
LOOKAHEAD( { getAsBoolean(Feature.allowUnsupportedStatements) } )
3795+
LOOKAHEAD( { getAsBoolean(Feature.allowUnsupportedStatements) && isOpaqueStatementAhead() } )
37133796
stm = UnsupportedStatement()
37143797
{
37153798
if (!( (UnsupportedStatement) stm).isEmpty()) {
@@ -3726,6 +3809,19 @@ Statements Statements() #Statements: {
37263809
}
37273810
}
37283811

3812+
JAVACODE
3813+
UnsupportedStatement captureStatementTokens(Token statementStart) {
3814+
List<String> tokenImages = new ArrayList<String>();
3815+
Token statementEnd = getToken(0);
3816+
for (Token t = statementStart.next; ; t = t.next) {
3817+
tokenImages.add(t.image);
3818+
if (t == statementEnd) {
3819+
break;
3820+
}
3821+
}
3822+
return new UnsupportedStatement(tokenImages);
3823+
}
3824+
37293825
JAVACODE
37303826
List<String> error_skipto(int kind) {
37313827
ArrayList<String> tokenImages = new ArrayList<String>();
@@ -4754,6 +4850,33 @@ SessionStatement SessionStatement():
47544850
}
47554851
}
47564852

4853+
/** Preserves existing opaque SET forms while requiring their complete syntax. */
4854+
private UnsupportedStatement OpaqueSetStatement(): {
4855+
Token statementStart = getToken(0);
4856+
SetIdentityInsertStatement identityInsert;
4857+
boolean repeatableRead = false;
4858+
}
4859+
{
4860+
(
4861+
LOOKAHEAD({ "IDENTITY_INSERT".equalsIgnoreCase(getToken(2).image) })
4862+
identityInsert = SetIdentityInsert()
4863+
|
4864+
<K_SET> AccessKeyword("ISOLATION") [ <K_TO> ]
4865+
(
4866+
LOOKAHEAD({ isKeywordAhead("DIRTY") }) AccessKeyword("DIRTY") <K_READ>
4867+
[ LOOKAHEAD({ getToken(1).kind == K_WITH && "WARNING".equalsIgnoreCase(getToken(2).image) })
4868+
<K_WITH> AccessKeyword("WARNING") ]
4869+
| LOOKAHEAD({ isKeywordAhead("COMMITTED") }) AccessKeyword("COMMITTED") <K_READ>
4870+
[ <K_LAST> AccessKeyword("COMMITTED") ]
4871+
| LOOKAHEAD({ isKeywordAhead("CURSOR") }) AccessKeyword("CURSOR") AccessKeyword("STABILITY")
4872+
| <K_REPEATABLE> <K_READ> { repeatableRead = true; }
4873+
)
4874+
[ LOOKAHEAD({ isKeywordAhead("RETAIN") }) AccessKeyword("RETAIN") <K_UPDATE> AccessKeyword("LOCKS")
4875+
{ requireAccessSyntax(!repeatableRead, "REPEATABLE READ cannot retain update locks"); } ]
4876+
)
4877+
{ return captureStatementTokens(statementStart); }
4878+
}
4879+
47574880
SetStatement Set(): {
47584881
SetStatement set;
47594882
}

‎src/site/sphinx/usage.rst‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -786,7 +786,12 @@ open. These empty-input results replace the previous null returns of these metho
786786
By default a syntax error aborts the whole parse. Two features let a script survive one bad statement:
787787

788788
- ``parser.withErrorRecovery(true)`` skips to the next statement separator and returns an empty statement.
789-
- ``parser.withUnsupportedStatements(true)`` returns an ``UnsupportedStatement`` holding the raw text instead — though the **first** statement must be a regular one.
789+
- ``parser.withUnsupportedStatements(true)`` captures an unrecognised statement root as
790+
``UnsupportedStatement``; it may be the first statement. Failures in recognised statement
791+
grammar still throw, or use ``withErrorRecovery(true)`` and record the error. Thus
792+
``SELECT * FROM``, ``INSERT INTO t (a`` and ``UPDATE t SET`` do not become opaque statements.
793+
The two options serve different purposes; enabling unsupported capture does not suppress
794+
recognised syntax errors.
790795

791796
.. code-block:: java
792797
:caption: Error Recovery
@@ -806,19 +811,27 @@ By default a syntax error aborts the whole parse. Two features let a script surv
806811
:caption: Unsupported Statement
807812
808813
Statements statements = CCJSqlParserUtil.parseStatements(
809-
"select * from mytable; select from; select * from mytable2; select 4;"
814+
"select * from mytable; shutdown defrag; select * from mytable2; select 4;"
810815
, parser -> parser.withUnsupportedStatements() );
811816
812817
// 4 statements with one Unsupported Statement holding the content
813818
assertEquals(4, statements.size());
814819
assertInstanceOf(UnsupportedStatement.class, statements.get(1));
815-
assertEquals("select from", statements.get(1).toString());
820+
assertEquals("shutdown defrag", statements.get(1).toString());
816821
817822
// no errors records, because a statement has been returned
818823
assertEquals(0, parser.getParseErrors().size());
819824
820825
.. note::
821826

827+
Opaque capture is not SQL validation. An unrecognised root or an existing opaque
828+
grammar branch cannot establish validity in an unknown dialect. Callers that need
829+
validity must validate against that dialect. Recognised syntax failures formerly
830+
captured by the generic fallback now throw or produce an error-recovery placeholder;
831+
partial IF results and typed prefixes with invalid trailing clauses are not published
832+
as successful statements. ``SingleStatement()`` remains an incremental entry point;
833+
use ``Statement()`` or ``Statements()`` to check complete statement boundaries.
834+
822835
An ``UnsupportedStatement`` is reported as ``OPAQUE`` by :ref:`Classify a Statement` — nothing about its effects is knowable.
823836

824837

‎src/test/java/net/sf/jsqlparser/statement/StatementsTest.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,17 @@
99
*/
1010
package net.sf.jsqlparser.statement;
1111

12+
import static org.junit.jupiter.api.Assertions.assertEquals;
13+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
14+
import static org.junit.jupiter.api.Assertions.assertNull;
15+
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
16+
1217
import net.sf.jsqlparser.JSQLParserException;
1318
import net.sf.jsqlparser.parser.CCJSqlParser;
1419
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
1520
import net.sf.jsqlparser.parser.ParseException;
1621
import net.sf.jsqlparser.parser.StringProvider;
1722
import net.sf.jsqlparser.statement.select.Select;
18-
import static org.junit.jupiter.api.Assertions.assertEquals;
19-
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
20-
import static org.junit.jupiter.api.Assertions.assertNull;
21-
2223
import net.sf.jsqlparser.test.TestUtils;
2324
import org.junit.jupiter.api.Test;
2425

@@ -93,7 +94,7 @@ public void testStatementsErrorRecovery3() throws JSQLParserException, ParseExce
9394
@Test
9495
public void testStatementsErrorRecovery4() throws JSQLParserException {
9596
Statements statements = CCJSqlParserUtil.parseStatements(
96-
"select * from mytable; select from; select * from mytable2; select 4 from dual;",
97+
"select * from mytable; shutdown defrag; select * from mytable2; select 4 from dual;",
9798
parser -> parser.withUnsupportedStatements());
9899

99100
assertEquals(4, statements.size());
@@ -103,6 +104,14 @@ public void testStatementsErrorRecovery4() throws JSQLParserException {
103104
assertInstanceOf(Select.class, statements.get(2));
104105
assertInstanceOf(Select.class, statements.get(3));
105106

106-
TestUtils.assertStatementCanBeDeparsedAs(statements.get(1), "select from", true);
107+
TestUtils.assertStatementCanBeDeparsedAs(statements.get(1), "shutdown defrag", true);
107108
}
109+
110+
@Test
111+
void malformedKnownStatementDoesNotUseUnsupportedCapture() {
112+
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parseStatements(
113+
"select * from mytable; select from; select * from mytable2; select 4 from dual;",
114+
parser -> parser.withUnsupportedStatements()));
115+
}
116+
108117
}

‎src/test/java/net/sf/jsqlparser/statement/UnsupportedIfBoundaryTest.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*-
2+
* #%L
3+
* JSQLParser library
4+
* %%
5+
* Copyright (C) 2004 - 2026 JSQLParser
6+
* %%
7+
* Dual licensed under GNU LGPL 2.1 or Apache License 2.0
8+
* #L%
9+
*/
10+
package net.sf.jsqlparser.statement;
11+
12+
import static org.junit.jupiter.api.Assertions.assertEquals;
13+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
14+
import static org.junit.jupiter.api.Assertions.assertNull;
15+
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
16+
17+
import net.sf.jsqlparser.JSQLParserException;
18+
import net.sf.jsqlparser.parser.CCJSqlParser;
19+
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
20+
import net.sf.jsqlparser.statement.select.Select;
21+
import org.junit.jupiter.params.ParameterizedTest;
22+
import org.junit.jupiter.params.provider.ValueSource;
23+
24+
class UnsupportedIfBoundaryTest {
25+
@ParameterizedTest
26+
@ValueSource(strings = {"IF x > 0 SELECT 1 WHERE", "IF x > 0 SELECT * FROM"})
27+
void malformedIfBodyCannotBecomeAnOpaqueSuffix(String sql) {
28+
for (boolean unsupported : new boolean[] {false, true}) {
29+
for (String prefix : new String[] {"", "SELECT 0;"}) {
30+
assertThrowsExactly(JSQLParserException.class,
31+
() -> CCJSqlParserUtil.parseStatements(prefix + sql + ";SELECT 2;",
32+
p -> p.withUnsupportedStatements(unsupported)));
33+
}
34+
}
35+
}
36+
37+
@ParameterizedTest
38+
@ValueSource(strings = {"IF x > 0 SELECT 1 WHERE", "IF x > 0 SELECT * FROM"})
39+
void malformedIfBodyRecordsOneErrorWithoutPublishingPartialIf(String sql) throws Exception {
40+
for (boolean unsupported : new boolean[] {false, true}) {
41+
for (String prefix : new String[] {"", "SELECT 0;"}) {
42+
CCJSqlParser parser = CCJSqlParserUtil.newParser(prefix + sql + ";SELECT 2;")
43+
.withUnsupportedStatements(unsupported).withErrorRecovery(true);
44+
Statements statements = parser.Statements();
45+
int index = prefix.isEmpty() ? 0 : 1;
46+
assertEquals(index + 2, statements.size());
47+
assertNull(statements.get(index));
48+
assertEquals("SELECT 2",
49+
assertInstanceOf(Select.class, statements.get(index + 1)).toString());
50+
assertEquals(1, parser.getParseErrors().size());
51+
}
52+
}
53+
}
54+
}

0 commit comments

Comments
 (0)