Skip to content

Commit b7c937d

Browse files
fudianchnfudianchn
committed
fix: preserve complete tokens during unsupported statement recovery
Signed-off-by: 付典 <fudianchn@gmail.com>
1 parent bb55bb9 commit b7c937d

2 files changed

Lines changed: 260 additions & 3 deletions

File tree

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3269,6 +3269,7 @@ Token KeywordOrIdentifier():
32693269

32703270
Statement Statement() #Statement:
32713271
{
3272+
Token statementStart = getToken(0);
32723273
IfElseStatement ifElseStatement = null;
32733274
Statement stm = null;
32743275
Statement stm2 = null;
@@ -3306,7 +3307,8 @@ Statement Statement() #Statement:
33063307
LOOKAHEAD( { stm==null && getAsBoolean(Feature.allowUnsupportedStatements) } ) stm = UnsupportedStatement()
33073308
} catch (ParseException ex) {
33083309
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3309-
stm = new UnsupportedStatement( stm.toString(), error_skipto(ST_SEMICOLON) );
3310+
stm = recoverUnsupportedStatement(statementStart);
3311+
ifElseStatement = null;
33103312
} else if ( errorRecovery ) {
33113313
parseErrors.add(ex);
33123314
error_skipto(ST_SEMICOLON);
@@ -3607,6 +3609,7 @@ Block Block() #Block : {
36073609
}
36083610

36093611
Statements Statements() #Statements: {
3612+
Token statementStart;
36103613
Statements stmts = new Statements();
36113614
IfElseStatement ifElseStatement = null;
36123615
Statement stm = null;
@@ -3617,6 +3620,7 @@ Statements Statements() #Statements: {
36173620
(
36183621
(
36193622
( <ST_SEMICOLON> )*
3623+
{ statementStart = getToken(0); }
36203624

36213625
// todo: allow also first statement to be an `UnsupportedStatement`
36223626
try {
@@ -3652,7 +3656,7 @@ Statements Statements() #Statements: {
36523656

36533657
} catch (ParseException ex) {
36543658
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3655-
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "", error_skipto(ST_SEMICOLON) );
3659+
UnsupportedStatement unsupportedStatement = recoverUnsupportedStatement(statementStart);
36563660
if (!unsupportedStatement.isEmpty()) {
36573661
stmts.add( unsupportedStatement );
36583662
}
@@ -3669,6 +3673,7 @@ Statements Statements() #Statements: {
36693673

36703674
( LOOKAHEAD(2)
36713675
( <ST_SEMICOLON> )*
3676+
{ statementStart = getToken(0); }
36723677
try {
36733678
(
36743679
<K_IF> condition=Condition()
@@ -3687,7 +3692,7 @@ Statements Statements() #Statements: {
36873692
) { stmts.add(stm); stm=null; }
36883693
} catch (ParseException ex) {
36893694
if ( getAsBoolean(Feature.allowUnsupportedStatements) ) {
3690-
UnsupportedStatement unsupportedStatement = new UnsupportedStatement( stm!=null ? stm.toString() : "" , error_skipto(ST_SEMICOLON) );
3695+
UnsupportedStatement unsupportedStatement = recoverUnsupportedStatement(statementStart);
36913696
if (!unsupportedStatement.isEmpty()) {
36923697
stmts.add( unsupportedStatement );
36933698
}
@@ -3726,6 +3731,18 @@ Statements Statements() #Statements: {
37263731
}
37273732
}
37283733

3734+
JAVACODE
3735+
UnsupportedStatement recoverUnsupportedStatement(Token statementStart) {
3736+
error_skipto(ST_SEMICOLON);
3737+
List<String> tokenImages = new ArrayList<String>();
3738+
Token statementEnd = getToken(0);
3739+
// Include tokens consumed by a failed production, without reserializing a partial AST.
3740+
for (Token t = statementStart.next; t != statementEnd; t = t.next) {
3741+
tokenImages.add(t.image);
3742+
}
3743+
return new UnsupportedStatement(tokenImages);
3744+
}
3745+
37293746
JAVACODE
37303747
List<String> error_skipto(int kind) {
37313748
ArrayList<String> tokenImages = new ArrayList<String>();

‎src/test/java/net/sf/jsqlparser/statement/UnsupportedStatementRecoveryTest.java‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
/*-
2+
* #%L
3+
* JSQLParser library
4+
* %%
5+
* Copyright (C) 2004 - 2026 JSQLParser
6+
* %%
7+
* Dual licensed under GNU LGPL 2.1 or Apache License 2.0
8+
* #L%
9+
*/
10+
package net.sf.jsqlparser.statement;
11+
12+
import static org.junit.jupiter.api.Assertions.assertEquals;
13+
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
14+
import static org.junit.jupiter.api.Assertions.assertNull;
15+
import static org.junit.jupiter.api.Assertions.assertThrowsExactly;
16+
import static org.junit.jupiter.api.Assertions.assertTrue;
17+
18+
import java.io.ByteArrayInputStream;
19+
import java.io.StringReader;
20+
import java.nio.charset.StandardCharsets;
21+
import java.util.stream.Stream;
22+
import net.sf.jsqlparser.JSQLParserException;
23+
import net.sf.jsqlparser.parser.CCJSqlParser;
24+
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
25+
import net.sf.jsqlparser.parser.ParseException;
26+
import net.sf.jsqlparser.parser.StreamProvider;
27+
import net.sf.jsqlparser.statement.select.Select;
28+
import net.sf.jsqlparser.statement.update.Update;
29+
import net.sf.jsqlparser.test.TestUtils;
30+
import org.junit.jupiter.api.Test;
31+
import org.junit.jupiter.params.ParameterizedTest;
32+
import org.junit.jupiter.params.provider.Arguments;
33+
import org.junit.jupiter.params.provider.MethodSource;
34+
import org.junit.jupiter.params.provider.ValueSource;
35+
36+
class UnsupportedStatementRecoveryTest {
37+
static Stream<Arguments> incompleteStatements() {
38+
return Stream.of(
39+
Arguments.of("INSERT INTO t (a", "INSERT INTO t ( a"),
40+
Arguments.of("UPDATE t SET", "UPDATE t SET"),
41+
Arguments.of("select * from", "select * from"),
42+
Arguments.of("insert into t values (1,", "insert into t values ( 1 ,"),
43+
Arguments.of("update \"Mixed\" set a = 'x;y',", "update \"Mixed\" set a = 'x;y' ,"),
44+
Arguments.of("select 1 +", "select 1 +"),
45+
Arguments.of("select 1 unexpected trailing", "select 1 unexpected trailing"),
46+
Arguments.of("select f(1, 2) from t where x =",
47+
"select f ( 1 , 2 ) from t where x ="),
48+
Arguments.of("IF x > 0 UPDATE t SET", "IF x > 0 UPDATE t SET"));
49+
}
50+
51+
static Stream<Arguments> failedStatementProductions() {
52+
return Stream.of(
53+
Arguments.of("INSERT INTO t (a", "INSERT INTO t ( a"),
54+
Arguments.of("UPDATE t SET", "UPDATE t SET"),
55+
Arguments.of("insert into t values (1,", "insert into t values ( 1 ,"),
56+
Arguments.of("select 1 +", "select 1 +"),
57+
Arguments.of("select f(1, 2) from t where x =",
58+
"select f ( 1 , 2 ) from t where x ="),
59+
Arguments.of("IF x > 0 UPDATE t SET", "IF x > 0 UPDATE t SET"));
60+
}
61+
62+
static Stream<Arguments> incrementallyParsedPrefixes() {
63+
return Stream.of(
64+
Arguments.of("select * from", Select.class, "SELECT *", "from"),
65+
Arguments.of("update \"Mixed\" set a = 'x;y',", Update.class,
66+
"UPDATE \"Mixed\" SET a = 'x;y'", ","),
67+
Arguments.of("select 1 unexpected trailing", Select.class,
68+
"SELECT 1 unexpected", "trailing"));
69+
}
70+
71+
@ParameterizedTest
72+
@MethodSource("incompleteStatements")
73+
void singleStatementPreservesConsumedTokens(String sql, String expected) throws Exception {
74+
for (boolean recovery : new boolean[] {false, true}) {
75+
for (String delimiter : new String[] {"", ";"}) {
76+
Statement statement = CCJSqlParserUtil.parse(sql + delimiter,
77+
p -> p.withUnsupportedStatements(true).withErrorRecovery(recovery));
78+
assertInstanceOf(UnsupportedStatement.class, statement);
79+
assertEquals(expected, statement.toString());
80+
TestUtils.assertStatementCanBeDeparsedAs(statement, expected, true);
81+
}
82+
}
83+
}
84+
85+
@ParameterizedTest
86+
@MethodSource("failedStatementProductions")
87+
void statementListPreservesTextAndNextStatement(String sql, String expected) throws Exception {
88+
for (boolean recovery : new boolean[] {false, true}) {
89+
for (String prefix : new String[] {"", "SELECT 0;", ";;SELECT 0;;;"}) {
90+
int index = prefix.contains("SELECT") ? 1 : 0;
91+
CCJSqlParser parser = CCJSqlParserUtil.newParser(prefix + sql + ";;SELECT 2;")
92+
.withUnsupportedStatements(true).withErrorRecovery(recovery);
93+
Statements statements = parser.Statements();
94+
assertEquals(index + 2, statements.size());
95+
if (index > 0) {
96+
assertInstanceOf(Select.class, statements.get(0));
97+
assertEquals("SELECT 0", statements.get(0).toString());
98+
}
99+
assertInstanceOf(UnsupportedStatement.class, statements.get(index));
100+
assertEquals(expected, statements.get(index).toString());
101+
assertInstanceOf(Select.class, statements.get(index + 1));
102+
assertEquals("SELECT 2", statements.get(index + 1).toString());
103+
assertTrue(parser.getParseErrors().isEmpty());
104+
}
105+
}
106+
}
107+
108+
@ParameterizedTest
109+
@MethodSource("incompleteStatements")
110+
void firstStatementInListPreservesText(String sql, String expected) throws Exception {
111+
for (String delimiter : new String[] {"", ";", ";SELECT 2;"}) {
112+
Statements statements = CCJSqlParserUtil.parseStatements(sql + delimiter,
113+
p -> p.withUnsupportedStatements(true));
114+
assertEquals(delimiter.contains("SELECT") ? 2 : 1, statements.size());
115+
assertInstanceOf(UnsupportedStatement.class, statements.get(0));
116+
assertEquals(expected, statements.get(0).toString());
117+
if (statements.size() > 1) {
118+
assertEquals("SELECT 2",
119+
assertInstanceOf(Select.class, statements.get(1)).toString());
120+
}
121+
}
122+
}
123+
124+
@ParameterizedTest
125+
@MethodSource("incrementallyParsedPrefixes")
126+
void laterStatementKeepsIncrementallyParsedPrefix(String sql, Class<?> expectedType,
127+
String expectedPrefix, String expectedSuffix) throws Exception {
128+
for (boolean recovery : new boolean[] {false, true}) {
129+
Statements statements =
130+
CCJSqlParserUtil.parseStatements("SELECT 0;" + sql + ";SELECT 2;",
131+
p -> p.withUnsupportedStatements(true).withErrorRecovery(recovery));
132+
assertEquals(4, statements.size());
133+
assertEquals("SELECT 0", assertInstanceOf(Select.class, statements.get(0)).toString());
134+
assertInstanceOf(expectedType, statements.get(1));
135+
assertEquals(expectedPrefix, statements.get(1).toString());
136+
assertEquals(expectedSuffix,
137+
assertInstanceOf(UnsupportedStatement.class, statements.get(2)).toString());
138+
assertEquals("SELECT 2", assertInstanceOf(Select.class, statements.get(3)).toString());
139+
}
140+
}
141+
142+
@ParameterizedTest
143+
@MethodSource("incompleteStatements")
144+
void configuredReaderAndInputStreamPreserveText(String sql, String expected) throws Exception {
145+
for (boolean multiple : new boolean[] {false, true}) {
146+
for (boolean recovery : new boolean[] {false, true}) {
147+
CCJSqlParser readerParser =
148+
new CCJSqlParser(new StreamProvider(new StringReader(sql)))
149+
.withUnsupportedStatements(true).withErrorRecovery(recovery);
150+
Statement readerStatement = multiple ? readerParser.Statements().get(0)
151+
: readerParser.Statement();
152+
assertEquals(expected, assertInstanceOf(UnsupportedStatement.class,
153+
readerStatement).toString());
154+
CCJSqlParser streamParser = CCJSqlParserUtil.newParser(
155+
new ByteArrayInputStream(sql.getBytes(StandardCharsets.UTF_8)), "UTF-8")
156+
.withUnsupportedStatements(true).withErrorRecovery(recovery);
157+
Statement streamStatement = multiple ? streamParser.Statements().get(0)
158+
: streamParser.Statement();
159+
assertEquals(expected, assertInstanceOf(UnsupportedStatement.class,
160+
streamStatement).toString());
161+
}
162+
}
163+
}
164+
165+
@ParameterizedTest
166+
@ValueSource(strings = {"INSERT INTO t (a", "UPDATE t SET", "SELECT * FROM"})
167+
void disabledUnsupportedStatementsKeepsErrors(String sql) throws Exception {
168+
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parse(sql));
169+
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parseStatements(sql));
170+
assertThrowsExactly(JSQLParserException.class,
171+
() -> CCJSqlParserUtil.parse(new StringReader(sql)));
172+
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parse(
173+
new ByteArrayInputStream(sql.getBytes(StandardCharsets.UTF_8))));
174+
}
175+
176+
@ParameterizedTest
177+
@ValueSource(strings = {"INSERT INTO t (a", "UPDATE t SET", "SELECT * FROM"})
178+
void disabledUnsupportedStatementsKeepsNullRecovery(String sql) throws Exception {
179+
CCJSqlParser parser = CCJSqlParserUtil.newParser(sql).withErrorRecovery(true);
180+
assertNull(parser.Statement());
181+
assertEquals(1, parser.getParseErrors().size());
182+
parser = CCJSqlParserUtil.newParser(sql + "; SELECT 2").withErrorRecovery(true);
183+
Statements statements = parser.Statements();
184+
assertEquals(2, statements.size());
185+
assertNull(statements.get(0));
186+
assertEquals("SELECT 2", assertInstanceOf(Select.class, statements.get(1)).toString());
187+
assertEquals(1, parser.getParseErrors().size());
188+
}
189+
190+
@Test
191+
void validStatementRemainsTypedAndRoundTrips() throws Exception {
192+
String sql = "SELECT a FROM t WHERE a = 1";
193+
for (boolean unsupported : new boolean[] {false, true}) {
194+
for (boolean recovery : new boolean[] {false, true}) {
195+
assertInstanceOf(Select.class, TestUtils.assertSqlCanBeParsedAndDeparsed(sql, true,
196+
p -> p.withUnsupportedStatements(unsupported).withErrorRecovery(recovery)));
197+
}
198+
}
199+
}
200+
201+
@Test
202+
void validIfElseRemainsTypedAndRoundTrips() throws Exception {
203+
String sql = "IF x > 0 SELECT 1 ELSE SELECT 2";
204+
for (boolean unsupported : new boolean[] {false, true}) {
205+
for (boolean recovery : new boolean[] {false, true}) {
206+
assertInstanceOf(IfElseStatement.class,
207+
TestUtils.assertSqlCanBeParsedAndDeparsed(sql, true,
208+
p -> p.withUnsupportedStatements(unsupported)
209+
.withErrorRecovery(recovery)));
210+
}
211+
}
212+
}
213+
214+
@Test
215+
void singleStatementStillRejectsMultipleStatements() {
216+
assertThrowsExactly(JSQLParserException.class,
217+
() -> CCJSqlParserUtil.parse("SELECT 1; SELECT 2"));
218+
}
219+
220+
@ParameterizedTest
221+
@ValueSource(strings = {"select 1; select 2", "IF x > 0 SELECT 1 trailing garbage"})
222+
void singleStatementPreservesTrailingInput(String sql) throws Exception {
223+
Statement statement = CCJSqlParserUtil.parse(sql, p -> p.withUnsupportedStatements(true));
224+
assertInstanceOf(UnsupportedStatement.class, statement);
225+
assertEquals(sql.replace(";", " ;"), statement.toString());
226+
}
227+
228+
@Test
229+
void lexicalErrorsAreNotSwallowed() {
230+
assertThrowsExactly(JSQLParserException.class, () -> CCJSqlParserUtil.parse(
231+
"UPDATE t SET a = 'unterminated", p -> p.withUnsupportedStatements(true)));
232+
}
233+
234+
@Test
235+
void directSingleStatementDoesNotRecover() {
236+
CCJSqlParser parser = CCJSqlParserUtil.newParser("UPDATE t SET")
237+
.withUnsupportedStatements(true).withErrorRecovery(true);
238+
assertThrowsExactly(ParseException.class, parser::SingleStatement);
239+
}
240+
}

0 commit comments

Comments
 (0)