feat(parser): concatenate adjacent string literals behind adjacentStringLiterals (#2514)

* feat(parser): concatenate adjacent string literals behind adjacentStringLiterals

Three readings of two neighboring string literals: alias (MySQL, SQL
Server, the current behavior), concatenated across a newline (the SQL
standard and PostgreSQL), concatenated across any whitespace (GoogleSQL,
Spark/Databricks). The mode lives in a String valued Feature like
Feature.dialect, the merge is one guarded loop in PrimaryExpression
using the token line numbers for the newline rule, producing a single
merged StringValue. ANSI_SQL and POSTGRESQL presets carry NEWLINE.
Implements item 3 of #2512.

* docs(site): document adjacentStringLiterals in the parser features section

* hardening: null-safe mode check in isAdjacentStringConcat

The FeatureConfiguration seeds every configurable Feature with its
declared default, so the mode is never null today; the guard keeps the
predicate locally correct regardless.

* feat(parser): withAdjacentStringLiterals(boolean) mapping false = OFF, true = NEWLINE

stores the AdjacentStringLiterals enum constant itself as the feature value

* docs(parser): move isAdjacentStringConcat below isNamedExprListAhead, reattaching its javadoc

Author: fudianchn

src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java

@@ -24,24 +24,41 @@ public abstract class AbstractJSqlParser<P> {
     protected boolean errorRecovery = false;
     protected List<ParseException> parseErrors = new ArrayList<>();
 
+    public enum AdjacentStringLiterals {
+        OFF, NEWLINE, WHITESPACE
+    }
+
     public enum Dialect {
-        ANSI_SQL, ORACLE, MYSQL(Feature.allowBackslashEscapeCharacter,
+        ANSI_SQL(AdjacentStringLiterals.NEWLINE), ORACLE, MYSQL(AdjacentStringLiterals.OFF,
+                Feature.allowBackslashEscapeCharacter,
                 Feature.allowHashLineComments,
-                Feature.allowDoubleQuotedStrings), MARIADB(Feature.allowBackslashEscapeCharacter,
+                Feature.allowDoubleQuotedStrings), MARIADB(AdjacentStringLiterals.OFF,
+                        Feature.allowBackslashEscapeCharacter,
                         Feature.allowHashLineComments,
-                        Feature.allowDoubleQuotedStrings), SQLSERVER(
-                                Feature.allowSquareBracketQuotation), POSTGRESQL, H2, EXASOL;
+                        Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF,
+                                Feature.allowSquareBracketQuotation), POSTGRESQL(
+                                        AdjacentStringLiterals.NEWLINE), H2, EXASOL;
 
         private final Set<Feature> lexerFeatures;
+        private final AdjacentStringLiterals adjacentStringLiterals;
 
-        Dialect(Feature... lexerFeatures) {
+        Dialect(AdjacentStringLiterals adjacentStringLiterals, Feature... lexerFeatures) {
+            this.adjacentStringLiterals = adjacentStringLiterals;
             this.lexerFeatures = lexerFeatures.length == 0 ? EnumSet.noneOf(Feature.class)
                     : EnumSet.copyOf(Arrays.asList(lexerFeatures));
         }
 
+        Dialect(Feature... lexerFeatures) {
+            this(AdjacentStringLiterals.OFF, lexerFeatures);
+        }
+
         public Set<Feature> getLexerFeatures() {
             return lexerFeatures;
         }
+
+        public AdjacentStringLiterals getAdjacentStringLiterals() {
+            return adjacentStringLiterals;
+        }
     }
 
     public P withSquareBracketQuotation() {
@@ -82,12 +99,30 @@ public P withTimeOut(long timeOutMillSeconds) {
 
     public P withDialect(Dialect dialect) {
         withFeature(Feature.dialect, dialect.name());
+        if (dialect.getAdjacentStringLiterals() != AdjacentStringLiterals.OFF) {
+            withAdjacentStringLiterals(dialect.getAdjacentStringLiterals());
+        }
         for (Feature lexerFeature : dialect.getLexerFeatures()) {
             withFeature(lexerFeature, true);
         }
         return me();
     }
 
+    public P withAdjacentStringLiterals() {
+        return withAdjacentStringLiterals(AdjacentStringLiterals.NEWLINE);
+    }
+
+    public P withAdjacentStringLiterals(boolean adjacentStringLiterals) {
+        return withAdjacentStringLiterals(
+                adjacentStringLiterals ? AdjacentStringLiterals.NEWLINE
+                        : AdjacentStringLiterals.OFF);
+    }
+
+    public P withAdjacentStringLiterals(AdjacentStringLiterals adjacentStringLiterals) {
+        getConfiguration().setValue(Feature.adjacentStringLiterals, adjacentStringLiterals);
+        return me();
+    }
+
     public P withAllowedNestingDepth(int allowedNestingDepth) {
         return withFeature(Feature.allowedNestingDepth, allowedNestingDepth);
     }

src/main/java/net/sf/jsqlparser/parser/feature/Feature.java

@@ -15,6 +15,7 @@
 import net.sf.jsqlparser.expression.OracleHierarchicalExpression;
 import net.sf.jsqlparser.expression.OracleHint;
 import net.sf.jsqlparser.expression.operators.relational.SupportsOldOracleJoinSyntax;
+import net.sf.jsqlparser.parser.AbstractJSqlParser;
 import net.sf.jsqlparser.statement.Block;
 import net.sf.jsqlparser.statement.Commit;
 import net.sf.jsqlparser.statement.CreateFunctionalStatement;
@@ -801,6 +802,14 @@ public enum Feature {
      */
     allowDoubleQuotedStrings(false),
 
+    /**
+     * concatenates adjacent String Literals: NEWLINE when separated by whitespace with at least one
+     * newline (the SQL standard and PostgreSQL), WHITESPACE across any whitespace (GoogleSQL,
+     * Spark/Databricks); OFF by default, where the second literal stays an alias (MySQL, SQL
+     * Server) or fails (everywhere else)
+     */
+    adjacentStringLiterals(AbstractJSqlParser.AdjacentStringLiterals.OFF),
+
     /**
      * allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary
      * operator (#2507: PostgreSQL bitwise XOR / geometric intersection)

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -1127,6 +1127,24 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
         }
     }
 
+    /**
+     * Semantic lookahead for the adjacent string literal concatenation in PrimaryExpression:
+     * true when Feature.adjacentStringLiterals is on and the next token is a string literal
+     * meeting the mode's separation rule (WHITESPACE: any, NEWLINE: a newline between the
+     * tokens, detected through the token line numbers).
+     */
+    protected boolean isAdjacentStringConcat() {
+        String mode = getAsString(Feature.adjacentStringLiterals);
+        if (mode == null || AdjacentStringLiterals.OFF.name().equals(mode)) {
+            return false;
+        }
+        if (getToken(1).kind != S_CHAR_LITERAL) {
+            return false;
+        }
+        return AdjacentStringLiterals.WHITESPACE.name().equals(mode)
+                || getToken(1).beginLine > getToken(0).endLine;
+    }
+
     /**
      * Checks if the next token can start a condition suffix
      * (comparison, IN, BETWEEN, LIKE, IS NULL, etc.)
@@ -8319,6 +8337,7 @@ Expression PrimaryExpression() #PrimaryExpression:
     Expression timezoneRightExpr = null;
     Token token = null;
     Token sign = null;
+    Token adjacentToken = null;
     String tmp = "";
     ColDataType type = null;
     boolean not = false;
@@ -8419,6 +8438,13 @@ Expression PrimaryExpression() #PrimaryExpression:
         | LOOKAHEAD(2, {!interrupted}) (token=<K_TRUE> | token=<K_FALSE>) { retval = new BooleanValue(token.image); }
 
         | token=<S_CHAR_LITERAL> { retval = new StringValue(token.image); linkAST(retval,jjtThis); }
+        ( LOOKAHEAD({ isAdjacentStringConcat() })
+            adjacentToken=<S_CHAR_LITERAL>
+            {
+                ((StringValue) retval)
+                        .setValue(((StringValue) retval).getValue() + new StringValue(adjacentToken.image).getValue());
+            }
+        )*
 
         | "{d" token=<S_CHAR_LITERAL> "}"  { retval = new DateValue(token.image); }
 

src/site/sphinx/usage.rst

@@ -279,7 +279,7 @@ Define the Parser Features
 
 JSQLParser interprets Squared Brackets ``[..]`` as Arrays, which does not work with MS SQL Server and T-SQL. Please use the Parser Features to instruct JSQLParser to read Squared Brackets as Quotes instead.
 
-JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature.
+JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature. Adjacent String Literals concatenate optionally: only across a newline (``NEWLINE``, the SQL standard and PostgreSQL) or across any whitespace (``WHITESPACE``, GoogleSQL and Spark/Databricks); ``withAdjacentStringLiterals(true)`` selects the standard ``NEWLINE`` mode, ``false`` switches it off.
 
 Additionally there are Features to control the Parser's effort at the cost of the performance.
 
@@ -319,7 +319,7 @@ Additionally there are Features to control the Parser's effort at the cost of th
                 .withBackslashEscapeCharacter(true)
     );
 
-Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. Features set explicitly after the dialect preset win over the preset.
+Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. ``withDialect(Dialect.POSTGRESQL)`` and ``withDialect(Dialect.ANSI_SQL)`` turn on the newline rule for adjacent String Literals. Features set explicitly after the dialect preset win over the preset.
 
 .. code-block:: java
 

src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java

@@ -655,4 +655,82 @@ public void testDoubleQuotedStringsPreset() throws Exception {
                 p -> p.withDialect(AbstractJSqlParser.Dialect.SQLSERVER))).getSelectBody();
         assertTrue(select.getSelectItems().get(0).getExpression() instanceof Column);
     }
+
+    @Test
+    public void testAdjacentStringLiteralsNewline() throws Exception {
+        // default (off): the MySQL / SQL Server alias reading, unchanged
+        assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'").toString());
+        assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'").toString());
+        assertThrows(JSQLParserException.class,
+                () -> CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'"));
+        // newline mode: the standard / Postgres reading, "Two string constants
+        // that are only separated by whitespace with at least one newline are
+        // concatenated"
+        PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                p -> p.withAdjacentStringLiterals(
+                        AbstractJSqlParser.AdjacentStringLiterals.NEWLINE)))
+                .getSelectBody();
+        Expression expression = select.getSelectItems().get(0).getExpression();
+        assertTrue(expression instanceof StringValue);
+        assertEquals("ab", ((StringValue) expression).getValue());
+        assertEquals("SELECT 'ab'", select.toString());
+        // same line keeps the alias reading under newline mode
+        assertEquals("SELECT 'a' 'b'",
+                CCJSqlParserUtil.parse("SELECT 'a' 'b'",
+                        p -> p.withAdjacentStringLiterals(
+                                AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
+                        .toString());
+        // expression positions concatenate too
+        assertEquals("SELECT * FROM t WHERE x = 'ab'",
+                CCJSqlParserUtil.parse("SELECT * FROM t WHERE x = 'a'\n'b'",
+                        p -> p.withAdjacentStringLiterals(
+                                AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
+                        .toString());
+        // three parts merge into one literal
+        assertEquals("SELECT 'abc'",
+                CCJSqlParserUtil.parse("SELECT 'a'\n'b'\n'c'",
+                        p -> p.withAdjacentStringLiterals(
+                                AbstractJSqlParser.AdjacentStringLiterals.NEWLINE))
+                        .toString());
+        // the ANSI SQL and Postgres presets carry it, the MySQL preset does not
+        for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {
+                AbstractJSqlParser.Dialect.ANSI_SQL, AbstractJSqlParser.Dialect.POSTGRESQL}) {
+            assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                    p -> p.withDialect(dialect)).toString());
+        }
+        assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString());
+    }
+
+    @Test
+    public void testAdjacentStringLiteralsWhitespace() throws Exception {
+        // whitespace mode: the GoogleSQL / Spark reading, same line
+        // concatenates (their literal chunking)
+        assertEquals("SELECT 'ab'",
+                CCJSqlParserUtil.parse("SELECT 'a' 'b'",
+                        p -> p.withAdjacentStringLiterals(
+                                AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE))
+                        .toString());
+        // combined with allowDoubleQuotedStrings: the BigQuery chunking shape
+        assertEquals("SELECT \"12\"",
+                CCJSqlParserUtil.parse("SELECT \"1\" \"2\"",
+                        p -> p.withDoubleQuotedStrings(true).withAdjacentStringLiterals(
+                                AbstractJSqlParser.AdjacentStringLiterals.WHITESPACE))
+                        .toString());
+    }
+
+    @Test
+    public void testAdjacentStringLiteralsBoolean() throws Exception {
+        // true: the standard (newline) mode, the same line keeps the alias reading
+        assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                p -> p.withAdjacentStringLiterals(true)).toString());
+        assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a' 'b'",
+                p -> p.withAdjacentStringLiterals(true)).toString());
+        // false: off, the alias reading also across newlines
+        assertEquals("SELECT 'a' 'b'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                p -> p.withAdjacentStringLiterals(false)).toString());
+        // the no-arg variant enables the standard mode
+        assertEquals("SELECT 'ab'", CCJSqlParserUtil.parse("SELECT 'a'\n'b'",
+                p -> p.withAdjacentStringLiterals()).toString());
+    }
 }