feat(parser): support double quoted string literals behind allowDoubleQuotedStrings (#2513)

One lexeme, two readings: ANSI/Postgres/Oracle and SQL Server quote
identifiers with double quotes, while BigQuery, Spark/Databricks and
MySQL default sql_mode read them as string literals. The switch rewrites
the token kind in the S_QUOTED_IDENTIFIER action (the square bracket
machinery below), StringValue keeps the double quote on round-trip via
its quoteStr. MYSQL and MARIADB presets carry the switch. Implements
item 1 of #2512.

Author: fudianchn

src/main/java/net/sf/jsqlparser/expression/StringValue.java

@@ -36,6 +36,12 @@ public StringValue(String escapedValue) {
                 && escapedValue.endsWith("'")) {
             value = escapedValue.substring(1, escapedValue.length() - 1);
             return;
+        } else if (escapedValue.length() >= 2 && escapedValue.startsWith("\"")
+                && escapedValue.endsWith("\"")) {
+            // double quoted String Literals (Feature.allowDoubleQuotedStrings)
+            value = escapedValue.substring(1, escapedValue.length() - 1);
+            quoteStr = "\"";
+            return;
         } else if (escapedValue.length() >= 4 && escapedValue.startsWith("$$")
                 && escapedValue.endsWith("$$")) {
             value = escapedValue.substring(2, escapedValue.length() - 2);

src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java

@@ -26,8 +26,10 @@ public abstract class AbstractJSqlParser<P> {
 
     public enum Dialect {
         ANSI_SQL, ORACLE, MYSQL(Feature.allowBackslashEscapeCharacter,
-                Feature.allowHashLineComments), MARIADB(Feature.allowBackslashEscapeCharacter,
-                        Feature.allowHashLineComments), SQLSERVER(
+                Feature.allowHashLineComments,
+                Feature.allowDoubleQuotedStrings), MARIADB(Feature.allowBackslashEscapeCharacter,
+                        Feature.allowHashLineComments,
+                        Feature.allowDoubleQuotedStrings), SQLSERVER(
                                 Feature.allowSquareBracketQuotation), POSTGRESQL, H2, EXASOL;
 
         private final Set<Feature> lexerFeatures;
@@ -98,6 +100,14 @@ public P withBackslashEscapeCharacter(boolean allowBackslashEscapeCharacter) {
         return withFeature(Feature.allowBackslashEscapeCharacter, allowBackslashEscapeCharacter);
     }
 
+    public P withDoubleQuotedStrings() {
+        return withFeature(Feature.allowDoubleQuotedStrings, true);
+    }
+
+    public P withDoubleQuotedStrings(boolean allowDoubleQuotedStrings) {
+        return withFeature(Feature.allowDoubleQuotedStrings, allowDoubleQuotedStrings);
+    }
+
     public P withHashLineComments() {
         return withFeature(Feature.allowHashLineComments, true);
     }

src/main/java/net/sf/jsqlparser/parser/feature/Feature.java

@@ -795,6 +795,12 @@ public enum Feature {
      */
     allowBackslashEscapeCharacter(false),
 
+    /**
+     * allows double quoted String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode);
+     * disabled by default, where double quotes stay quoted identifiers (ANSI SQL)
+     */
+    allowDoubleQuotedStrings(false),
+
     /**
      * allows MySQL `#` line comments; disabled by default, where a lone `#` stays the binary
      * operator (#2507: PostgreSQL bitwise XOR / geometric intersection)

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -2077,6 +2077,11 @@ TOKEN:
             matchedToken.kind = squaredBracketOpenIndex;
             input_stream.backup(image.length() - 1);
         }
+        if ( configuration.getAsBoolean(Feature.allowDoubleQuotedStrings)
+            && matchedToken.image.charAt(0) == '"' ) {
+            // `charLiteralIndex` defined in TokenManagerDeclaration above
+            matchedToken.kind = charLiteralIndex;
+        }
     }
 }
 

src/site/sphinx/usage.rst

@@ -279,7 +279,7 @@ Define the Parser Features
 
 JSQLParser interprets Squared Brackets ``[..]`` as Arrays, which does not work with MS SQL Server and T-SQL. Please use the Parser Features to instruct JSQLParser to read Squared Brackets as Quotes instead.
 
-JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature.
+JSQLParser allows for standard compliant Single Quote ``'..`` Escaping. Additional Back-slash ``\..`` Escaping needs to be activated by setting the ``BackSlashEscapeCharacter`` parser feature. JSQLParser reads Double Quotes ``".."`` as quoted identifiers (ANSI SQL); reading them as String Literals (BigQuery, Spark/Databricks, MySQL default sql_mode) needs the ``DoubleQuotedStrings`` parser feature.
 
 Additionally there are Features to control the Parser's effort at the cost of the performance.
 
@@ -319,7 +319,7 @@ Additionally there are Features to control the Parser's effort at the cost of th
                 .withBackslashEscapeCharacter(true)
     );
 
-Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter`` and ``withHashLineComments`` (both MySQL and MariaDB syntax), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. Features set explicitly after the dialect preset win over the preset.
+Instead of turning the individual Parser Features on one by one, a ``Dialect`` preset selects the features of that database dialect: ``withDialect(Dialect.MYSQL)`` turns on ``withBackslashEscapeCharacter``, ``withHashLineComments`` and ``withDoubleQuotedStrings`` (MySQL and MariaDB syntax, the latter for the default sql_mode), ``withDialect(Dialect.SQLSERVER)`` turns on ``withSquareBracketQuotation``. Features set explicitly after the dialect preset win over the preset.
 
 .. code-block:: java
 

src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java

@@ -12,6 +12,7 @@
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.LongValue;
+import net.sf.jsqlparser.expression.StringValue;
 import net.sf.jsqlparser.expression.operators.arithmetic.Addition;
 import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication;
 import net.sf.jsqlparser.expression.operators.relational.ParenthesedExpressionList;
@@ -20,6 +21,7 @@
 import net.sf.jsqlparser.statement.Statements;
 import net.sf.jsqlparser.statement.UnsupportedStatement;
 import net.sf.jsqlparser.statement.select.PlainSelect;
+import net.sf.jsqlparser.statement.select.Select;
 import net.sf.jsqlparser.statement.select.TableStatement;
 import net.sf.jsqlparser.test.MemoryLeakVerifier;
 import net.sf.jsqlparser.test.TestUtils;
@@ -44,6 +46,7 @@
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class CCJSqlParserUtilTest {
 
@@ -611,4 +614,45 @@ public void testDialectPresetSwitchOverrides() throws Exception {
                                 .withDialect(AbstractJSqlParser.Dialect.MYSQL))
                         .toString());
     }
+
+    @Test
+    public void testDoubleQuotedStringsFeature() throws Exception {
+        // default (flag off): double quotes are quoted identifiers, unchanged
+        PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil
+                .parse("SELECT \"not an identifier\"")).getSelectBody();
+        assertTrue(select.getSelectItems().get(0).getExpression() instanceof Column);
+        // on: the BigQuery/Spark/MySQL-default reading, double quotes are
+        // string literals, the quote kept on round-trip
+        select = (PlainSelect) ((Select) CCJSqlParserUtil.parse("SELECT \"not an identifier\"",
+                p -> p.withDoubleQuotedStrings(true))).getSelectBody();
+        Expression expression = select.getSelectItems().get(0).getExpression();
+        assertTrue(expression instanceof StringValue);
+        assertEquals("not an identifier", ((StringValue) expression).getValue());
+        assertEquals("SELECT \"not an identifier\"", select.toString());
+        // empty string and doubled quotes take the single-quote treatment
+        assertEquals("SELECT \"\"", CCJSqlParserUtil
+                .parse("SELECT \"\"", p -> p.withDoubleQuotedStrings(true)).toString());
+        assertEquals("SELECT \"a\"\"b\"", CCJSqlParserUtil
+                .parse("SELECT \"a\"\"b\"", p -> p.withDoubleQuotedStrings(true)).toString());
+        // identifier positions: a `"..."` token follows the existing
+        // string-as-table branch (`FROM 'file.csv'`), the same leniency
+        // single quotes already have; MySQL itself would error here
+        assertEquals("SELECT * FROM \"t\"", CCJSqlParserUtil
+                .parse("SELECT * FROM \"t\"", p -> p.withDoubleQuotedStrings(true)).toString());
+    }
+
+    @Test
+    public void testDoubleQuotedStringsPreset() throws Exception {
+        // MYSQL and MARIADB presets carry the switch (default sql_mode reading)
+        for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {
+                AbstractJSqlParser.Dialect.MYSQL, AbstractJSqlParser.Dialect.MARIADB}) {
+            PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil
+                    .parse("SELECT \"abc\"", p -> p.withDialect(dialect))).getSelectBody();
+            assertTrue(select.getSelectItems().get(0).getExpression() instanceof StringValue);
+        }
+        // the identifier-default dialects keep the quoted identifier reading
+        PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse("SELECT \"abc\"",
+                p -> p.withDialect(AbstractJSqlParser.Dialect.SQLSERVER))).getSelectBody();
+        assertTrue(select.getSelectItems().get(0).getExpression() instanceof Column);
+    }
 }