Parse PostgreSQL Unicode escape string constants

src/main/java/net/sf/jsqlparser/expression/StringValue.java

@@ -25,6 +25,9 @@ public final class StringValue extends ASTNodeAccessImpl implements Expression {
     public static final List<String> ALLOWED_PREFIXES =
             Arrays.asList("N", "U", "E", "R", "B", "RB", "_utf8", "Q");
 
+    /** The prefix of a PostgreSQL Unicode escape string, as in {@code U&'\0041'}. */
+    private static final String UNICODE_ESCAPE_PREFIX = "U&";
+
     /**
      * How a backslash in the literal body was read. The parser records the escaping of the
      * configuration that lexed the literal; constructed values default to {@link #STANDARD}.
@@ -35,13 +38,21 @@ public enum Escaping {
         /** MySQL/MariaDB backslash escapes, for example {@code \n}, {@code \'} and {@code \\}. */
         MYSQL_BACKSLASH,
         /** PostgreSQL escape string escapes, for example {@code E'\n'} and {@code E'\x41'}. */
-        POSTGRESQL_ESCAPE_STRING
+        POSTGRESQL_ESCAPE_STRING,
+        /**
+         * PostgreSQL Unicode escapes of a {@code U&'...'} string: the escape character followed by
+         * four hexadecimal digits or by {@code +} and six, and a doubled escape character. The
+         * escape character is a backslash unless {@link #getUnicodeEscapeCharacter()} names another
+         * one.
+         */
+        POSTGRESQL_UNICODE_ESCAPE
     }
 
     private String value = "";
     private String prefix = null;
     private String quoteStr = "'";
     private Escaping escaping = Escaping.STANDARD;
+    private Character unicodeEscapeCharacter = null;
 
     public StringValue() {
         // empty constructor
@@ -70,6 +81,10 @@ public StringValue(String escapedValue) {
             return;
         }
 
+        if (readUnicodeEscapeString(escapedValue)) {
+            return;
+        }
+
         if (escapedValue.length() > 2) {
             for (String p : ALLOWED_PREFIXES) {
                 if (escapedValue.length() > p.length()
@@ -85,6 +100,46 @@ public StringValue(String escapedValue) {
         value = escapedValue;
     }
 
+    /**
+     * Reads {@code U&'body'} with an optional {@code UESCAPE 'c'} clause. Returns false, leaving
+     * this value unchanged, when the text has another shape.
+     */
+    private boolean readUnicodeEscapeString(String text) {
+        if (!text.regionMatches(true, 0, UNICODE_ESCAPE_PREFIX + "'", 0, 3)) {
+            return false;
+        }
+        int close = 3;
+        while (close < text.length()) {
+            if (text.charAt(close) != '\'') {
+                close++;
+            } else if (close + 1 < text.length() && text.charAt(close + 1) == '\'') {
+                close += 2;
+            } else {
+                break;
+            }
+        }
+        if (close >= text.length()) {
+            return false;
+        }
+        String clause = text.substring(close + 1).trim();
+        Character escape = null;
+        if (!clause.isEmpty()) {
+            if (!clause.regionMatches(true, 0, "UESCAPE", 0, 7)) {
+                return false;
+            }
+            String quoted = clause.substring(7).trim();
+            if (quoted.length() != 3 || quoted.charAt(0) != '\'' || quoted.charAt(2) != '\'') {
+                return false;
+            }
+            escape = quoted.charAt(1);
+        }
+        prefix = UNICODE_ESCAPE_PREFIX;
+        value = text.substring(3, close);
+        escaping = Escaping.POSTGRESQL_UNICODE_ESCAPE;
+        unicodeEscapeCharacter = escape;
+        return true;
+    }
+
     /**
      * Returns the opening PostgreSQL dollar-quote delimiter, or null if there is none. A tag
      * follows unquoted identifier rules, excluding dollar signs. This method does not require the
@@ -149,17 +204,47 @@ public StringValue withEscaping(Escaping escaping) {
         return this;
     }
 
+    /**
+     * Returns the escape character named by the {@code UESCAPE} clause of a PostgreSQL Unicode
+     * escape string, or null when the literal has no such clause and the escape character is a
+     * backslash.
+     */
+    public Character getUnicodeEscapeCharacter() {
+        return unicodeEscapeCharacter;
+    }
+
+    /**
+     * Sets the escape character rendered as {@code UESCAPE 'c'} after the literal; null removes the
+     * clause. It applies to {@link Escaping#POSTGRESQL_UNICODE_ESCAPE} values, whose
+     * {@link #getNotExcapedValue()} rejects a character that PostgreSQL rejects.
+     */
+    public void setUnicodeEscapeCharacter(Character unicodeEscapeCharacter) {
+        this.unicodeEscapeCharacter = unicodeEscapeCharacter;
+    }
+
+    public StringValue withUnicodeEscapeCharacter(Character unicodeEscapeCharacter) {
+        setUnicodeEscapeCharacter(unicodeEscapeCharacter);
+        return this;
+    }
+
     /**
      * Returns the literal's characters with its escapes resolved: a doubled quote character and,
-     * depending on {@link #getEscaping()}, backslash escape sequences. PostgreSQL octal and
-     * hexadecimal escapes denote bytes, which are decoded as UTF-8. Dollar-quoted bodies are
+     * depending on {@link #getEscaping()}, backslash or Unicode escape sequences. PostgreSQL octal
+     * and hexadecimal escapes denote bytes, which are decoded as UTF-8. Dollar-quoted bodies are
      * returned unchanged.
+     *
+     * @throws IllegalArgumentException if a {@link Escaping#POSTGRESQL_UNICODE_ESCAPE} value has an
+     *         escape that PostgreSQL rejects; the parser rejects such literals
      */
     public String getNotExcapedValue() {
         if (quoteStr != null && quoteStr.startsWith("$")) {
             return value;
         }
         char quote = "\"".equals(quoteStr) ? '"' : '\'';
+        if (escaping == Escaping.POSTGRESQL_UNICODE_ESCAPE) {
+            return unescapeUnicode(value, quote,
+                    unicodeEscapeCharacter == null ? '\\' : unicodeEscapeCharacter);
+        }
         Unescaped result = new Unescaped(value.length());
         int i = 0;
         while (i < value.length()) {
@@ -270,6 +355,82 @@ private static int appendNumber(Unescaped result, String text, int start, int ra
         return end;
     }
 
+    /**
+     * Resolves PostgreSQL Unicode escapes the way the server does: {@code escape} followed by four
+     * hexadecimal digits, or by {@code +} and six, names a code point, and a UTF-16 surrogate pair
+     * must be written as two consecutive escapes. A doubled escape or quote character stands for
+     * itself.
+     */
+    private static String unescapeUnicode(String text, char quote, char escape) {
+        checkUnicodeEscapeCharacter(escape);
+        UnicodeUnescaped result = new UnicodeUnescaped(text.length());
+        int i = 0;
+        while (i < text.length()) {
+            char c = text.charAt(i);
+            boolean doubled = i + 1 < text.length() && text.charAt(i + 1) == c;
+            if (c == escape && !doubled) {
+                i = appendUnicodeEscape(result, text, i, escape);
+            } else {
+                result.append(c, i);
+                i += doubled && (c == escape || c == quote) ? 2 : 1;
+            }
+        }
+        return result.finish(text.length());
+    }
+
+    private static void checkUnicodeEscapeCharacter(char escape) {
+        if (escape > 0x7F || isHexDigit(escape) || "+'\" \t\n\r\f\u000B".indexOf(escape) >= 0) {
+            throw new IllegalArgumentException("invalid Unicode escape character '" + escape + "'");
+        }
+    }
+
+    /**
+     * Appends the code point of the escape at {@code position}, {@code escape} followed by four
+     * hexadecimal digits or by {@code +} and six, and returns the next index.
+     */
+    private static int appendUnicodeEscape(UnicodeUnescaped result, String text, int position,
+            char escape) {
+        boolean plus = position + 1 < text.length() && text.charAt(position + 1) == '+';
+        int start = plus ? position + 2 : position + 1;
+        int end = start + (plus ? 6 : 4);
+        int codePoint = parseHex(text, start, end);
+        if (codePoint < 0) {
+            throw new IllegalArgumentException("invalid Unicode escape at position " + position
+                    + ": Unicode escapes must be " + escape + "XXXX or " + escape + "+XXXXXX");
+        }
+        if (codePoint == 0 || codePoint > Character.MAX_CODE_POINT) {
+            throw new IllegalArgumentException(
+                    "invalid Unicode escape value at position " + position);
+        }
+        result.appendCodePoint(codePoint, position);
+        return end;
+    }
+
+    private static IllegalArgumentException invalidSurrogatePair(int position) {
+        return new IllegalArgumentException(
+                "invalid Unicode surrogate pair at position " + position);
+    }
+
+    private static boolean isHexDigit(char c) {
+        return c >= '0' && c <= '9' || c >= 'a' && c <= 'f' || c >= 'A' && c <= 'F';
+    }
+
+    /** Returns the ASCII hexadecimal number spelled by text[start, end), or -1. */
+    private static int parseHex(String text, int start, int end) {
+        if (end > text.length()) {
+            return -1;
+        }
+        int number = 0;
+        for (int i = start; i < end; i++) {
+            char c = text.charAt(i);
+            if (!isHexDigit(c)) {
+                return -1;
+            }
+            number = number * 16 + Character.digit(c, 16);
+        }
+        return number;
+    }
+
     /** Collects unescaped characters; consecutive escaped bytes are decoded as UTF-8. */
     private static final class Unescaped {
         private final StringBuilder chars;
@@ -307,14 +468,65 @@ public String toString() {
         }
     }
 
+    /**
+     * Collects resolved Unicode escapes. A UTF-16 surrogate pair must be written as two consecutive
+     * escapes, as PostgreSQL requires.
+     */
+    private static final class UnicodeUnescaped {
+        private final StringBuilder chars;
+        private char highSurrogate;
+
+        UnicodeUnescaped(int capacity) {
+            chars = new StringBuilder(capacity);
+        }
+
+        void append(char c, int position) {
+            requireNoPendingSurrogate(position);
+            chars.append(c);
+        }
+
+        void appendCodePoint(int codePoint, int position) {
+            if (codePoint >= Character.MIN_LOW_SURROGATE
+                    && codePoint <= Character.MAX_LOW_SURROGATE) {
+                if (highSurrogate == 0) {
+                    throw invalidSurrogatePair(position);
+                }
+                chars.appendCodePoint(Character.toCodePoint(highSurrogate, (char) codePoint));
+                highSurrogate = 0;
+                return;
+            }
+            requireNoPendingSurrogate(position);
+            if (codePoint >= Character.MIN_HIGH_SURROGATE
+                    && codePoint <= Character.MAX_HIGH_SURROGATE) {
+                highSurrogate = (char) codePoint;
+            } else {
+                chars.appendCodePoint(codePoint);
+            }
+        }
+
+        String finish(int position) {
+            requireNoPendingSurrogate(position);
+            return chars.toString();
+        }
+
+        private void requireNoPendingSurrogate(int position) {
+            if (highSurrogate != 0) {
+                throw invalidSurrogatePair(position);
+            }
+        }
+    }
+
     @Override
     public <T, S> T accept(ExpressionVisitor<T> expressionVisitor, S context) {
         return expressionVisitor.visit(this, context);
     }
 
     @Override
     public String toString() {
-        return (prefix != null ? prefix : "") + quoteStr + value + quoteStr;
+        String unicodeEscape = unicodeEscapeCharacter != null
+                ? " UESCAPE '" + unicodeEscapeCharacter + "'"
+                : "";
+        return (prefix != null ? prefix : "") + quoteStr + value + quoteStr + unicodeEscape;
     }
 
     public StringValue withPrefix(String prefix) {

src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java

@@ -715,6 +715,10 @@ public <S> StringBuilder visit(StringValue stringValue, S context) {
         }
         builder.append(stringValue.getQuoteStr()).append(stringValue.getValue())
                 .append(stringValue.getQuoteStr());
+        if (stringValue.getUnicodeEscapeCharacter() != null) {
+            builder.append(" UESCAPE '").append(stringValue.getUnicodeEscapeCharacter())
+                    .append('\'');
+        }
 
         return builder;
     }

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -2573,7 +2573,9 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
     /** Creates a string literal that remembers how this configuration lexed its backslashes. */
     private StringValue stringValue(Token token) {
         StringValue value = new StringValue(token.image);
-        if (value.getQuoteStr() == null || value.getQuoteStr().startsWith("$")) {
+        // a U&'...' string brings its own escaping
+        if (value.getQuoteStr() == null || value.getQuoteStr().startsWith("$")
+                || value.getEscaping() == StringValue.Escaping.POSTGRESQL_UNICODE_ESCAPE) {
             return value;
         }
         String dialect = getAsString(Feature.dialect);
@@ -2744,14 +2746,40 @@ TOKEN_MGR_DECLS : {
     }
 
     // Like closingQuote for a single-quoted literal after its optional prefix. Oracle q'..'
-    // literals have their own delimiters and are left unchanged.
+    // literals have their own delimiters and PostgreSQL U&'..' strings their own lexing, so
+    // both are left unchanged.
     public static int closingSingleQuote(String s, boolean backslashEscapes) {
-        if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\'') {
+        if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\''
+                || isUnicodeEscapeString(s)) {
             return -1;
         }
         return closingQuote(s, s.indexOf('\''), '\'', backslashEscapes);
     }
 
+    private static boolean isUnicodeEscapeString(String s) {
+        return s.regionMatches(true, 0, "U&'", 0, 3);
+    }
+
+    // A PostgreSQL Unicode escape string U&'...' [UESCAPE 'c'] is one literal with
+    // Dialect.POSTGRESQL, which rejects invalid escapes as the server does. Other dialects keep
+    // reading the identifier U followed by the & operator.
+    private void lexUnicodeEscapeString(Token token) {
+        if (!AbstractJSqlParser.Dialect.POSTGRESQL.name().equals(configuration.getValue(Feature.dialect))) {
+            truncateIdentifierToken(token, 1);
+            token.kind = S_IDENTIFIER;
+            return;
+        }
+        try {
+            new StringValue(token.image).getNotExcapedValue();
+        } catch (IllegalArgumentException e) {
+            // Rewind so that every later attempt to read past the literal fails as well
+            input_stream.backup(token.image.length());
+            throw new TokenMgrException("Lexical error at line " + token.beginLine + ", column "
+                    + token.beginColumn + ". Invalid Unicode escape string " + token.image + ": "
+                    + e.getMessage(), TokenMgrException.LEXICAL_ERROR);
+        }
+    }
+
     private RowPatternTokenContext rowPatternContext;
 
     private void reportMySqlCommentError() {
@@ -3722,6 +3750,10 @@ TOKEN:
                             | ("q''" (~[])* "''")
                             // | ("q'\\" (~[])* "\\'") <--- Does not work
                         )
+                        // PostgreSQL Unicode escape string with an optional UESCAPE clause, see
+                        // lexUnicodeEscapeString(). Backslashes do not escape its quotes.
+                        | "U&'" ("''" | ~["'"])* "'"
+                            ( ([" ", "\t", "\r", "\n"])* "UESCAPE" ([" ", "\t", "\r", "\n"])* "'" ~["'"] "'" )?
                      ) >
     {
         // The longest match may run past the closing quote of the active rules. A backslash
@@ -3736,6 +3768,9 @@ TOKEN:
             matchedToken.image = image.substring(0, end);
             input_stream.backup(image.length() - end);
         }
+        if (isUnicodeEscapeString(matchedToken.image)) {
+            lexUnicodeEscapeString(matchedToken);
+        }
     }
 | < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\""
         // MySQL double-quoted string with backslash escapes; trimmed below when not enabled