feat(jdk8): move files to new folder to avoid resources compiled.

jdkSrc/jdk8/com/sun/tools/javac/parser/JavaTokenizer.java

@@ -0,0 +1,890 @@
+/*
+ * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
+import com.sun.tools.javac.util.*;
+
+import java.nio.CharBuffer;
+
+import static com.sun.tools.javac.parser.Tokens.*;
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The lexical analyzer maps an input stream consisting of
+ *  ASCII characters and Unicode escapes into a token sequence.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class JavaTokenizer {
+
+    private static final boolean scannerDebug = false;
+
+    /** Allow hex floating-point literals.
+     */
+    private boolean allowHexFloats;
+
+    /** Allow binary literals.
+     */
+    private boolean allowBinaryLiterals;
+
+    /** Allow underscores in literals.
+     */
+    private boolean allowUnderscoresInLiterals;
+
+    /** The source language setting.
+     */
+    private Source source;
+
+    /** The log to be used for error reporting.
+     */
+    private final Log log;
+
+    /** The token factory. */
+    private final Tokens tokens;
+
+    /** The token kind, set by nextToken().
+     */
+    protected TokenKind tk;
+
+    /** The token's radix, set by nextToken().
+     */
+    protected int radix;
+
+    /** The token's name, set by nextToken().
+     */
+    protected Name name;
+
+    /** The position where a lexical error occurred;
+     */
+    protected int errPos = Position.NOPOS;
+
+    /** The Unicode reader (low-level stream reader).
+     */
+    protected UnicodeReader reader;
+
+    protected ScannerFactory fac;
+
+    private static final boolean hexFloatsWork = hexFloatsWork();
+    private static boolean hexFloatsWork() {
+        try {
+            Float.valueOf("0x1.0p1");
+            return true;
+        } catch (NumberFormatException ex) {
+            return false;
+        }
+    }
+
+    /**
+     * Create a scanner from the input array.  This method might
+     * modify the array.  To avoid copying the input array, ensure
+     * that {@code inputLength < input.length} or
+     * {@code input[input.length -1]} is a white space character.
+     *
+     * @param fac the factory which created this Scanner
+     * @param buf the input, might be modified
+     * Must be positive and less than or equal to input.length.
+     */
+    protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
+        this(fac, new UnicodeReader(fac, buf));
+    }
+
+    protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
+        this(fac, new UnicodeReader(fac, buf, inputLength));
+    }
+
+    protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
+        this.fac = fac;
+        this.log = fac.log;
+        this.tokens = fac.tokens;
+        this.source = fac.source;
+        this.reader = reader;
+        this.allowBinaryLiterals = source.allowBinaryLiterals();
+        this.allowHexFloats = source.allowHexFloats();
+        this.allowUnderscoresInLiterals = source.allowUnderscoresInLiterals();
+    }
+
+    /** Report an error at the given position using the provided arguments.
+     */
+    protected void lexError(int pos, String key, Object... args) {
+        log.error(pos, key, args);
+        tk = TokenKind.ERROR;
+        errPos = pos;
+    }
+
+    /** Read next character in character or string literal and copy into sbuf.
+     */
+    private void scanLitChar(int pos) {
+        if (reader.ch == '\\') {
+            if (reader.peekChar() == '\\' && !reader.isUnicode()) {
+                reader.skipChar();
+                reader.putChar('\\', true);
+            } else {
+                reader.scanChar();
+                switch (reader.ch) {
+                case '0': case '1': case '2': case '3':
+                case '4': case '5': case '6': case '7':
+                    char leadch = reader.ch;
+                    int oct = reader.digit(pos, 8);
+                    reader.scanChar();
+                    if ('0' <= reader.ch && reader.ch <= '7') {
+                        oct = oct * 8 + reader.digit(pos, 8);
+                        reader.scanChar();
+                        if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
+                            oct = oct * 8 + reader.digit(pos, 8);
+                            reader.scanChar();
+                        }
+                    }
+                    reader.putChar((char)oct);
+                    break;
+                case 'b':
+                    reader.putChar('\b', true); break;
+                case 't':
+                    reader.putChar('\t', true); break;
+                case 'n':
+                    reader.putChar('\n', true); break;
+                case 'f':
+                    reader.putChar('\f', true); break;
+                case 'r':
+                    reader.putChar('\r', true); break;
+                case '\'':
+                    reader.putChar('\'', true); break;
+                case '\"':
+                    reader.putChar('\"', true); break;
+                case '\\':
+                    reader.putChar('\\', true); break;
+                default:
+                    lexError(reader.bp, "illegal.esc.char");
+                }
+            }
+        } else if (reader.bp != reader.buflen) {
+            reader.putChar(true);
+        }
+    }
+
+    private void scanDigits(int pos, int digitRadix) {
+        char saveCh;
+        int savePos;
+        do {
+            if (reader.ch != '_') {
+                reader.putChar(false);
+            } else {
+                if (!allowUnderscoresInLiterals) {
+                    lexError(pos, "unsupported.underscore.lit", source.name);
+                    allowUnderscoresInLiterals = true;
+                }
+            }
+            saveCh = reader.ch;
+            savePos = reader.bp;
+            reader.scanChar();
+        } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
+        if (saveCh == '_')
+            lexError(savePos, "illegal.underscore");
+    }
+
+    /** Read fractional part of hexadecimal floating point number.
+     */
+    private void scanHexExponentAndSuffix(int pos) {
+        if (reader.ch == 'p' || reader.ch == 'P') {
+            reader.putChar(true);
+            skipIllegalUnderscores();
+            if (reader.ch == '+' || reader.ch == '-') {
+                reader.putChar(true);
+            }
+            skipIllegalUnderscores();
+            if ('0' <= reader.ch && reader.ch <= '9') {
+                scanDigits(pos, 10);
+                if (!allowHexFloats) {
+                    lexError(pos, "unsupported.fp.lit", source.name);
+                    allowHexFloats = true;
+                }
+                else if (!hexFloatsWork)
+                    lexError(pos, "unsupported.cross.fp.lit");
+            } else
+                lexError(pos, "malformed.fp.lit");
+        } else {
+            lexError(pos, "malformed.fp.lit");
+        }
+        if (reader.ch == 'f' || reader.ch == 'F') {
+            reader.putChar(true);
+            tk = TokenKind.FLOATLITERAL;
+            radix = 16;
+        } else {
+            if (reader.ch == 'd' || reader.ch == 'D') {
+                reader.putChar(true);
+            }
+            tk = TokenKind.DOUBLELITERAL;
+            radix = 16;
+        }
+    }
+
+    /** Read fractional part of floating point number.
+     */
+    private void scanFraction(int pos) {
+        skipIllegalUnderscores();
+        if ('0' <= reader.ch && reader.ch <= '9') {
+            scanDigits(pos, 10);
+        }
+        int sp1 = reader.sp;
+        if (reader.ch == 'e' || reader.ch == 'E') {
+            reader.putChar(true);
+            skipIllegalUnderscores();
+            if (reader.ch == '+' || reader.ch == '-') {
+                reader.putChar(true);
+            }
+            skipIllegalUnderscores();
+            if ('0' <= reader.ch && reader.ch <= '9') {
+                scanDigits(pos, 10);
+                return;
+            }
+            lexError(pos, "malformed.fp.lit");
+            reader.sp = sp1;
+        }
+    }
+
+    /** Read fractional part and 'd' or 'f' suffix of floating point number.
+     */
+    private void scanFractionAndSuffix(int pos) {
+        radix = 10;
+        scanFraction(pos);
+        if (reader.ch == 'f' || reader.ch == 'F') {
+            reader.putChar(true);
+            tk = TokenKind.FLOATLITERAL;
+        } else {
+            if (reader.ch == 'd' || reader.ch == 'D') {
+                reader.putChar(true);
+            }
+            tk = TokenKind.DOUBLELITERAL;
+        }
+    }
+
+    /** Read fractional part and 'd' or 'f' suffix of floating point number.
+     */
+    private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
+        radix = 16;
+        Assert.check(reader.ch == '.');
+        reader.putChar(true);
+        skipIllegalUnderscores();
+        if (reader.digit(pos, 16) >= 0) {
+            seendigit = true;
+            scanDigits(pos, 16);
+        }
+        if (!seendigit)
+            lexError(pos, "invalid.hex.number");
+        else
+            scanHexExponentAndSuffix(pos);
+    }
+
+    private void skipIllegalUnderscores() {
+        if (reader.ch == '_') {
+            lexError(reader.bp, "illegal.underscore");
+            while (reader.ch == '_')
+                reader.scanChar();
+        }
+    }
+
+    /** Read a number.
+     *  @param radix  The radix of the number; one of 2, j8, 10, 16.
+     */
+    private void scanNumber(int pos, int radix) {
+        // for octal, allow base-10 digit in case it's a float literal
+        this.radix = radix;
+        int digitRadix = (radix == 8 ? 10 : radix);
+        boolean seendigit = false;
+        if (reader.digit(pos, digitRadix) >= 0) {
+            seendigit = true;
+            scanDigits(pos, digitRadix);
+        }
+        if (radix == 16 && reader.ch == '.') {
+            scanHexFractionAndSuffix(pos, seendigit);
+        } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
+            scanHexExponentAndSuffix(pos);
+        } else if (digitRadix == 10 && reader.ch == '.') {
+            reader.putChar(true);
+            scanFractionAndSuffix(pos);
+        } else if (digitRadix == 10 &&
+                   (reader.ch == 'e' || reader.ch == 'E' ||
+                    reader.ch == 'f' || reader.ch == 'F' ||
+                    reader.ch == 'd' || reader.ch == 'D')) {
+            scanFractionAndSuffix(pos);
+        } else {
+            if (reader.ch == 'l' || reader.ch == 'L') {
+                reader.scanChar();
+                tk = TokenKind.LONGLITERAL;
+            } else {
+                tk = TokenKind.INTLITERAL;
+            }
+        }
+    }
+
+    /** Read an identifier.
+     */
+    private void scanIdent() {
+        boolean isJavaIdentifierPart;
+        char high;
+        reader.putChar(true);
+        do {
+            switch (reader.ch) {
+            case 'A': case 'B': case 'C': case 'D': case 'E':
+            case 'F': case 'G': case 'H': case 'I': case 'J':
+            case 'K': case 'L': case 'M': case 'N': case 'O':
+            case 'P': case 'Q': case 'R': case 'S': case 'T':
+            case 'U': case 'V': case 'W': case 'X': case 'Y':
+            case 'Z':
+            case 'a': case 'b': case 'c': case 'd': case 'e':
+            case 'f': case 'g': case 'h': case 'i': case 'j':
+            case 'k': case 'l': case 'm': case 'n': case 'o':
+            case 'p': case 'q': case 'r': case 's': case 't':
+            case 'u': case 'v': case 'w': case 'x': case 'y':
+            case 'z':
+            case '$': case '_':
+            case '0': case '1': case '2': case '3': case '4':
+            case '5': case '6': case '7': case '8': case '9':
+                break;
+            case '\u0000': case '\u0001': case '\u0002': case '\u0003':
+            case '\u0004': case '\u0005': case '\u0006': case '\u0007':
+            case '\u0008': case '\u000E': case '\u000F': case '\u0010':
+            case '\u0011': case '\u0012': case '\u0013': case '\u0014':
+            case '\u0015': case '\u0016': case '\u0017':
+            case '\u0018': case '\u0019': case '\u001B':
+            case '\u007F':
+                reader.scanChar();
+                continue;
+            case '\u001A': // EOI is also a legal identifier part
+                if (reader.bp >= reader.buflen) {
+                    name = reader.name();
+                    tk = tokens.lookupKind(name);
+                    return;
+                }
+                reader.scanChar();
+                continue;
+            default:
+                if (reader.ch < '\u0080') {
+                    // all ASCII range chars already handled, above
+                    isJavaIdentifierPart = false;
+                } else {
+                    if (Character.isIdentifierIgnorable(reader.ch)) {
+                        reader.scanChar();
+                        continue;
+                    } else {
+                        high = reader.scanSurrogates();
+                        if (high != 0) {
+                            reader.putChar(high);
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(
+                                Character.toCodePoint(high, reader.ch));
+                        } else {
+                            isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
+                        }
+                    }
+                }
+                if (!isJavaIdentifierPart) {
+                    name = reader.name();
+                    tk = tokens.lookupKind(name);
+                    return;
+                }
+            }
+            reader.putChar(true);
+        } while (true);
+    }
+
+    /** Return true if reader.ch can be part of an operator.
+     */
+    private boolean isSpecial(char ch) {
+        switch (ch) {
+        case '!': case '%': case '&': case '*': case '?':
+        case '+': case '-': case ':': case '<': case '=':
+        case '>': case '^': case '|': case '~':
+        case '@':
+            return true;
+        default:
+            return false;
+        }
+    }
+
+    /** Read longest possible sequence of special characters and convert
+     *  to token.
+     */
+    private void scanOperator() {
+        while (true) {
+            reader.putChar(false);
+            Name newname = reader.name();
+            TokenKind tk1 = tokens.lookupKind(newname);
+            if (tk1 == TokenKind.IDENTIFIER) {
+                reader.sp--;
+                break;
+            }
+            tk = tk1;
+            reader.scanChar();
+            if (!isSpecial(reader.ch)) break;
+        }
+    }
+
+    /** Read token.
+     */
+    public Token readToken() {
+
+        reader.sp = 0;
+        name = null;
+        radix = 0;
+
+        int pos = 0;
+        int endPos = 0;
+        List<Comment> comments = null;
+
+        try {
+            loop: while (true) {
+                pos = reader.bp;
+                switch (reader.ch) {
+                case ' ': // (Spec 3.6)
+                case '\t': // (Spec 3.6)
+                case FF: // (Spec 3.6)
+                    do {
+                        reader.scanChar();
+                    } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
+                    processWhiteSpace(pos, reader.bp);
+                    break;
+                case LF: // (Spec 3.4)
+                    reader.scanChar();
+                    processLineTerminator(pos, reader.bp);
+                    break;
+                case CR: // (Spec 3.4)
+                    reader.scanChar();
+                    if (reader.ch == LF) {
+                        reader.scanChar();
+                    }
+                    processLineTerminator(pos, reader.bp);
+                    break;
+                case 'A': case 'B': case 'C': case 'D': case 'E':
+                case 'F': case 'G': case 'H': case 'I': case 'J':
+                case 'K': case 'L': case 'M': case 'N': case 'O':
+                case 'P': case 'Q': case 'R': case 'S': case 'T':
+                case 'U': case 'V': case 'W': case 'X': case 'Y':
+                case 'Z':
+                case 'a': case 'b': case 'c': case 'd': case 'e':
+                case 'f': case 'g': case 'h': case 'i': case 'j':
+                case 'k': case 'l': case 'm': case 'n': case 'o':
+                case 'p': case 'q': case 'r': case 's': case 't':
+                case 'u': case 'v': case 'w': case 'x': case 'y':
+                case 'z':
+                case '$': case '_':
+                    scanIdent();
+                    break loop;
+                case '0':
+                    reader.scanChar();
+                    if (reader.ch == 'x' || reader.ch == 'X') {
+                        reader.scanChar();
+                        skipIllegalUnderscores();
+                        if (reader.ch == '.') {
+                            scanHexFractionAndSuffix(pos, false);
+                        } else if (reader.digit(pos, 16) < 0) {
+                            lexError(pos, "invalid.hex.number");
+                        } else {
+                            scanNumber(pos, 16);
+                        }
+                    } else if (reader.ch == 'b' || reader.ch == 'B') {
+                        if (!allowBinaryLiterals) {
+                            lexError(pos, "unsupported.binary.lit", source.name);
+                            allowBinaryLiterals = true;
+                        }
+                        reader.scanChar();
+                        skipIllegalUnderscores();
+                        if (reader.digit(pos, 2) < 0) {
+                            lexError(pos, "invalid.binary.number");
+                        } else {
+                            scanNumber(pos, 2);
+                        }
+                    } else {
+                        reader.putChar('0');
+                        if (reader.ch == '_') {
+                            int savePos = reader.bp;
+                            do {
+                                reader.scanChar();
+                            } while (reader.ch == '_');
+                            if (reader.digit(pos, 10) < 0) {
+                                lexError(savePos, "illegal.underscore");
+                            }
+                        }
+                        scanNumber(pos, 8);
+                    }
+                    break loop;
+                case '1': case '2': case '3': case '4':
+                case '5': case '6': case '7': case '8': case '9':
+                    scanNumber(pos, 10);
+                    break loop;
+                case '.':
+                    reader.scanChar();
+                    if ('0' <= reader.ch && reader.ch <= '9') {
+                        reader.putChar('.');
+                        scanFractionAndSuffix(pos);
+                    } else if (reader.ch == '.') {
+                        int savePos = reader.bp;
+                        reader.putChar('.'); reader.putChar('.', true);
+                        if (reader.ch == '.') {
+                            reader.scanChar();
+                            reader.putChar('.');
+                            tk = TokenKind.ELLIPSIS;
+                        } else {
+                            lexError(savePos, "illegal.dot");
+                        }
+                    } else {
+                        tk = TokenKind.DOT;
+                    }
+                    break loop;
+                case ',':
+                    reader.scanChar(); tk = TokenKind.COMMA; break loop;
+                case ';':
+                    reader.scanChar(); tk = TokenKind.SEMI; break loop;
+                case '(':
+                    reader.scanChar(); tk = TokenKind.LPAREN; break loop;
+                case ')':
+                    reader.scanChar(); tk = TokenKind.RPAREN; break loop;
+                case '[':
+                    reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
+                case ']':
+                    reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
+                case '{':
+                    reader.scanChar(); tk = TokenKind.LBRACE; break loop;
+                case '}':
+                    reader.scanChar(); tk = TokenKind.RBRACE; break loop;
+                case '/':
+                    reader.scanChar();
+                    if (reader.ch == '/') {
+                        do {
+                            reader.scanCommentChar();
+                        } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
+                        if (reader.bp < reader.buflen) {
+                            comments = addComment(comments, processComment(pos, reader.bp, CommentStyle.LINE));
+                        }
+                        break;
+                    } else if (reader.ch == '*') {
+                        boolean isEmpty = false;
+                        reader.scanChar();
+                        CommentStyle style;
+                        if (reader.ch == '*') {
+                            style = CommentStyle.JAVADOC;
+                            reader.scanCommentChar();
+                            if (reader.ch == '/') {
+                                isEmpty = true;
+                            }
+                        } else {
+                            style = CommentStyle.BLOCK;
+                        }
+                        while (!isEmpty && reader.bp < reader.buflen) {
+                            if (reader.ch == '*') {
+                                reader.scanChar();
+                                if (reader.ch == '/') break;
+                            } else {
+                                reader.scanCommentChar();
+                            }
+                        }
+                        if (reader.ch == '/') {
+                            reader.scanChar();
+                            comments = addComment(comments, processComment(pos, reader.bp, style));
+                            break;
+                        } else {
+                            lexError(pos, "unclosed.comment");
+                            break loop;
+                        }
+                    } else if (reader.ch == '=') {
+                        tk = TokenKind.SLASHEQ;
+                        reader.scanChar();
+                    } else {
+                        tk = TokenKind.SLASH;
+                    }
+                    break loop;
+                case '\'':
+                    reader.scanChar();
+                    if (reader.ch == '\'') {
+                        lexError(pos, "empty.char.lit");
+                    } else {
+                        if (reader.ch == CR || reader.ch == LF)
+                            lexError(pos, "illegal.line.end.in.char.lit");
+                        scanLitChar(pos);
+                        char ch2 = reader.ch;
+                        if (reader.ch == '\'') {
+                            reader.scanChar();
+                            tk = TokenKind.CHARLITERAL;
+                        } else {
+                            lexError(pos, "unclosed.char.lit");
+                        }
+                    }
+                    break loop;
+                case '\"':
+                    reader.scanChar();
+                    while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
+                        scanLitChar(pos);
+                    if (reader.ch == '\"') {
+                        tk = TokenKind.STRINGLITERAL;
+                        reader.scanChar();
+                    } else {
+                        lexError(pos, "unclosed.str.lit");
+                    }
+                    break loop;
+                default:
+                    if (isSpecial(reader.ch)) {
+                        scanOperator();
+                    } else {
+                        boolean isJavaIdentifierStart;
+                        if (reader.ch < '\u0080') {
+                            // all ASCII range chars already handled, above
+                            isJavaIdentifierStart = false;
+                        } else {
+                            char high = reader.scanSurrogates();
+                            if (high != 0) {
+                                reader.putChar(high);
+
+                                isJavaIdentifierStart = Character.isJavaIdentifierStart(
+                                    Character.toCodePoint(high, reader.ch));
+                            } else {
+                                isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
+                            }
+                        }
+                        if (isJavaIdentifierStart) {
+                            scanIdent();
+                        } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
+                            tk = TokenKind.EOF;
+                            pos = reader.buflen;
+                        } else {
+                            String arg = (32 < reader.ch && reader.ch < 127) ?
+                                            String.format("%s", reader.ch) :
+                                            String.format("\\u%04x", (int)reader.ch);
+                            lexError(pos, "illegal.char", arg);
+                            reader.scanChar();
+                        }
+                    }
+                    break loop;
+                }
+            }
+            endPos = reader.bp;
+            switch (tk.tag) {
+                case DEFAULT: return new Token(tk, pos, endPos, comments);
+                case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
+                case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
+                case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
+                default: throw new AssertionError();
+            }
+        }
+        finally {
+            if (scannerDebug) {
+                    System.out.println("nextToken(" + pos
+                                       + "," + endPos + ")=|" +
+                                       new String(reader.getRawCharacters(pos, endPos))
+                                       + "|");
+            }
+        }
+    }
+    //where
+        List<Comment> addComment(List<Comment> comments, Comment comment) {
+            return comments == null ?
+                    List.of(comment) :
+                    comments.prepend(comment);
+        }
+
+    /** Return the position where a lexical error occurred;
+     */
+    public int errPos() {
+        return errPos;
+    }
+
+    /** Set the position where a lexical error occurred;
+     */
+    public void errPos(int pos) {
+        errPos = pos;
+    }
+
+    /**
+     * Called when a complete comment has been scanned. pos and endPos
+     * will mark the comment boundary.
+     */
+    protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
+        if (scannerDebug)
+            System.out.println("processComment(" + pos
+                               + "," + endPos + "," + style + ")=|"
+                               + new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+        char[] buf = reader.getRawCharacters(pos, endPos);
+        return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style);
+    }
+
+    /**
+     * Called when a complete whitespace run has been scanned. pos and endPos
+     * will mark the whitespace boundary.
+     */
+    protected void processWhiteSpace(int pos, int endPos) {
+        if (scannerDebug)
+            System.out.println("processWhitespace(" + pos
+                               + "," + endPos + ")=|" +
+                               new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+    }
+
+    /**
+     * Called when a line terminator has been processed.
+     */
+    protected void processLineTerminator(int pos, int endPos) {
+        if (scannerDebug)
+            System.out.println("processTerminator(" + pos
+                               + "," + endPos + ")=|" +
+                               new String(reader.getRawCharacters(pos, endPos))
+                               + "|");
+    }
+
+    /** Build a map for translating between line numbers and
+     * positions in the input.
+     *
+     * @return a LineMap */
+    public Position.LineMap getLineMap() {
+        return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
+    }
+
+
+    /**
+    * Scan a documentation comment; determine if a deprecated tag is present.
+    * Called once the initial /, * have been skipped, positioned at the second *
+    * (which is treated as the beginning of the first line).
+    * Stops positioned at the closing '/'.
+    */
+    protected static class BasicComment<U extends UnicodeReader> implements Comment {
+
+        CommentStyle cs;
+        U comment_reader;
+
+        protected boolean deprecatedFlag = false;
+        protected boolean scanned = false;
+
+        protected BasicComment(U comment_reader, CommentStyle cs) {
+            this.comment_reader = comment_reader;
+            this.cs = cs;
+        }
+
+        public String getText() {
+            return null;
+        }
+
+        public int getSourcePos(int pos) {
+            return -1;
+        }
+
+        public CommentStyle getStyle() {
+            return cs;
+        }
+
+        public boolean isDeprecated() {
+            if (!scanned && cs == CommentStyle.JAVADOC) {
+                scanDocComment();
+            }
+            return deprecatedFlag;
+        }
+
+        @SuppressWarnings("fallthrough")
+        protected void scanDocComment() {
+            try {
+                boolean deprecatedPrefix = false;
+
+                comment_reader.bp += 3; // '/**'
+                comment_reader.ch = comment_reader.buf[comment_reader.bp];
+
+                forEachLine:
+                while (comment_reader.bp < comment_reader.buflen) {
+
+                    // Skip optional WhiteSpace at beginning of line
+                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+                        comment_reader.scanCommentChar();
+                    }
+
+                    // Skip optional consecutive Stars
+                    while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
+                        comment_reader.scanCommentChar();
+                        if (comment_reader.ch == '/') {
+                            return;
+                        }
+                    }
+
+                    // Skip optional WhiteSpace after Stars
+                    while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
+                        comment_reader.scanCommentChar();
+                    }
+
+                    deprecatedPrefix = false;
+                    // At beginning of line in the JavaDoc sense.
+                    if (!deprecatedFlag) {
+                        String deprecated = "@deprecated";
+                        int i = 0;
+                        while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
+                            comment_reader.scanCommentChar();
+                            i++;
+                            if (i == deprecated.length()) {
+                                deprecatedPrefix = true;
+                                break;
+                            }
+                        }
+                    }
+
+                    if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
+                        if (Character.isWhitespace(comment_reader.ch)) {
+                            deprecatedFlag = true;
+                        } else if (comment_reader.ch == '*') {
+                            comment_reader.scanCommentChar();
+                            if (comment_reader.ch == '/') {
+                                deprecatedFlag = true;
+                                return;
+                            }
+                        }
+                    }
+
+                    // Skip rest of line
+                    while (comment_reader.bp < comment_reader.buflen) {
+                        switch (comment_reader.ch) {
+                            case '*':
+                                comment_reader.scanCommentChar();
+                                if (comment_reader.ch == '/') {
+                                    return;
+                                }
+                                break;
+                            case CR: // (Spec 3.4)
+                                comment_reader.scanCommentChar();
+                                if (comment_reader.ch != LF) {
+                                    continue forEachLine;
+                                }
+                            /* fall through to LF case */
+                            case LF: // (Spec 3.4)
+                                comment_reader.scanCommentChar();
+                                continue forEachLine;
+                            default:
+                                comment_reader.scanCommentChar();
+                        }
+                    } // rest of line
+                } // forEachLine
+                return;
+            } finally {
+                scanned = true;
+            }
+        }
+    }
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/JavadocTokenizer.java

@@ -0,0 +1,447 @@
+/*
+ * Copyright (c) 2004, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.parser.Tokens.Comment;
+import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
+import com.sun.tools.javac.util.*;
+
+import java.nio.*;
+
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** An extension to the base lexical analyzer that captures
+ *  and processes the contents of doc comments.  It does so by
+ *  translating Unicode escape sequences and by stripping the
+ *  leading whitespace and starts from each line of the comment.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class JavadocTokenizer extends JavaTokenizer {
+
+    /** Create a scanner from the input buffer.  buffer must implement
+     *  array() and compact(), and remaining() must be less than limit().
+     */
+    protected JavadocTokenizer(ScannerFactory fac, CharBuffer buffer) {
+        super(fac, buffer);
+    }
+
+    /** Create a scanner from the input array.  The array must have at
+     *  least a single character of extra space.
+     */
+    protected JavadocTokenizer(ScannerFactory fac, char[] input, int inputLength) {
+        super(fac, input, inputLength);
+    }
+
+    @Override
+    protected Comment processComment(int pos, int endPos, CommentStyle style) {
+        char[] buf = reader.getRawCharacters(pos, endPos);
+        return new JavadocComment(new DocReader(fac, buf, buf.length, pos), style);
+    }
+
+    /**
+     * This is a specialized version of UnicodeReader that keeps track of the
+     * column position within a given character stream (used for Javadoc processing),
+     * and which builds a table for mapping positions in the comment string to
+     * positions in the source file.
+     */
+    static class DocReader extends UnicodeReader {
+
+         int col;
+         int startPos;
+
+         /**
+          * A buffer for building a table for mapping positions in {@link #sbuf}
+          * to positions in the source buffer.
+          *
+          * The array is organized as a series of pairs of integers: the first
+          * number in each pair specifies a position in the comment text,
+          * the second number in each pair specifies the corresponding position
+          * in the source buffer. The pairs are sorted in ascending order.
+          *
+          * Since the mapping function is generally continuous, with successive
+          * positions in the string corresponding to successive positions in the
+          * source buffer, the table only needs to record discontinuities in
+          * the mapping. The values of intermediate positions can be inferred.
+          *
+          * Discontinuities may occur in a number of places: when a newline
+          * is followed by whitespace and asterisks (which are ignored),
+          * when a tab is expanded into spaces, and when unicode escapes
+          * are used in the source buffer.
+          *
+          * Thus, to find the source position of any position, p, in the comment
+          * string, find the index, i, of the pair whose string offset
+          * ({@code pbuf[i] }) is closest to but not greater than p. Then,
+          * {@code sourcePos(p) = pbuf[i+1] + (p - pbuf[i]) }.
+          */
+         int[] pbuf = new int[128];
+
+         /**
+          * The index of the next empty slot in the pbuf buffer.
+          */
+         int pp = 0;
+
+         DocReader(ScannerFactory fac, char[] input, int inputLength, int startPos) {
+             super(fac, input, inputLength);
+             this.startPos = startPos;
+         }
+
+         @Override
+         protected void convertUnicode() {
+             if (ch == '\\' && unicodeConversionBp != bp) {
+                 bp++; ch = buf[bp]; col++;
+                 if (ch == 'u') {
+                     do {
+                         bp++; ch = buf[bp]; col++;
+                     } while (ch == 'u');
+                     int limit = bp + 3;
+                     if (limit < buflen) {
+                         int d = digit(bp, 16);
+                         int code = d;
+                         while (bp < limit && d >= 0) {
+                             bp++; ch = buf[bp]; col++;
+                             d = digit(bp, 16);
+                             code = (code << 4) + d;
+                         }
+                         if (d >= 0) {
+                             ch = (char)code;
+                             unicodeConversionBp = bp;
+                             return;
+                         }
+                     }
+                     // "illegal.Unicode.esc", reported by base scanner
+                 } else {
+                     bp--;
+                     ch = '\\';
+                     col--;
+                 }
+             }
+         }
+
+         @Override
+         protected void scanCommentChar() {
+             scanChar();
+             if (ch == '\\') {
+                 if (peekChar() == '\\' && !isUnicode()) {
+                     putChar(ch, false);
+                     bp++; col++;
+                 } else {
+                     convertUnicode();
+                 }
+             }
+         }
+
+         @Override
+         protected void scanChar() {
+             bp++;
+             ch = buf[bp];
+             switch (ch) {
+             case '\r': // return
+                 col = 0;
+                 break;
+             case '\n': // newline
+                 if (bp == 0 || buf[bp-1] != '\r') {
+                     col = 0;
+                 }
+                 break;
+             case '\t': // tab
+                 col = (col / TabInc * TabInc) + TabInc;
+                 break;
+             case '\\': // possible Unicode
+                 col++;
+                 convertUnicode();
+                 break;
+             default:
+                 col++;
+                 break;
+             }
+         }
+
+         @Override
+         public void putChar(char ch, boolean scan) {
+             // At this point, bp is the position of the current character in buf,
+             // and sp is the position in sbuf where this character will be put.
+             // Record a new entry in pbuf if pbuf is empty or if sp and its
+             // corresponding source position are not equidistant from the
+             // corresponding values in the latest entry in the pbuf array.
+             // (i.e. there is a discontinuity in the map function.)
+             if ((pp == 0)
+                     || (sp - pbuf[pp - 2] != (startPos + bp) - pbuf[pp - 1])) {
+                 if (pp + 1 >= pbuf.length) {
+                     int[] new_pbuf = new int[pbuf.length * 2];
+                     System.arraycopy(pbuf, 0, new_pbuf, 0, pbuf.length);
+                     pbuf = new_pbuf;
+                 }
+                 pbuf[pp] = sp;
+                 pbuf[pp + 1] = startPos + bp;
+                 pp += 2;
+             }
+             super.putChar(ch, scan);
+         }
+     }
+
+     protected static class JavadocComment extends JavaTokenizer.BasicComment<DocReader> {
+
+        /**
+        * Translated and stripped contents of doc comment
+        */
+        private String docComment = null;
+        private int[] docPosns = null;
+
+        JavadocComment(DocReader reader, CommentStyle cs) {
+            super(reader, cs);
+        }
+
+        @Override
+        public String getText() {
+            if (!scanned && cs == CommentStyle.JAVADOC) {
+                scanDocComment();
+            }
+            return docComment;
+        }
+
+        @Override
+        public int getSourcePos(int pos) {
+            // Binary search to find the entry for which the string index is
+            // less than pos. Since docPosns is a list of pairs of integers
+            // we must make sure the index is always even.
+            // If we find an exact match for pos, the other item in the pair
+            // gives the source pos; otherwise, compute the source position
+            // relative to the best match found in the array.
+            if (pos == Position.NOPOS)
+                return Position.NOPOS;
+            if (pos < 0 || pos > docComment.length())
+                throw new StringIndexOutOfBoundsException(String.valueOf(pos));
+            if (docPosns == null)
+                return Position.NOPOS;
+            int start = 0;
+            int end = docPosns.length;
+            while (start < end - 2) {
+                // find an even index midway between start and end
+                int index = ((start  + end) / 4) * 2;
+                if (docPosns[index] < pos)
+                    start = index;
+                else if (docPosns[index] == pos)
+                    return docPosns[index + 1];
+                else
+                    end = index;
+            }
+            return docPosns[start + 1] + (pos - docPosns[start]);
+        }
+
+        @Override
+        @SuppressWarnings("fallthrough")
+        protected void scanDocComment() {
+             try {
+                 boolean firstLine = true;
+
+                 // Skip over first slash
+                 comment_reader.scanCommentChar();
+                 // Skip over first star
+                 comment_reader.scanCommentChar();
+
+                 // consume any number of stars
+                 while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
+                     comment_reader.scanCommentChar();
+                 }
+                 // is the comment in the form /**/, /***/, /****/, etc. ?
+                 if (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '/') {
+                     docComment = "";
+                     return;
+                 }
+
+                 // skip a newline on the first line of the comment.
+                 if (comment_reader.bp < comment_reader.buflen) {
+                     if (comment_reader.ch == LF) {
+                         comment_reader.scanCommentChar();
+                         firstLine = false;
+                     } else if (comment_reader.ch == CR) {
+                         comment_reader.scanCommentChar();
+                         if (comment_reader.ch == LF) {
+                             comment_reader.scanCommentChar();
+                             firstLine = false;
+                         }
+                     }
+                 }
+
+             outerLoop:
+
+                 // The outerLoop processes the doc comment, looping once
+                 // for each line.  For each line, it first strips off
+                 // whitespace, then it consumes any stars, then it
+                 // puts the rest of the line into our buffer.
+                 while (comment_reader.bp < comment_reader.buflen) {
+                     int begin_bp = comment_reader.bp;
+                     char begin_ch = comment_reader.ch;
+                     // The wsLoop consumes whitespace from the beginning
+                     // of each line.
+                 wsLoop:
+
+                     while (comment_reader.bp < comment_reader.buflen) {
+                         switch(comment_reader.ch) {
+                         case ' ':
+                             comment_reader.scanCommentChar();
+                             break;
+                         case '\t':
+                             comment_reader.col = ((comment_reader.col - 1) / TabInc * TabInc) + TabInc;
+                             comment_reader.scanCommentChar();
+                             break;
+                         case FF:
+                             comment_reader.col = 0;
+                             comment_reader.scanCommentChar();
+                             break;
+         // Treat newline at beginning of line (blank line, no star)
+         // as comment text.  Old Javadoc compatibility requires this.
+         /*---------------------------------*
+                         case CR: // (Spec 3.4)
+                             doc_reader.scanCommentChar();
+                             if (ch == LF) {
+                                 col = 0;
+                                 doc_reader.scanCommentChar();
+                             }
+                             break;
+                         case LF: // (Spec 3.4)
+                             doc_reader.scanCommentChar();
+                             break;
+         *---------------------------------*/
+                         default:
+                             // we've seen something that isn't whitespace;
+                             // jump out.
+                             break wsLoop;
+                         }
+                     }
+
+                     // Are there stars here?  If so, consume them all
+                     // and check for the end of comment.
+                     if (comment_reader.ch == '*') {
+                         // skip all of the stars
+                         do {
+                             comment_reader.scanCommentChar();
+                         } while (comment_reader.ch == '*');
+
+                         // check for the closing slash.
+                         if (comment_reader.ch == '/') {
+                             // We're done with the doc comment
+                             // scanChar() and breakout.
+                             break outerLoop;
+                         }
+                     } else if (! firstLine) {
+                         // The current line does not begin with a '*' so we will
+                         // treat it as comment
+                         comment_reader.bp = begin_bp;
+                         comment_reader.ch = begin_ch;
+                     }
+                     // The textLoop processes the rest of the characters
+                     // on the line, adding them to our buffer.
+                 textLoop:
+                     while (comment_reader.bp < comment_reader.buflen) {
+                         switch (comment_reader.ch) {
+                         case '*':
+                             // Is this just a star?  Or is this the
+                             // end of a comment?
+                             comment_reader.scanCommentChar();
+                             if (comment_reader.ch == '/') {
+                                 // This is the end of the comment,
+                                 // set ch and return our buffer.
+                                 break outerLoop;
+                             }
+                             // This is just an ordinary star.  Add it to
+                             // the buffer.
+                             comment_reader.putChar('*', false);
+                             break;
+                         case ' ':
+                         case '\t':
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                             break;
+                         case FF:
+                             comment_reader.scanCommentChar();
+                             break textLoop; // treat as end of line
+                         case CR: // (Spec 3.4)
+                             comment_reader.scanCommentChar();
+                             if (comment_reader.ch != LF) {
+                                 // Canonicalize CR-only line terminator to LF
+                                 comment_reader.putChar((char)LF, false);
+                                 break textLoop;
+                             }
+                             /* fall through to LF case */
+                         case LF: // (Spec 3.4)
+                             // We've seen a newline.  Add it to our
+                             // buffer and break out of this loop,
+                             // starting fresh on a new line.
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                             break textLoop;
+                         default:
+                             // Add the character to our buffer.
+                             comment_reader.putChar(comment_reader.ch, false);
+                             comment_reader.scanCommentChar();
+                         }
+                     } // end textLoop
+                     firstLine = false;
+                 } // end outerLoop
+
+                 if (comment_reader.sp > 0) {
+                     int i = comment_reader.sp - 1;
+                 trailLoop:
+                     while (i > -1) {
+                         switch (comment_reader.sbuf[i]) {
+                         case '*':
+                             i--;
+                             break;
+                         default:
+                             break trailLoop;
+                         }
+                     }
+                     comment_reader.sp = i + 1;
+
+                     // Store the text of the doc comment
+                    docComment = comment_reader.chars();
+                    docPosns = new int[comment_reader.pp];
+                    System.arraycopy(comment_reader.pbuf, 0, docPosns, 0, docPosns.length);
+                } else {
+                    docComment = "";
+                }
+            } finally {
+                scanned = true;
+                comment_reader = null;
+                if (docComment != null &&
+                        docComment.matches("(?sm).*^\\s*@deprecated( |$).*")) {
+                    deprecatedFlag = true;
+                }
+            }
+        }
+    }
+
+    @Override
+    public Position.LineMap getLineMap() {
+        char[] buf = reader.getRawCharacters();
+        return Position.makeLineMap(buf, buf.length, true);
+    }
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/LazyDocCommentTable.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import com.sun.tools.javac.parser.Tokens.Comment;
+import com.sun.tools.javac.tree.DCTree.DCDocComment;
+import com.sun.tools.javac.tree.DocCommentTable;
+import com.sun.tools.javac.tree.JCTree;
+import com.sun.tools.javac.util.DiagnosticSource;
+
+
+/**
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class LazyDocCommentTable implements DocCommentTable {
+    private static class Entry {
+        final Comment comment;
+        DCDocComment tree;
+
+        Entry(Comment c) {
+            comment = c;
+        }
+    }
+
+    ParserFactory fac;
+    DiagnosticSource diagSource;
+    Map<JCTree, Entry> table;
+
+    LazyDocCommentTable(ParserFactory fac) {
+        this.fac = fac;
+        diagSource = fac.log.currentSource();
+        table = new HashMap<JCTree, Entry>();
+    }
+
+    public boolean hasComment(JCTree tree) {
+        return table.containsKey(tree);
+    }
+
+    public Comment getComment(JCTree tree) {
+        Entry e = table.get(tree);
+        return (e == null) ? null : e.comment;
+    }
+
+    public String getCommentText(JCTree tree) {
+        Comment c = getComment(tree);
+        return (c == null) ? null : c.getText();
+    }
+
+    public DCDocComment getCommentTree(JCTree tree) {
+        Entry e = table.get(tree);
+        if (e == null)
+            return null;
+        if (e.tree == null)
+            e.tree = new DocCommentParser(fac, diagSource, e.comment).parse();
+        return e.tree;
+    }
+
+    public void putComment(JCTree tree, Comment c) {
+        table.put(tree, new Entry(c));
+    }
+
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/Lexer.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2005, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.parser.Tokens.*;
+import com.sun.tools.javac.util.Position.LineMap;
+
+/**
+ * The lexical analyzer maps an input stream consisting of ASCII
+ * characters and Unicode escapes into a token sequence.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public interface Lexer {
+
+    /**
+     * Consume the next token.
+     */
+    void nextToken();
+
+    /**
+     * Return current token.
+     */
+    Token token();
+
+    /**
+     * Return token with given lookahead.
+     */
+    Token token(int lookahead);
+
+    /**
+     * Return the last character position of the previous token.
+     */
+    Token prevToken();
+
+    /**
+     * Splits the current token in two and return the first (splitted) token.
+     * For instance {@literal '<<<'} is split into two tokens
+     * {@literal '<'} and {@literal '<<'} respectively,
+     * and the latter is returned.
+     */
+    Token split();
+
+    /**
+     * Return the position where a lexical error occurred;
+     */
+    int errPos();
+
+    /**
+     * Set the position where a lexical error occurred;
+     */
+    void errPos(int pos);
+
+    /**
+     * Build a map for translating between line numbers and
+     * positions in the input.
+     *
+     * @return a LineMap
+     */
+    LineMap getLineMap();
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/Parser.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 1999, 2008, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+
+package com.sun.tools.javac.parser;
+
+import com.sun.tools.javac.tree.JCTree.JCCompilationUnit;
+import com.sun.tools.javac.tree.JCTree.JCExpression;
+import com.sun.tools.javac.tree.JCTree.JCStatement;
+
+/**
+ * Reads syntactic units from source code.
+ * Parsers are normally created from a ParserFactory.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public interface Parser {
+    /**
+     * Parse a compilation unit.
+     * @return a compilation unit
+     */
+    JCCompilationUnit parseCompilationUnit();
+
+    /**
+     * Parse an expression.
+     * @return an expression
+     */
+    JCExpression parseExpression();
+
+    /**
+     * Parse a statement.
+     * @return an expression
+     */
+    JCStatement parseStatement();
+
+    /**
+     * Parse a type.
+     * @return an expression for a type
+     */
+    JCExpression parseType();
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/ParserFactory.java

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 1999, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.Locale;
+
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.tree.DocTreeMaker;
+import com.sun.tools.javac.tree.TreeMaker;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.Names;
+import com.sun.tools.javac.util.Options;
+
+/**
+ * A factory for creating parsers.
+ *
+ * <p><b>This is NOT part of any supported API.
+ * If you write code that depends on this, you do so at your own risk.
+ * This code and its internal interfaces are subject to change or
+ * deletion without notice.</b>
+ */
+public class ParserFactory {
+
+    /** The context key for the parser factory. */
+    protected static final Context.Key<ParserFactory> parserFactoryKey = new Context.Key<ParserFactory>();
+
+    public static ParserFactory instance(Context context) {
+        ParserFactory instance = context.get(parserFactoryKey);
+        if (instance == null) {
+            instance = new ParserFactory(context);
+        }
+        return instance;
+    }
+
+    final TreeMaker F;
+    final DocTreeMaker docTreeMaker;
+    final Log log;
+    final Tokens tokens;
+    final Source source;
+    final Names names;
+    final Options options;
+    final ScannerFactory scannerFactory;
+    final Locale locale;
+
+    protected ParserFactory(Context context) {
+        super();
+        context.put(parserFactoryKey, this);
+        this.F = TreeMaker.instance(context);
+        this.docTreeMaker = DocTreeMaker.instance(context);
+        this.log = Log.instance(context);
+        this.names = Names.instance(context);
+        this.tokens = Tokens.instance(context);
+        this.source = Source.instance(context);
+        this.options = Options.instance(context);
+        this.scannerFactory = ScannerFactory.instance(context);
+        this.locale = context.get(Locale.class);
+    }
+
+    public JavacParser newParser(CharSequence input, boolean keepDocComments, boolean keepEndPos, boolean keepLineMap) {
+        Lexer lexer = scannerFactory.newScanner(input, keepDocComments);
+        return new JavacParser(this, lexer, keepDocComments, keepLineMap, keepEndPos);
+    }
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/Scanner.java

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.nio.*;
+import java.util.List;
+import java.util.ArrayList;
+
+import com.sun.tools.javac.util.Position.LineMap;
+import com.sun.tools.javac.parser.JavaTokenizer.*;
+
+import static com.sun.tools.javac.parser.Tokens.*;
+
+/** The lexical analyzer maps an input stream consisting of
+ *  ASCII characters and Unicode escapes into a token sequence.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class Scanner implements Lexer {
+
+    private Tokens tokens;
+
+    /** The token, set by nextToken().
+     */
+    private Token token;
+
+    /** The previous token, set by nextToken().
+     */
+    private Token prevToken;
+
+    /** Buffer of saved tokens (used during lookahead)
+     */
+    private List<Token> savedTokens = new ArrayList<Token>();
+
+    private JavaTokenizer tokenizer;
+
+    /**
+     * Create a scanner from the input array.  This method might
+     * modify the array.  To avoid copying the input array, ensure
+     * that {@code inputLength < input.length} or
+     * {@code input[input.length -1]} is a white space character.
+     *
+     * @param fac the factory which created this Scanner
+     * @param buf the input, might be modified
+     * Must be positive and less than or equal to input.length.
+     */
+    protected Scanner(ScannerFactory fac, CharBuffer buf) {
+        this(fac, new JavaTokenizer(fac, buf));
+    }
+
+    protected Scanner(ScannerFactory fac, char[] buf, int inputLength) {
+        this(fac, new JavaTokenizer(fac, buf, inputLength));
+    }
+
+    protected Scanner(ScannerFactory fac, JavaTokenizer tokenizer) {
+        this.tokenizer = tokenizer;
+        tokens = fac.tokens;
+        token = prevToken = DUMMY;
+    }
+
+    public Token token() {
+        return token(0);
+    }
+
+    public Token token(int lookahead) {
+        if (lookahead == 0) {
+            return token;
+        } else {
+            ensureLookahead(lookahead);
+            return savedTokens.get(lookahead - 1);
+        }
+    }
+    //where
+        private void ensureLookahead(int lookahead) {
+            for (int i = savedTokens.size() ; i < lookahead ; i ++) {
+                savedTokens.add(tokenizer.readToken());
+            }
+        }
+
+    public Token prevToken() {
+        return prevToken;
+    }
+
+    public void nextToken() {
+        prevToken = token;
+        if (!savedTokens.isEmpty()) {
+            token = savedTokens.remove(0);
+        } else {
+            token = tokenizer.readToken();
+        }
+    }
+
+    public Token split() {
+        Token[] splitTokens = token.split(tokens);
+        prevToken = splitTokens[0];
+        token = splitTokens[1];
+        return token;
+    }
+
+    public LineMap getLineMap() {
+        return tokenizer.getLineMap();
+    }
+
+    public int errPos() {
+        return tokenizer.errPos();
+    }
+
+    public void errPos(int pos) {
+        tokenizer.errPos(pos);
+    }
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/ScannerFactory.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.nio.CharBuffer;
+
+import com.sun.tools.javac.code.Source;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.Names;
+
+
+/**
+ * A factory for creating scanners.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own
+ *  risk.  This code and its internal interfaces are subject to change
+ *  or deletion without notice.</b>
+ */
+public class ScannerFactory {
+    /** The context key for the scanner factory. */
+    public static final Context.Key<ScannerFactory> scannerFactoryKey =
+        new Context.Key<ScannerFactory>();
+
+    /** Get the Factory instance for this context. */
+    public static ScannerFactory instance(Context context) {
+        ScannerFactory instance = context.get(scannerFactoryKey);
+        if (instance == null)
+            instance = new ScannerFactory(context);
+        return instance;
+    }
+
+    final Log log;
+    final Names names;
+    final Source source;
+    final Tokens tokens;
+
+    /** Create a new scanner factory. */
+    protected ScannerFactory(Context context) {
+        context.put(scannerFactoryKey, this);
+        this.log = Log.instance(context);
+        this.names = Names.instance(context);
+        this.source = Source.instance(context);
+        this.tokens = Tokens.instance(context);
+    }
+
+    public Scanner newScanner(CharSequence input, boolean keepDocComments) {
+        if (input instanceof CharBuffer) {
+            CharBuffer buf = (CharBuffer) input;
+            if (keepDocComments)
+                return new Scanner(this, new JavadocTokenizer(this, buf));
+            else
+                return new Scanner(this, buf);
+        } else {
+            char[] array = input.toString().toCharArray();
+            return newScanner(array, array.length, keepDocComments);
+        }
+    }
+
+    public Scanner newScanner(char[] input, int inputLength, boolean keepDocComments) {
+        if (keepDocComments)
+            return new Scanner(this, new JavadocTokenizer(this, input, inputLength));
+        else
+            return new Scanner(this, input, inputLength);
+    }
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/Tokens.java

@@ -0,0 +1,481 @@
+/*
+ * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.util.Locale;
+
+import com.sun.tools.javac.api.Formattable;
+import com.sun.tools.javac.api.Messages;
+import com.sun.tools.javac.parser.Tokens.Token.Tag;
+import com.sun.tools.javac.util.List;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Context;
+import com.sun.tools.javac.util.Filter;
+import com.sun.tools.javac.util.ListBuffer;
+import com.sun.tools.javac.util.Names;
+
+/** A class that defines codes/utilities for Java source tokens
+ *  returned from lexical analysis.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class Tokens {
+
+    private final Names names;
+
+    /**
+     * Keyword array. Maps name indices to Token.
+     */
+    private final TokenKind[] key;
+
+    /**  The number of the last entered keyword.
+     */
+    private int maxKey = 0;
+
+    /** The names of all tokens.
+     */
+    private Name[] tokenName = new Name[TokenKind.values().length];
+
+    public static final Context.Key<Tokens> tokensKey =
+        new Context.Key<Tokens>();
+
+    public static Tokens instance(Context context) {
+        Tokens instance = context.get(tokensKey);
+        if (instance == null)
+            instance = new Tokens(context);
+        return instance;
+    }
+
+    protected Tokens(Context context) {
+        context.put(tokensKey, this);
+        names = Names.instance(context);
+        for (TokenKind t : TokenKind.values()) {
+            if (t.name != null)
+                enterKeyword(t.name, t);
+            else
+                tokenName[t.ordinal()] = null;
+        }
+
+        key = new TokenKind[maxKey+1];
+        for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
+        for (TokenKind t : TokenKind.values()) {
+            if (t.name != null)
+            key[tokenName[t.ordinal()].getIndex()] = t;
+        }
+    }
+
+    private void enterKeyword(String s, TokenKind token) {
+        Name n = names.fromString(s);
+        tokenName[token.ordinal()] = n;
+        if (n.getIndex() > maxKey) maxKey = n.getIndex();
+    }
+
+    /**
+     * Create a new token given a name; if the name corresponds to a token name,
+     * a new token of the corresponding kind is returned; otherwise, an
+     * identifier token is returned.
+     */
+    TokenKind lookupKind(Name name) {
+        return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
+    }
+
+    TokenKind lookupKind(String name) {
+        return lookupKind(names.fromString(name));
+    }
+
+    /**
+     * This enum defines all tokens used by the javac scanner. A token is
+     * optionally associated with a name.
+     */
+    public enum TokenKind implements Formattable, Filter<TokenKind> {
+        EOF(),
+        ERROR(),
+        IDENTIFIER(Tag.NAMED),
+        ABSTRACT("abstract"),
+        ASSERT("assert", Tag.NAMED),
+        BOOLEAN("boolean", Tag.NAMED),
+        BREAK("break"),
+        BYTE("byte", Tag.NAMED),
+        CASE("case"),
+        CATCH("catch"),
+        CHAR("char", Tag.NAMED),
+        CLASS("class"),
+        CONST("const"),
+        CONTINUE("continue"),
+        DEFAULT("default"),
+        DO("do"),
+        DOUBLE("double", Tag.NAMED),
+        ELSE("else"),
+        ENUM("enum", Tag.NAMED),
+        EXTENDS("extends"),
+        FINAL("final"),
+        FINALLY("finally"),
+        FLOAT("float", Tag.NAMED),
+        FOR("for"),
+        GOTO("goto"),
+        IF("if"),
+        IMPLEMENTS("implements"),
+        IMPORT("import"),
+        INSTANCEOF("instanceof"),
+        INT("int", Tag.NAMED),
+        INTERFACE("interface"),
+        LONG("long", Tag.NAMED),
+        NATIVE("native"),
+        NEW("new"),
+        PACKAGE("package"),
+        PRIVATE("private"),
+        PROTECTED("protected"),
+        PUBLIC("public"),
+        RETURN("return"),
+        SHORT("short", Tag.NAMED),
+        STATIC("static"),
+        STRICTFP("strictfp"),
+        SUPER("super", Tag.NAMED),
+        SWITCH("switch"),
+        SYNCHRONIZED("synchronized"),
+        THIS("this", Tag.NAMED),
+        THROW("throw"),
+        THROWS("throws"),
+        TRANSIENT("transient"),
+        TRY("try"),
+        VOID("void", Tag.NAMED),
+        VOLATILE("volatile"),
+        WHILE("while"),
+        INTLITERAL(Tag.NUMERIC),
+        LONGLITERAL(Tag.NUMERIC),
+        FLOATLITERAL(Tag.NUMERIC),
+        DOUBLELITERAL(Tag.NUMERIC),
+        CHARLITERAL(Tag.NUMERIC),
+        STRINGLITERAL(Tag.STRING),
+        TRUE("true", Tag.NAMED),
+        FALSE("false", Tag.NAMED),
+        NULL("null", Tag.NAMED),
+        UNDERSCORE("_", Tag.NAMED),
+        ARROW("->"),
+        COLCOL("::"),
+        LPAREN("("),
+        RPAREN(")"),
+        LBRACE("{"),
+        RBRACE("}"),
+        LBRACKET("["),
+        RBRACKET("]"),
+        SEMI(";"),
+        COMMA(","),
+        DOT("."),
+        ELLIPSIS("..."),
+        EQ("="),
+        GT(">"),
+        LT("<"),
+        BANG("!"),
+        TILDE("~"),
+        QUES("?"),
+        COLON(":"),
+        EQEQ("=="),
+        LTEQ("<="),
+        GTEQ(">="),
+        BANGEQ("!="),
+        AMPAMP("&&"),
+        BARBAR("||"),
+        PLUSPLUS("++"),
+        SUBSUB("--"),
+        PLUS("+"),
+        SUB("-"),
+        STAR("*"),
+        SLASH("/"),
+        AMP("&"),
+        BAR("|"),
+        CARET("^"),
+        PERCENT("%"),
+        LTLT("<<"),
+        GTGT(">>"),
+        GTGTGT(">>>"),
+        PLUSEQ("+="),
+        SUBEQ("-="),
+        STAREQ("*="),
+        SLASHEQ("/="),
+        AMPEQ("&="),
+        BAREQ("|="),
+        CARETEQ("^="),
+        PERCENTEQ("%="),
+        LTLTEQ("<<="),
+        GTGTEQ(">>="),
+        GTGTGTEQ(">>>="),
+        MONKEYS_AT("@"),
+        CUSTOM;
+
+        public final String name;
+        public final Tag tag;
+
+        TokenKind() {
+            this(null, Tag.DEFAULT);
+        }
+
+        TokenKind(String name) {
+            this(name, Tag.DEFAULT);
+        }
+
+        TokenKind(Tag tag) {
+            this(null, tag);
+        }
+
+        TokenKind(String name, Tag tag) {
+            this.name = name;
+            this.tag = tag;
+        }
+
+        public String toString() {
+            switch (this) {
+            case IDENTIFIER:
+                return "token.identifier";
+            case CHARLITERAL:
+                return "token.character";
+            case STRINGLITERAL:
+                return "token.string";
+            case INTLITERAL:
+                return "token.integer";
+            case LONGLITERAL:
+                return "token.long-integer";
+            case FLOATLITERAL:
+                return "token.float";
+            case DOUBLELITERAL:
+                return "token.double";
+            case ERROR:
+                return "token.bad-symbol";
+            case EOF:
+                return "token.end-of-input";
+            case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
+            case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
+                return "'" + name + "'";
+            default:
+                return name;
+            }
+        }
+
+        public String getKind() {
+            return "Token";
+        }
+
+        public String toString(Locale locale, Messages messages) {
+            return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
+        }
+
+        @Override
+        public boolean accepts(TokenKind that) {
+            return this == that;
+        }
+    }
+
+    public interface Comment {
+
+        enum CommentStyle {
+            LINE,
+            BLOCK,
+            JAVADOC,
+        }
+
+        String getText();
+        int getSourcePos(int index);
+        CommentStyle getStyle();
+        boolean isDeprecated();
+    }
+
+    /**
+     * This is the class representing a javac token. Each token has several fields
+     * that are set by the javac lexer (i.e. start/end position, string value, etc).
+     */
+    public static class Token {
+
+        /** tags constants **/
+        enum Tag {
+            DEFAULT,
+            NAMED,
+            STRING,
+            NUMERIC;
+        }
+
+        /** The token kind */
+        public final TokenKind kind;
+
+        /** The start position of this token */
+        public final int pos;
+
+        /** The end position of this token */
+        public final int endPos;
+
+        /** Comment reader associated with this token */
+        public final List<Comment> comments;
+
+        Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
+            this.kind = kind;
+            this.pos = pos;
+            this.endPos = endPos;
+            this.comments = comments;
+            checkKind();
+        }
+
+        Token[] split(Tokens tokens) {
+            if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
+                throw new AssertionError("Cant split" + kind);
+            }
+
+            TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
+            TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
+
+            if (t1 == null || t2 == null) {
+                throw new AssertionError("Cant split - bad subtokens");
+            }
+            return new Token[] {
+                new Token(t1, pos, pos + t1.name.length(), comments),
+                new Token(t2, pos + t1.name.length(), endPos, null)
+            };
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.DEFAULT) {
+                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+            }
+        }
+
+        public Name name() {
+            throw new UnsupportedOperationException();
+        }
+
+        public String stringVal() {
+            throw new UnsupportedOperationException();
+        }
+
+        public int radix() {
+            throw new UnsupportedOperationException();
+        }
+
+        /**
+         * Preserve classic semantics - if multiple javadocs are found on the token
+         * the last one is returned
+         */
+        public Comment comment(Comment.CommentStyle style) {
+            List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
+            return comments.isEmpty() ?
+                    null :
+                    comments.head;
+        }
+
+        /**
+         * Preserve classic semantics - deprecated should be set if at least one
+         * javadoc comment attached to this token contains the '@deprecated' string
+         */
+        public boolean deprecatedFlag() {
+            for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
+                if (c.isDeprecated()) {
+                    return true;
+                }
+            }
+            return false;
+        }
+
+        private List<Comment> getComments(Comment.CommentStyle style) {
+            if (comments == null) {
+                return List.nil();
+            } else {
+                ListBuffer<Comment> buf = new ListBuffer<>();
+                for (Comment c : comments) {
+                    if (c.getStyle() == style) {
+                        buf.add(c);
+                    }
+                }
+                return buf.toList();
+            }
+        }
+    }
+
+    final static class NamedToken extends Token {
+        /** The name of this token */
+        public final Name name;
+
+        public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
+            super(kind, pos, endPos, comments);
+            this.name = name;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.NAMED) {
+                throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
+            }
+        }
+
+        @Override
+        public Name name() {
+            return name;
+        }
+    }
+
+    static class StringToken extends Token {
+        /** The string value of this token */
+        public final String stringVal;
+
+        public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
+            super(kind, pos, endPos, comments);
+            this.stringVal = stringVal;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.STRING) {
+                throw new AssertionError("Bad token kind - expected " + Tag.STRING);
+            }
+        }
+
+        @Override
+        public String stringVal() {
+            return stringVal;
+        }
+    }
+
+    final static class NumericToken extends StringToken {
+        /** The 'radix' value of this token */
+        public final int radix;
+
+        public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
+            super(kind, pos, endPos, stringVal, comments);
+            this.radix = radix;
+        }
+
+        protected void checkKind() {
+            if (kind.tag != Tag.NUMERIC) {
+                throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
+            }
+        }
+
+        @Override
+        public int radix() {
+            return radix;
+        }
+    }
+
+    public static final Token DUMMY =
+                new Token(TokenKind.ERROR, 0, 0, null);
+}

jdkSrc/jdk8/com/sun/tools/javac/parser/UnicodeReader.java

@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2011, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.sun.tools.javac.parser;
+
+import java.nio.CharBuffer;
+import java.util.Arrays;
+
+import com.sun.tools.javac.file.JavacFileManager;
+import com.sun.tools.javac.util.ArrayUtils;
+import com.sun.tools.javac.util.Log;
+import com.sun.tools.javac.util.Name;
+import com.sun.tools.javac.util.Names;
+
+import static com.sun.tools.javac.util.LayoutCharacters.*;
+
+/** The char reader used by the javac lexer/tokenizer. Returns the sequence of
+ * characters contained in the input stream, handling unicode escape accordingly.
+ * Additionally, it provides features for saving chars into a buffer and to retrieve
+ * them at a later stage.
+ *
+ *  <p><b>This is NOT part of any supported API.
+ *  If you write code that depends on this, you do so at your own risk.
+ *  This code and its internal interfaces are subject to change or
+ *  deletion without notice.</b>
+ */
+public class UnicodeReader {
+
+    /** The input buffer, index of next character to be read,
+     *  index of one past last character in buffer.
+     */
+    protected char[] buf;
+    protected int bp;
+    protected final int buflen;
+
+    /** The current character.
+     */
+    protected char ch;
+
+    /** The buffer index of the last converted unicode character
+     */
+    protected int unicodeConversionBp = -1;
+
+    protected Log log;
+    protected Names names;
+
+    /** A character buffer for saved chars.
+     */
+    protected char[] sbuf = new char[128];
+    protected int sp;
+
+    /**
+     * Create a scanner from the input array.  This method might
+     * modify the array.  To avoid copying the input array, ensure
+     * that {@code inputLength < input.length} or
+     * {@code input[input.length -1]} is a white space character.
+     *
+     * @param sf the factory which created this Scanner
+     * @param buffer the input, might be modified
+     * Must be positive and less than or equal to input.length.
+     */
+    protected UnicodeReader(ScannerFactory sf, CharBuffer buffer) {
+        this(sf, JavacFileManager.toArray(buffer), buffer.limit());
+    }
+
+    protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) {
+        log = sf.log;
+        names = sf.names;
+        if (inputLength == input.length) {
+            if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) {
+                inputLength--;
+            } else {
+                input = Arrays.copyOf(input, inputLength + 1);
+            }
+        }
+        buf = input;
+        buflen = inputLength;
+        buf[buflen] = EOI;
+        bp = -1;
+        scanChar();
+    }
+
+    /** Read next character.
+     */
+    protected void scanChar() {
+        if (bp < buflen) {
+            ch = buf[++bp];
+            if (ch == '\\') {
+                convertUnicode();
+            }
+        }
+    }
+
+    /** Read next character in comment, skipping over double '\' characters.
+     */
+    protected void scanCommentChar() {
+        scanChar();
+        if (ch == '\\') {
+            if (peekChar() == '\\' && !isUnicode()) {
+                skipChar();
+            } else {
+                convertUnicode();
+            }
+        }
+    }
+
+    /** Append a character to sbuf.
+     */
+    protected void putChar(char ch, boolean scan) {
+        sbuf = ArrayUtils.ensureCapacity(sbuf, sp);
+        sbuf[sp++] = ch;
+        if (scan)
+            scanChar();
+    }
+
+    protected void putChar(char ch) {
+        putChar(ch, false);
+    }
+
+    protected void putChar(boolean scan) {
+        putChar(ch, scan);
+    }
+
+    Name name() {
+        return names.fromChars(sbuf, 0, sp);
+    }
+
+    String chars() {
+        return new String(sbuf, 0, sp);
+    }
+
+    /** Convert unicode escape; bp points to initial '\' character
+     *  (Spec 3.3).
+     */
+    protected void convertUnicode() {
+        if (ch == '\\' && unicodeConversionBp != bp) {
+            bp++; ch = buf[bp];
+            if (ch == 'u') {
+                do {
+                    bp++; ch = buf[bp];
+                } while (ch == 'u');
+                int limit = bp + 3;
+                if (limit < buflen) {
+                    int d = digit(bp, 16);
+                    int code = d;
+                    while (bp < limit && d >= 0) {
+                        bp++; ch = buf[bp];
+                        d = digit(bp, 16);
+                        code = (code << 4) + d;
+                    }
+                    if (d >= 0) {
+                        ch = (char)code;
+                        unicodeConversionBp = bp;
+                        return;
+                    }
+                }
+                log.error(bp, "illegal.unicode.esc");
+            } else {
+                bp--;
+                ch = '\\';
+            }
+        }
+    }
+
+    /** Are surrogates supported?
+     */
+    final static boolean surrogatesSupported = surrogatesSupported();
+    private static boolean surrogatesSupported() {
+        try {
+            Character.isHighSurrogate('a');
+            return true;
+        } catch (NoSuchMethodError ex) {
+            return false;
+        }
+    }
+
+    /** Scan surrogate pairs.  If 'ch' is a high surrogate and
+     *  the next character is a low surrogate, then put the low
+     *  surrogate in 'ch', and return the high surrogate.
+     *  otherwise, just return 0.
+     */
+    protected char scanSurrogates() {
+        if (surrogatesSupported && Character.isHighSurrogate(ch)) {
+            char high = ch;
+
+            scanChar();
+
+            if (Character.isLowSurrogate(ch)) {
+                return high;
+            }
+
+            ch = high;
+        }
+
+        return 0;
+    }
+
+    /** Convert an ASCII digit from its base (8, 10, or 16)
+     *  to its value.
+     */
+    protected int digit(int pos, int base) {
+        char c = ch;
+        int result = Character.digit(c, base);
+        if (result >= 0 && c > 0x7f) {
+            log.error(pos + 1, "illegal.nonascii.digit");
+            ch = "0123456789abcdef".charAt(result);
+        }
+        return result;
+    }
+
+    protected boolean isUnicode() {
+        return unicodeConversionBp == bp;
+    }
+
+    protected void skipChar() {
+        bp++;
+    }
+
+    protected char peekChar() {
+        return buf[bp + 1];
+    }
+
+    /**
+     * Returns a copy of the input buffer, up to its inputLength.
+     * Unicode escape sequences are not translated.
+     */
+    public char[] getRawCharacters() {
+        char[] chars = new char[buflen];
+        System.arraycopy(buf, 0, chars, 0, buflen);
+        return chars;
+    }
+
+    /**
+     * Returns a copy of a character array subset of the input buffer.
+     * The returned array begins at the {@code beginIndex} and
+     * extends to the character at index {@code endIndex - 1}.
+     * Thus the length of the substring is {@code endIndex-beginIndex}.
+     * This behavior is like
+     * {@code String.substring(beginIndex, endIndex)}.
+     * Unicode escape sequences are not translated.
+     *
+     * @param beginIndex the beginning index, inclusive.
+     * @param endIndex the ending index, exclusive.
+     * @throws ArrayIndexOutOfBoundsException if either offset is outside of the
+     *         array bounds
+     */
+    public char[] getRawCharacters(int beginIndex, int endIndex) {
+        int length = endIndex - beginIndex;
+        char[] chars = new char[length];
+        System.arraycopy(buf, beginIndex, chars, 0, length);
+        return chars;
+    }
+}