Webiyo

org/webiyo/examples/viewjava/Lexer.java

package org.webiyo.examples.viewjava;

import org.webiyo.util.ImpossibleException;

import java.io.IOException;
import java.io.PushbackReader;
import java.io.StringReader;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

import static org.webiyo.examples.viewjava.TokenType.*;

class Lexer implements Iterator<Token> {

    private static final Set<String> KEYWORDS =
            new HashSet<String>(Arrays.asList("package", "import",
                                              "public", "protected", "private",
                                              "class", "extends", "implements", "throws",
                                              "return", "throw", "if", "while", "break", "continue", "switch", "case",
                                              "void", "boolean", "char", "short", "int", "long", "float", "double",
                                              "new",
                                              "try", "catch", "finally"));

    private final PushbackReader reader;
    private final StringBuilder currentValue = new StringBuilder();

    public Lexer(String input) {
        reader = new PushbackReader(new StringReader(input), 2);
    }

    public boolean hasNext() {
        return peek() != -1;
    }

    public Token next() {
        currentValue.setLength(0);

        int c = read();
        if (c == '/') {
            if (peek() == '*') {
                readBlockComment();
                return newToken(COMMENT);
            } else if (peek() == '/') {
                readLineComment();
                return newToken(COMMENT);
            }
        }

        if (c == '"') {
            readStringLiteral();
            return newToken(STRING);
        } else if (c == '\'') {
            readCharLiteral();
            return newToken(STRING);
        } else if (Character.isJavaIdentifierStart(c)) {
            readIdentifier();
            return newToken(KEYWORDS.contains(currentValue()) ? KEYWORD : NAME);
        } else if (Character.isWhitespace(c)) {
            readWhitespace();
            return newToken(WHITESPACE);
        } else {
            return newToken(SYMBOL);
        }
    }

    public void remove() {
        throw new UnsupportedOperationException();
    }
    
    // end of public methods

    private Token newToken(TokenType type) {
        return new Token(type, currentValue());
    }

    private void readBlockComment() {
        read();
        int c = read();
        while (c != -1 && c != '/') {
            c = read();
        }
    }

    private void readLineComment() {
        read();
        int c = read();
        while (c != -1 && c != '\n') {
            c = read();
        }
        if (c != -1) unread();
    }

    private void readStringLiteral() {
        int c = read();
        if (c == '\\') read();
        while (c != -1 && c != '"') {
            c = read();
            if (c == '\\') read();
        }
    }

    private void readCharLiteral() {
        int c = read();
        if (c == '\\') read();
        while (c != -1 && c != '\'') {
            c = read();
            if (c == '\\') read();
        }
    }

    private void readIdentifier() {
        int c;
        c = read();
        while (c != -1 && Character.isJavaIdentifierPart(c)) {
            c = read();
        }
        if (c != -1) unread();
    }

    private void readWhitespace() {
        int c = read();
        while (Character.isWhitespace(c)) {
            c = read();
        }
        if (c != -1) unread();
    }

    private String currentValue() {
        return currentValue.toString();
    }

    private int read() {
        try {
            int c = reader.read();
            if (c != -1) currentValue.append((char) c);
            return c;
        } catch (IOException e) {
            throw new ImpossibleException(e);
        }
    }

    private void unread() {
        try {
            if (currentValue.length() == 0) return;
            int c = currentValue.charAt(currentValue.length() - 1);
            currentValue.setLength(currentValue.length() - 1);
            reader.unread(c);
        } catch (IOException e) {
            throw new ImpossibleException(e);
        }
    }

    private int peek() {
        int c = read();
        unread();
        return c;
    }

}