MarkdownLexer.java

package flow.netbeans.markdown.highlighter;

import flow.netbeans.markdown.options.MarkdownGlobalOptions;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.netbeans.api.lexer.Token;
import org.netbeans.spi.lexer.Lexer;
import org.netbeans.spi.lexer.LexerInput;
import org.netbeans.spi.lexer.LexerRestartInfo;
import org.pegdown.ParsingTimeoutException;
import org.pegdown.PegDownProcessor;
import org.pegdown.ast.RootNode;


class MarkdownLexer implements Lexer<MarkdownTokenId> {
    private static final Logger LOG = Logger.getLogger(MarkdownLexer.class.getName());

    static Lexer<MarkdownTokenId> create(LexerRestartInfo<MarkdownTokenId> info) {
        return new MarkdownLexer(info);
    }
    
    private final LexerRestartInfo<MarkdownTokenId> info;
    boolean inited = false;
    private Iterator<MarkdownToken> tokenIterator = null;

    MarkdownLexer(LexerRestartInfo<MarkdownTokenId> info) {
        this.info = info;
    }

    @Override
    public Token<MarkdownTokenId> nextToken() {
        // Tokenize the input on the first nextToken() call
        if (!inited) {
            synchronized (this) {
                if (!inited) {
                    // TODO: Handle ParsingTimeoutException.
                    tokenizeInput();
                    inited = true;
                }
            }
        }

        final int readLength = info.input().readLength();

        // Nothing here
        if(readLength <= 0) {
            return null;
        }
        
        // Retrieve the next token from the iterator.
        if(tokenIterator.hasNext()) {
            MarkdownToken token = tokenIterator.next();
            return info.tokenFactory().createToken(token.getId(), token.getLength());
        }

        //  Legacy safety net. @todo remove this
        if(readLength > 0) {
            LOG.log(Level.WARNING, "Caught by legacy safety net");
            return info.tokenFactory().createToken(MarkdownTokenId.PLAIN, readLength);
        }
        
        return null;
    }
    
    /**
     * Tokenizes the input
     */
    private void tokenizeInput() {
        MarkdownGlobalOptions markdownOptions = MarkdownGlobalOptions.getInstance();
        PegDownProcessor markdownProcessor = new PegDownProcessor(markdownOptions.getExtensionsValue());

        // Read the complete input and feed the PegDown Parser
        // Selected extensions over plain markdown are taken in account
        char[] markdownSource = readAll(info.input());

        int inputLength = markdownSource.length;

        List<MarkdownToken> tokens;
        try {
            RootNode rootNode = markdownProcessor.parseMarkdown(markdownSource);

            MarkdownTokenListBuilder builder = new MarkdownTokenListBuilder(inputLength);
            MarkdownLexerVisitor visitor = new MarkdownLexerVisitor(builder);
            rootNode.accept(visitor);
            tokens = builder.build();
        }
        catch (ParsingTimeoutException ex) {
            LOG.log(Level.WARNING, "Time out while parsing Markdown source, falling back to no highlighting");
            tokens = Collections.singletonList(new MarkdownToken(MarkdownTokenId.PLAIN, 0, inputLength));
        }

        tokenIterator = tokens.iterator();
    }
    
    private char[] readAll(LexerInput lexerInput) {
        StringBuilder sb = new StringBuilder();
        int i;
        while ((i = lexerInput.read()) != LexerInput.EOF) {
            sb.append((char) i);
        }
        char[] markdownSource = sb.toString().toCharArray();
        return markdownSource;
    }
    
    @Override
    public Object state() {
        return null;
    }

    @Override
    public void release() {
    }
}