NdogenLexer.java

/**
 *  Copyright 2011 Oliver Buchtala
 *
 *	This file is part of ndogen.
 *
 *  ndogen is free software: you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation, either version 3 of the License, or
 *  (at your option) any later version.
 *
 *  ndogen is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with ndogen.  If not, see <http://www.gnu.org/licenses/>.
 */
package org.ndogen.antlr;

import java.util.BitSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;
import java.util.Stack;

import org.antlr.runtime.CharStream;
import org.antlr.runtime.Lexer;
import org.antlr.runtime.RecognizerSharedState;
import org.antlr.runtime.Token;
import org.ndogen.NamedToken;

public abstract class NdogenLexer extends Lexer {
	
	private Queue<Token> tokenQueue = new LinkedList<Token>();
	protected Stack<Token> passedTokens = new Stack<Token>();
	protected Token lastToken = null;
	
	protected BitSet states = new BitSet();
	
	LinkedList<OnEmit> onEmits = new LinkedList<NdogenLexer.OnEmit>();
	
	public NdogenLexer() {
		super();
	}

	public NdogenLexer(CharStream input) {
		super(input);
		initialize();
	}

	public NdogenLexer(CharStream input, RecognizerSharedState state) {
		super(input, state);
		initialize();
	}
	
	public abstract void initialize();
	
	@Override
	public void emit(Token token) {
		
		if(!(token instanceof NamedToken)) {
			token = new NamedToken(getTokenName(token.getType()), token);
		}
		super.emit(token);
		
		lastToken = token;
		this.tokenQueue.add(token);
		
		for (OnEmit onEmit : onEmits) {
			onEmit.emitted(token);
		}
	}

	@Override
	public Token nextToken() {
		if(!this.tokenQueue.isEmpty()) {
			return deliver(tokenQueue.poll());
		}
		Token nextToken = super.nextToken();
		if(!this.tokenQueue.isEmpty()) {
			return deliver(tokenQueue.poll());
		} else {
			return deliver(nextToken);
		}
	}
	
	private Token deliver(Token token) {
		this.passedTokens.push(token);
		return token;
	}
	
	protected int last() {
		if(lastToken== null) {
			return -42;
		} else {
			return lastToken.getType();
		}
	}
	
	public abstract String getTokenName(int type);

	protected void onEmit(OnEmit onemit) {
		onEmits.add(onemit);
	}

	public interface OnEmit {
		void emitted(Token t);
	}
	
}