PythonTokenMarker.java

/*
 * Copyright 2004-2014 SmartBear Software
 *
 * Licensed under the EUPL, Version 1.1 or - as soon as they will be approved by the European Commission - subsequent
 * versions of the EUPL (the "Licence");
 * You may not use this work except in compliance with the Licence.
 * You may obtain a copy of the Licence at:
 *
 * http://ec.europa.eu/idabc/eupl
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the Licence is
 * distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the Licence for the specific language governing permissions and limitations
 * under the Licence.
*/

package org.syntax.jedit.tokenmarker;

import javax.swing.text.Segment;

import org.syntax.jedit.KeywordMap;
import org.syntax.jedit.SyntaxUtilities;

/**
 * Python token marker.
 *
 * @author Jonathan Revusky
 * @version $Id: PythonTokenMarker.java,v 1.3 1999/12/14 04:20:35 sp Exp $
 */
public class PythonTokenMarker extends TokenMarker {
    private static final byte TRIPLEQUOTE1 = Token.INTERNAL_FIRST;
    private static final byte TRIPLEQUOTE2 = Token.INTERNAL_LAST;

    public PythonTokenMarker() {
        this.keywords = getKeywords();
    }

    public byte markTokensImpl(byte token, Segment line, int lineIndex) {
        char[] array = line.array;
        int offset = line.offset;
        lastOffset = offset;
        lastKeyword = offset;
        int length = line.count + offset;
        boolean backslash = false;

        loop:
        for (int i = offset; i < length; i++) {
            int i1 = (i + 1);

            char c = array[i];
            if (c == '\\') {
                backslash = !backslash;
                continue;
            }

            switch (token) {
                case Token.NULL:
                    switch (c) {
                        case '#':
                            if (backslash) {
                                backslash = false;
                            } else {
                                doKeyword(line, i, c);
                                addToken(i - lastOffset, token);
                                addToken(length - i, Token.COMMENT1);
                                lastOffset = lastKeyword = length;
                                break loop;
                            }
                            break;
                        case '"':
                            doKeyword(line, i, c);
                            if (backslash) {
                                backslash = false;
                            } else {
                                addToken(i - lastOffset, token);
                                if (SyntaxUtilities.regionMatches(false, line, i1, "\"\"")) {
                                    token = TRIPLEQUOTE1;
                                } else {
                                    token = Token.LITERAL1;
                                }
                                lastOffset = lastKeyword = i;
                            }
                            break;
                        case '\'':
                            doKeyword(line, i, c);
                            if (backslash) {
                                backslash = false;
                            } else {
                                addToken(i - lastOffset, token);
                                if (SyntaxUtilities.regionMatches(false, line, i1, "''")) {
                                    token = TRIPLEQUOTE2;
                                } else {
                                    token = Token.LITERAL2;
                                }
                                lastOffset = lastKeyword = i;
                            }
                            break;
                        default:
                            backslash = false;
                            if (!Character.isLetterOrDigit(c) && c != '_') {
                                doKeyword(line, i, c);
                            }
                            break;
                    }
                    break;
                case Token.LITERAL1:
                    if (backslash) {
                        backslash = false;
                    } else if (c == '"') {
                        addToken(i1 - lastOffset, token);
                        token = Token.NULL;
                        lastOffset = lastKeyword = i1;
                    }
                    break;
                case Token.LITERAL2:
                    if (backslash) {
                        backslash = false;
                    } else if (c == '\'') {
                        addToken(i1 - lastOffset, Token.LITERAL1);
                        token = Token.NULL;
                        lastOffset = lastKeyword = i1;
                    }
                    break;
                case TRIPLEQUOTE1:
                    if (backslash) {
                        backslash = false;
                    } else if (SyntaxUtilities.regionMatches(false, line, i, "\"\"\"")) {
                        addToken((i += 4) - lastOffset, Token.LITERAL1);
                        token = Token.NULL;
                        lastOffset = lastKeyword = i;
                    }
                    break;
                case TRIPLEQUOTE2:
                    if (backslash) {
                        backslash = false;
                    } else if (SyntaxUtilities.regionMatches(false, line, i, "'''")) {
                        addToken((i += 4) - lastOffset, Token.LITERAL1);
                        token = Token.NULL;
                        lastOffset = lastKeyword = i;
                    }
                    break;
                default:
                    throw new InternalError("Invalid state: " + token);
            }
        }

        switch (token) {
            case TRIPLEQUOTE1:
            case TRIPLEQUOTE2:
                addToken(length - lastOffset, Token.LITERAL1);
                break;
            case Token.NULL:
                doKeyword(line, length, '\0');
            default:
                addToken(length - lastOffset, token);
                break;
        }

        return token;
    }

    public static KeywordMap getKeywords() {
        if (pyKeywords == null) {
            pyKeywords = new KeywordMap(false);
            pyKeywords.add("and", Token.KEYWORD3);
            pyKeywords.add("not", Token.KEYWORD3);
            pyKeywords.add("or", Token.KEYWORD3);
            pyKeywords.add("if", Token.KEYWORD1);
            pyKeywords.add("for", Token.KEYWORD1);
            pyKeywords.add("assert", Token.KEYWORD1);
            pyKeywords.add("break", Token.KEYWORD1);
            pyKeywords.add("continue", Token.KEYWORD1);
            pyKeywords.add("elif", Token.KEYWORD1);
            pyKeywords.add("else", Token.KEYWORD1);
            pyKeywords.add("except", Token.KEYWORD1);
            pyKeywords.add("exec", Token.KEYWORD1);
            pyKeywords.add("finally", Token.KEYWORD1);
            pyKeywords.add("raise", Token.KEYWORD1);
            pyKeywords.add("return", Token.KEYWORD1);
            pyKeywords.add("try", Token.KEYWORD1);
            pyKeywords.add("while", Token.KEYWORD1);
            pyKeywords.add("def", Token.KEYWORD2);
            pyKeywords.add("class", Token.KEYWORD2);
            pyKeywords.add("del", Token.KEYWORD2);
            pyKeywords.add("from", Token.KEYWORD2);
            pyKeywords.add("global", Token.KEYWORD2);
            pyKeywords.add("import", Token.KEYWORD2);
            pyKeywords.add("in", Token.KEYWORD2);
            pyKeywords.add("is", Token.KEYWORD2);
            pyKeywords.add("lambda", Token.KEYWORD2);
            pyKeywords.add("pass", Token.KEYWORD2);
            pyKeywords.add("print", Token.KEYWORD2);
        }
        return pyKeywords;
    }

    // private members
    private static KeywordMap pyKeywords;

    private KeywordMap keywords;
    private int lastOffset;
    private int lastKeyword;

    private boolean doKeyword(Segment line, int i, char c) {
        int i1 = i + 1;

        int len = i - lastKeyword;
        byte id = keywords.lookup(line, lastKeyword, len);
        if (id != Token.NULL) {
            if (lastKeyword != lastOffset) {
                addToken(lastKeyword - lastOffset, Token.NULL);
            }
            addToken(len, id);
            lastOffset = i;
        }
        lastKeyword = i1;
        return false;
    }
}