/* * Copyright (c) 2012, the Dart project authors. * * Licensed under the Eclipse Public License v1.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at * * http://www.eclipse.org/legal/epl-v10.html * * Unless required by applicable law or agreed to in writing, software distributed under the License * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express * or implied. See the License for the specific language governing permissions and limitations under * the License. */ package com.google.dart.tools.ui.web.css.model; import com.google.dart.tools.ui.web.css.CssNumberDetector; import com.google.dart.tools.ui.web.css.CssPartitionScanner; import com.google.dart.tools.ui.web.css.CssWordDetector; import com.google.dart.tools.ui.web.utils.AnyWordRule; import com.google.dart.tools.ui.web.utils.Token; import com.google.dart.tools.ui.web.utils.WhitespaceDetector; import org.eclipse.jface.text.BadLocationException; import org.eclipse.jface.text.Document; import org.eclipse.jface.text.IDocument; import org.eclipse.jface.text.rules.EndOfLineRule; import org.eclipse.jface.text.rules.IRule; import org.eclipse.jface.text.rules.IToken; import org.eclipse.jface.text.rules.MultiLineRule; import org.eclipse.jface.text.rules.RuleBasedScanner; import org.eclipse.jface.text.rules.SingleLineRule; import org.eclipse.jface.text.rules.WhitespaceRule; import java.io.CharArrayWriter; import java.io.IOException; import java.io.Reader; import java.util.ArrayList; import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; import java.util.Stack; /** * Convert css content into a stream of tokens. */ class Tokenizer implements Iterator<Token> { private static IDocument createDocument(Reader in) throws IOException { String content = readContent(in); return new Document(content); } private static String readContent(Reader in) throws IOException { CharArrayWriter out = new CharArrayWriter(); char[] buffer = new char[4096]; int count = in.read(buffer); while (count != -1) { out.write(buffer, 0, count); count = in.read(buffer); } in.close(); return out.toString(); } private RuleBasedScanner scanner; private IDocument document; private Stack<Token> tokenStack = new Stack<Token>(); private static IToken COMMENT_TOKEN = new org.eclipse.jface.text.rules.Token( CssPartitionScanner.CSS_COMMENT); public Tokenizer(IDocument document) { this.document = document; setupScanner(document); } public Tokenizer(Reader reader) throws IOException { this(createDocument(reader)); } @Override public boolean hasNext() { if (!tokenStack.isEmpty()) { return true; } if (scanner == null) { return false; } advance(); return !tokenStack.isEmpty(); } @Override public Token next() { if (!tokenStack.isEmpty()) { return tokenStack.pop(); } advance(); if (tokenStack.isEmpty()) { throw new NoSuchElementException(); } else { return tokenStack.pop(); } } public boolean peek(String str) { if (!hasNext()) { return false; } Token token = next(); if (token != null) { boolean result = str.equals(token.getValue()); pushBack(token); return result; } else { return false; } } public void pushBack(Token token) { tokenStack.push(token); } @Override public void remove() { throw new UnsupportedOperationException(); } private void advance() { while (scanner != null && tokenStack.isEmpty()) { IToken t = scanner.nextToken(); if (t.isEOF()) { scanner = null; } else if (!t.isWhitespace() && !isComment(t)) { try { tokenStack.push(new Token( document.get(scanner.getTokenOffset(), scanner.getTokenLength()), scanner.getTokenOffset(), t.isWhitespace())); } catch (BadLocationException e) { throw new RuntimeException(e); } } } } private boolean isComment(IToken t) { return t == COMMENT_TOKEN; } private void setupScanner(IDocument document) { IToken stringToken = new org.eclipse.jface.text.rules.Token("word"); scanner = new RuleBasedScanner(); List<IRule> rules = new ArrayList<IRule>(); rules.add(new MultiLineRule("/*", "*/", COMMENT_TOKEN)); rules.add(new EndOfLineRule("//", COMMENT_TOKEN)); rules.add(new SingleLineRule("\"", "\"", stringToken, '\\')); rules.add(new SingleLineRule("'", "'", stringToken, '\\')); rules.add(new AnyWordRule(new CssWordDetector())); rules.add(new AnyWordRule(new CssNumberDetector())); rules.add(new WhitespaceRule(new WhitespaceDetector())); scanner.setRules(rules.toArray(new IRule[rules.size()])); scanner.setRange(document, 0, document.getLength()); } }