/* The following code was generated by JFlex 1.2.2 on 8/16/05 3:07 PM */ /******************************************************************************* * Copyright (c) 2004 IBM Corporation and others. * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: * IBM Corporation - initial API and implementation *******************************************************************************/ /*nlsXXX*/ package org.eclipse.dltk.javascript.internal.ui.text; import java.io.CharArrayReader; import java.io.IOException; import java.util.ArrayList; import java.util.Iterator; import java.util.List; /* * The Apache Software License, Version 1.1 * * * Copyright (c) 2000-2002 The Apache Software Foundation. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * 3. The end-user documentation included with the redistribution, if any, must * include the following acknowledgment: "This product includes software * developed by the Apache Software Foundation (http://www.apache.org/)." * Alternately, this acknowledgment may appear in the software itself, if and * wherever such third-party acknowledgments normally appear. * * 4. The names "Xerces" and "Apache Software Foundation" must not be used to * endorse or promote products derived from this software without prior written * permission. For written permission, please contact apache@apache.org. * * 5. Products derived from this software may not be called "Apache", nor may * "Apache" appear in their name, without prior written permission of the Apache * Software Foundation. * * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND * FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE APACHE * SOFTWARE FOUNDATION OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, * OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * ==================================================================== * * This software consists of voluntary contributions made by many individuals on * behalf of the Apache Software Foundation and was originally based on software * copyright (c) 1999, International Business Machines, Inc., * http://www.apache.org. For more information on the Apache Software * Foundation, please see <http://www.apache.org/>. */ /** * A simple integer based stack. * * moved to org.apache.xerces.util by neilg to support the XPathMatcher. * * @author Andy Clark, IBM * * @version $Id: XMLTokenizer.java,v 1.1 2007/05/24 09:03:20 aplatov Exp $ */ final class IntStack { // // Data // /** Stack depth. */ private int fDepth; /** Stack data. */ private int[] fData; // // Public methods // /** Returns the size of the stack. */ public int size() { return fDepth; } /** Pushes a value onto the stack. */ public void push(int value) { ensureCapacity(fDepth + 1); fData[fDepth++] = value; } /** Peeks at the top of the stack. */ public int peek() { return fData[fDepth - 1]; } /** Pops a value off of the stack. */ public int pop() { return fData[--fDepth]; } /** Clears the stack. */ public void clear() { fDepth = 0; } // debugging /** Prints the stack. */ public void print() { System.out.print('('); System.out.print(fDepth); System.out.print(") {"); for (int i = 0; i < fDepth; i++) { if (i == 3) { System.out.print(" ..."); break; } System.out.print(' '); System.out.print(fData[i]); if (i < fDepth - 1) { System.out.print(','); } } System.out.print(" }"); System.out.println(); } // // Private methods // /** Ensures capacity. */ private void ensureCapacity(int size) { if (fData == null) { fData = new int[32]; } else if (fData.length <= size) { int[] newdata = new int[fData.length * 2]; System.arraycopy(fData, 0, newdata, 0, fData.length); fData = newdata; } } } // class IntStack /** * This class is a scanner generated by <a * href="http://www.informatik.tu-muenchen.de/~kleing/jflex/">JFlex</a> 1.2.2 * on 8/16/05 3:07 PM from the specification file * <tt>file:/D:/eclipse.wtp/workspace/org.eclipse.wst.sse.core/DevTimeSupport/SedModel/HTMLTokenizer/devel/XMLTokenizer.jflex</tt> */ public class XMLTokenizer { /** this character denotes the end of file */ final public static int YYEOF = -1; /** lexical states */ final public static int ST_XML_DOCTYPE_EXTERNAL_ID = 23; final public static int ST_XML_ELEMENT_DECLARATION_CONTENT = 27; final public static int ST_DHTML_ATTRIBUTE_NAME = 12; final public static int ST_XML_PI_TAG_CLOSE = 11; final public static int ST_XML_DECLARATION_CLOSE = 21; final public static int ST_XML_PI_ATTRIBUTE_VALUE = 10; final public static int ST_DHTML_EQUALS = 13; final public static int ST_XML_TAG_NAME = 16; final public static int ST_XML_ATTRIBUTE_VALUE = 19; final public static int ST_DHTML_ATTRIBUTE_VALUE = 14; final public static int ST_XML_DOCTYPE_ID_SYSTEM = 25; final public static int ST_XML_ATTRIBUTE_NAME = 17; final public static int ST_XML_ELEMENT_DECLARATION = 26; final public static int ST_XML_DOCTYPE_DECLARATION = 22; final public static int ST_XML_ATTLIST_DECLARATION = 28; final public static int ST_XML_COMMENT_END = 4; final public static int ST_CDATA_TEXT = 1; final public static int ST_DHTML_TAG_CLOSE = 15; final public static int ST_XML_COMMENT = 3; final public static int ST_PI_CONTENT = 7; final public static int ST_PI_WS = 6; final public static int ST_CDATA_END = 2; final public static int ST_XML_ATTLIST_DECLARATION_CONTENT = 29; final public static int ST_BLOCK_TAG_SCAN = 30; final public static int ST_XML_PI_EQUALS = 9; final public static int ST_XML_DECLARATION = 20; final public static int YYINITIAL = 0; final public static int ST_XML_DOCTYPE_ID_PUBLIC = 24; final public static int ST_XML_EQUALS = 18; final public static int ST_PI = 5; final public static int ST_XML_PI_ATTRIBUTE_NAME = 8; /** * Translates characters to character classes */ final private static String yycmap_packed = "\11\0\1\5\1\22\2\0\1\14\22\0\1\14\1\21\1\11\1\51" + "\1\16\1\17\1\12\1\13\1\16\1\16\1\16\1\16\1\16\1\7" + "\1\6\1\3\12\15\1\10\1\54\1\1\1\43\1\2\1\4\1\16" + "\1\32\1\55\1\30\1\31\1\35\1\52\1\34\1\34\1\40\1\34" + "\1\34\1\26\1\25\1\42\1\41\1\45\1\34\1\36\1\37\1\33" + "\1\53\2\34\1\23\1\44\1\34\1\27\1\0\1\20\1\0\1\10" + "\1\0\1\47\1\55\1\56\1\50\1\35\1\52\1\34\1\34\1\40" + "\2\34\1\26\1\25\1\42\1\41\1\45\1\34\1\36\1\37\1\46" + "\1\53\1\34\1\34\1\24\1\44\1\34\1\0\1\0\72\0\1\60" + "\10\0\27\57\1\0\37\57\1\0\72\57\2\0\13\57\2\0\10\57" + "\1\0\65\57\1\0\104\57\11\0\44\57\3\0\2\57\4\0\36\57" + "\70\0\131\57\22\0\7\57\16\0\2\60\56\0\106\60\32\0\2\60" + "\44\0\1\57\1\60\3\57\1\0\1\57\1\0\24\57\1\0\54\57" + "\1\0\7\57\3\0\1\57\1\0\1\57\1\0\1\57\1\0\1\57" + "\1\0\22\57\15\0\14\57\1\0\102\57\1\0\14\57\1\0\44\57" + "\1\0\4\60\11\0\65\57\2\0\2\57\2\0\2\57\3\0\34\57" + "\2\0\10\57\2\0\2\57\67\0\46\57\2\0\1\57\7\0\46\57" + "\12\0\21\60\1\0\27\60\1\0\3\60\1\0\1\60\1\0\2\60" + "\1\0\1\60\13\0\33\57\5\0\3\57\56\0\32\57\5\0\1\60" + "\12\57\10\60\15\0\12\60\6\0\1\60\107\57\2\0\5\57\1\0" + "\17\57\1\0\4\57\1\0\1\57\17\60\2\57\2\60\1\0\4\60" + "\2\0\12\60\u0207\0\3\60\1\0\65\57\2\0\1\60\1\57\20\60" + "\3\0\4\60\3\0\12\57\2\60\2\0\12\60\21\0\3\60\1\0" + "\10\57\2\0\2\57\2\0\26\57\1\0\7\57\1\0\1\57\3\0" + "\4\57\2\0\1\60\1\0\7\60\2\0\2\60\2\0\3\60\11\0" + "\1\60\4\0\2\57\1\0\3\57\2\60\2\0\12\60\2\57\20\0" + "\1\60\2\0\6\57\4\0\2\57\2\0\26\57\1\0\7\57\1\0" + "\2\57\1\0\2\57\1\0\2\57\2\0\1\60\1\0\5\60\4\0" + "\2\60\2\0\3\60\13\0\4\57\1\0\1\57\7\0\12\60\2\60" + "\3\57\14\0\3\60\1\0\7\57\1\0\1\57\1\0\3\57\1\0" + "\26\57\1\0\7\57\1\0\2\57\1\0\5\57\2\0\1\60\1\57" + "\10\60\1\0\3\60\1\0\3\60\22\0\1\57\5\0\12\60\21\0" + "\3\60\1\0\10\57\2\0\2\57\2\0\26\57\1\0\7\57\1\0" + "\2\57\2\0\4\57\2\0\1\60\1\57\6\60\3\0\2\60\2\0" + "\3\60\10\0\2\60\4\0\2\57\1\0\3\57\4\0\12\60\22\0" + "\2\60\1\0\6\57\3\0\3\57\1\0\4\57\3\0\2\57\1\0" + "\1\57\1\0\2\57\3\0\2\57\3\0\3\57\3\0\10\57\1\0" + "\3\57\4\0\5\60\3\0\3\60\1\0\4\60\11\0\1\60\17\0" + "\11\60\21\0\3\60\1\0\10\57\1\0\3\57\1\0\27\57\1\0" + "\12\57\1\0\5\57\4\0\7\60\1\0\3\60\1\0\4\60\7\0" + "\2\60\11\0\2\57\4\0\12\60\22\0\2\60\1\0\10\57\1\0" + "\3\57\1\0\27\57\1\0\12\57\1\0\5\57\4\0\7\60\1\0" + "\3\60\1\0\4\60\7\0\2\60\7\0\1\57\1\0\2\57\4\0" + "\12\60\22\0\2\60\1\0\10\57\1\0\3\57\1\0\27\57\1\0" + "\20\57\4\0\6\60\2\0\3\60\1\0\4\60\11\0\1\60\10\0" + "\2\57\4\0\12\60\221\0\56\57\1\0\1\57\1\60\2\57\7\60" + "\5\0\6\57\1\60\10\60\1\0\12\60\47\0\2\57\1\0\1\57" + "\2\0\2\57\1\0\1\57\2\0\1\57\6\0\4\57\1\0\7\57" + "\1\0\3\57\1\0\1\57\1\0\1\57\2\0\2\57\1\0\2\57" + "\1\0\1\57\1\60\2\57\6\60\1\0\2\60\1\57\2\0\5\57" + "\1\0\1\60\1\0\6\60\2\0\12\60\76\0\2\60\6\0\12\60" + "\13\0\1\60\1\0\1\60\1\0\1\60\4\0\2\60\10\57\1\0" + "\41\57\7\0\24\60\1\0\6\60\4\0\6\60\1\0\1\60\1\0" + "\25\60\3\0\7\60\1\0\1\60\346\0\46\57\12\0\47\57\11\0" + "\1\57\1\0\2\57\1\0\3\57\1\0\1\57\1\0\2\57\1\0" + "\5\57\51\0\1\57\1\0\1\57\1\0\1\57\13\0\1\57\1\0" + "\1\57\1\0\1\57\3\0\2\57\3\0\1\57\5\0\3\57\1\0" + "\1\57\1\0\1\57\1\0\1\57\1\0\1\57\3\0\2\57\3\0" + "\2\57\1\0\1\57\50\0\1\57\11\0\1\57\2\0\1\57\2\0" + "\2\57\7\0\2\57\1\0\1\57\1\0\7\57\50\0\1\57\4\0" + "\1\57\10\0\1\57\u0c06\0\234\57\4\0\132\57\6\0\26\57\2\0" + "\6\57\2\0\46\57\2\0\6\57\2\0\10\57\1\0\1\57\1\0" + "\1\57\1\0\1\57\1\0\37\57\2\0\65\57\1\0\7\57\1\0" + "\1\57\3\0\3\57\1\0\7\57\3\0\4\57\2\0\6\57\4\0" + "\15\57\5\0\3\57\1\0\7\57\323\0\15\60\4\0\1\60\104\0" + "\1\57\3\0\2\57\2\0\1\57\121\0\3\57\u0e82\0\1\60\1\0" + "\1\57\31\0\11\57\6\60\1\0\5\60\13\0\124\57\4\0\2\60" + "\2\0\2\60\2\0\132\57\1\0\3\60\6\0\50\57\u1cd3\0\u51a6\57" + "\u0c5a\0\u2ba4\57\134\0\u0800\0\u1ffe\0\2\0"; /** * Translates characters to character classes */ final private static char[] yycmap = yy_unpack_cmap(yycmap_packed); /** * Translates a state to a row index in the transition table */ final private static int yy_rowMap[] = { 0, 49, 98, 147, 196, 245, 294, 343, 392, 441, 490, 539, 588, 637, 686, 735, 784, 833, 882, 931, 980, 1029, 1078, 1127, 1176, 1225, 1274, 1323, 1372, 1421, 1470, 1519, 1568, 1617, 1666, 1715, 1764, 1715, 1764, 1813, 1715, 1715, 1764, 1862, 1911, 1960, 2009, 2058, 2107, 2156, 1715, 1764, 2205, 2254, 2303, 1715, 2352, 2352, 2401, 2450, 2499, 2205, 1715, 2548, 2597, 1715, 2646, 2695, 2744, 2793, 2842, 2891, 1715, 2940, 2989, 3038, 3087, 1715, 3136, 3185, 3234, 3283, 3332, 1715, 3381, 3430, 3479, 3528, 3577, 3626, 3675, 3724, 3724, 3773, 3822, 3871, 3920, 3920, 3969, 4018, 4067, 4116, 4116, 4165, 4214, 4263, 4312, 1715, 4361, 4361, 4410, 4459, 4508, 4557, 1715, 1715, 1764, 1715, 1715, 4606, 4655, 4704, 4753, 4802, 4851, 4900, 4949, 1715, 4998, 5047, 1715, 1715, 2352, 5096, 2450, 1715, 5145, 2499, 2548, 2646, 2695, 5194, 2744, 1715, 5243, 2793, 1715, 3136, 5292, 3234, 1715, 5341, 3283, 4606, 5390, 5439, 5488, 3528, 1715, 5537, 5586, 3724, 5635, 3773, 1715, 5684, 5733, 5782, 5782, 5831, 5880, 3871, 3724, 3920, 5929, 3969, 1715, 5978, 4018, 4067, 3920, 4116, 6027, 4165, 1715, 6076, 6125, 6174, 6174, 6223, 6272, 6321, 4361, 6370, 4410, 1715, 6419, 6468, 6517, 6517, 6566, 6615, 6664, 6713, 6762, 6811, 6860, 1715, 6909, 6958, 1715, 1715, 1715, 2009, 7007, 7056, 7105, 7154, 7203, 7252, 5684, 7301, 7301, 6076, 7350, 7350, 7399, 6419, 7448, 7448, 7497, 1715, 7546, 7595, 1715, 7644, 7693, 7742, 7791, 7840, 7889, 7938, 5831, 6223, 7987, 6566, 8036, 8085, 8134, 8183, 8232, 8281, 8330, 8379, 8428, 8477, 8526, 8575, 8624, 8673, 8722, 1715, 1715, 8771, 8820, 8869, 8918, 1715, 1715, 1715, 8967, 9016, 9065, 9114, 9163, 1715, 4263, 4508 }; /** * The packed transition table of the DFA */ final private static String yy_packed = "\1\40\1\41\10\40\1\42\4\40\1\43\41\40\1\44" + "\1\45\57\44\1\46\1\47\16\46\1\50\1\46\1\51" + "\36\46\1\52\1\53\57\52\1\46\1\47\5\46\1\54" + "\12\46\1\51\37\46\1\47\2\46\1\55\1\56\2\46" + "\1\57\3\46\1\56\5\46\1\56\2\60\2\57\1\46" + "\10\57\1\61\2\57\1\46\5\57\1\46\2\57\1\46" + "\3\57\2\46\1\47\2\46\1\55\1\62\6\46\1\62" + "\5\46\1\62\36\46\1\63\1\64\2\63\1\65\15\63" + "\1\51\36\63\1\46\1\47\2\46\1\66\1\56\2\46" + "\1\67\3\46\1\56\5\46\1\56\4\67\1\46\13\67" + "\1\46\5\67\1\46\2\67\1\46\3\67\2\46\1\47" + "\2\46\1\66\1\56\2\46\1\67\3\46\1\56\5\46" + "\1\56\4\67\1\46\13\67\1\70\5\67\1\46\2\67" + "\1\46\3\67\1\46\1\71\1\47\1\46\1\72\1\73" + "\1\56\3\71\1\74\1\71\1\75\1\56\5\71\1\56" + "\36\71\1\46\1\47\2\46\1\76\15\46\1\51\37\46" + "\1\47\1\77\1\100\1\46\1\56\2\46\1\101\3\46" + "\1\56\5\46\1\56\4\101\1\46\13\101\1\46\5\101" + "\1\46\2\101\1\46\3\101\2\46\1\47\1\77\1\100" + "\1\46\1\56\2\46\1\101\3\46\1\56\5\46\1\56" + "\4\101\1\46\13\101\1\102\5\101\1\46\2\101\1\46" + "\3\101\1\46\1\103\1\47\1\77\1\104\1\103\1\56" + "\3\103\1\105\1\103\1\106\1\56\5\103\1\56\36\103" + "\1\46\1\47\3\46\1\56\6\46\1\56\5\46\1\56" + "\36\46\1\107\1\110\1\111\1\112\4\107\1\113\12\107" + "\4\114\1\107\13\114\1\107\5\114\1\107\2\114\1\107" + "\3\114\1\107\1\46\1\110\1\111\1\112\1\46\1\56" + "\2\46\1\115\3\46\1\56\5\46\1\56\4\115\1\46" + "\13\115\1\46\5\115\1\46\2\115\1\46\3\115\2\46" + "\1\110\1\111\1\112\1\46\1\56\2\46\1\115\3\46" + "\1\56\5\46\1\56\4\115\1\46\13\115\1\116\5\115" + "\1\46\2\115\1\46\3\115\1\46\1\117\1\110\1\111" + "\1\120\1\117\1\56\3\117\1\121\1\117\1\122\1\56" + "\5\117\1\56\36\117\1\46\1\123\1\124\2\46\1\56" + "\6\46\1\56\5\46\1\56\6\46\1\125\1\126\2\46" + "\1\127\11\46\1\126\1\125\11\46\1\47\1\124\2\46" + "\1\56\6\46\1\56\5\46\1\56\4\46\1\130\32\46" + "\1\47\1\124\2\46\1\56\2\46\1\131\3\46\1\56" + "\5\46\1\56\4\131\1\130\13\131\1\46\5\131\1\46" + "\2\131\1\46\3\131\2\46\1\47\1\124\2\46\1\56" + "\6\46\1\56\5\46\1\56\4\46\1\130\7\46\1\132" + "\5\46\1\133\13\46\1\134\1\47\1\124\1\135\1\134" + "\1\56\3\134\1\136\1\134\1\137\1\56\5\134\1\56" + "\4\134\1\140\31\134\1\141\1\47\1\124\1\142\1\141" + "\1\56\3\141\1\143\1\141\1\144\1\56\5\141\1\56" + "\4\141\1\145\31\141\1\146\1\47\1\124\1\147\1\146" + "\1\56\3\146\1\150\1\146\1\151\1\56\5\146\1\56" + "\36\146\1\152\1\153\1\154\56\152\1\155\1\47\1\124" + "\1\156\1\155\1\56\3\155\1\157\1\155\1\160\1\56" + "\5\155\1\56\36\155\1\161\1\162\1\163\56\161\1\164" + "\1\165\57\164\1\40\1\0\10\40\1\0\4\40\1\0" + "\41\40\3\0\1\166\1\167\14\0\1\170\44\0\1\171" + "\2\0\1\172\3\0\1\171\5\0\1\171\4\172\1\0" + "\13\172\1\0\5\172\1\173\2\172\1\0\3\172\6\0" + "\1\171\2\0\1\174\3\0\1\171\5\0\1\171\4\174" + "\1\0\13\174\1\0\5\174\1\0\2\174\1\0\3\174" + "\103\0\1\175\57\0\1\176\47\0\1\177\53\0\1\200" + "\63\0\1\56\6\0\1\56\5\0\1\56\44\0\3\57" + "\4\0\1\57\5\0\4\57\1\0\13\57\1\0\5\57" + "\1\0\2\57\1\0\4\57\6\0\3\57\4\0\1\57" + "\5\0\2\57\1\201\1\57\1\0\13\57\1\0\5\57" + "\1\0\2\57\1\0\4\57\6\0\3\57\4\0\1\57" + "\5\0\2\57\1\202\1\57\1\0\13\57\1\0\5\57" + "\1\0\2\57\1\0\4\57\5\0\1\62\6\0\1\62" + "\5\0\1\62\40\0\1\203\60\0\1\204\64\0\3\67" + "\4\0\1\67\5\0\4\67\1\0\13\67\1\0\5\67" + "\1\0\2\67\1\0\4\67\1\71\2\0\1\205\1\71" + "\1\0\3\71\1\0\1\71\2\0\5\71\1\0\37\71" + "\1\0\1\204\1\205\1\71\1\0\3\71\1\0\1\71" + "\2\0\5\71\1\0\36\71\1\74\1\0\1\206\1\207" + "\1\74\1\206\3\74\1\210\1\74\2\206\5\74\1\206" + "\36\74\1\75\1\0\1\211\1\212\1\75\1\211\3\75" + "\1\211\1\75\1\210\1\211\5\75\1\211\36\75\2\0" + "\1\77\1\213\63\0\3\101\4\0\1\101\5\0\4\101" + "\1\0\13\101\1\0\5\101\1\0\2\101\1\0\4\101" + "\1\103\2\0\1\214\1\103\1\0\3\103\1\0\1\103" + "\2\0\5\103\1\0\37\103\1\0\1\77\1\215\1\103" + "\1\0\3\103\1\0\1\103\2\0\5\103\1\0\36\103" + "\1\105\1\0\1\216\1\217\1\105\1\216\3\105\1\220" + "\1\105\2\216\5\105\1\216\36\105\1\106\1\0\1\221" + "\1\222\1\106\1\221\3\106\1\221\1\106\1\220\1\221" + "\5\106\1\221\36\106\1\107\3\0\17\107\4\0\1\107" + "\13\0\1\107\5\0\1\107\2\0\1\107\3\0\1\107" + "\3\0\1\166\15\0\1\170\41\0\1\223\56\0\1\107" + "\3\0\2\107\3\113\4\107\1\113\5\107\4\114\1\107" + "\13\114\1\107\5\114\1\107\2\114\1\107\3\114\1\113" + "\6\0\3\114\4\0\1\114\5\0\4\114\1\0\13\114" + "\1\0\5\114\1\0\2\114\1\0\4\114\6\0\3\115" + "\4\0\1\115\5\0\4\115\1\0\13\115\1\0\5\115" + "\1\0\2\115\1\0\4\115\1\117\2\0\1\224\1\117" + "\1\0\3\117\1\0\1\117\2\0\5\117\1\0\37\117" + "\1\0\1\223\1\224\1\117\1\0\3\117\1\0\1\117" + "\2\0\5\117\1\0\36\117\1\121\1\0\1\225\1\226" + "\1\121\1\225\3\121\1\227\1\121\2\225\5\121\1\225" + "\36\121\1\122\1\0\1\230\1\231\1\122\1\230\3\122" + "\1\230\1\122\1\227\1\230\5\122\1\230\36\122\3\0" + "\1\166\15\0\1\232\100\0\1\233\52\0\1\234\12\0" + "\1\234\40\0\1\235\32\0\20\236\1\237\40\236\6\0" + "\3\131\4\0\1\131\5\0\4\131\1\0\13\131\1\0" + "\5\131\1\0\2\131\1\0\4\131\44\0\1\240\67\0" + "\1\241\5\0\1\134\2\0\1\242\1\134\1\0\3\134" + "\1\0\1\134\2\0\5\134\1\0\36\134\1\136\1\0" + "\1\243\1\244\1\136\1\243\3\136\1\245\1\136\2\243" + "\5\136\1\243\36\136\1\246\1\0\1\247\1\250\1\251" + "\1\247\3\251\1\247\1\246\1\252\1\253\3\251\1\246" + "\1\251\1\253\4\251\1\246\27\251\2\246\1\140\2\236" + "\1\254\1\140\1\236\3\140\1\236\1\140\2\236\3\140" + "\1\255\1\140\1\236\36\140\1\141\2\0\1\256\1\141" + "\1\0\3\141\1\0\1\141\2\0\5\141\1\0\36\141" + "\1\143\2\257\1\260\1\143\1\257\3\143\1\261\1\143" + "\2\257\5\143\1\257\36\143\1\144\2\262\1\263\1\144" + "\1\262\3\144\1\262\1\144\1\261\1\262\5\144\1\262" + "\36\144\1\145\2\236\1\264\1\145\1\236\3\145\1\236" + "\1\145\2\236\3\145\1\265\1\145\1\236\36\145\1\146" + "\2\0\1\266\1\146\1\0\3\146\1\0\1\146\2\0" + "\5\146\1\0\36\146\1\150\1\0\1\267\1\270\1\150" + "\1\267\3\150\1\271\1\150\2\267\5\150\1\267\36\150" + "\1\272\1\0\1\273\1\274\1\275\1\273\3\275\1\273" + "\1\272\1\276\1\277\3\275\1\272\1\275\1\277\4\275" + "\1\272\27\275\2\272\2\152\1\0\60\152\1\0\16\152" + "\1\300\37\152\1\155\2\0\1\301\1\155\1\0\3\155" + "\1\0\1\155\2\0\5\155\1\0\36\155\1\157\1\0" + "\1\302\1\303\1\157\1\302\3\157\1\304\1\157\2\302" + "\5\157\1\302\36\157\1\305\1\0\1\306\1\307\1\310" + "\1\306\3\310\1\306\1\305\1\311\1\312\3\310\1\305" + "\1\310\1\312\4\310\1\305\27\310\2\305\2\161\1\0" + "\60\161\1\0\16\161\1\313\37\161\7\0\1\314\17\0" + "\1\315\36\0\1\171\2\0\1\40\3\0\1\171\5\0" + "\1\171\4\40\1\0\13\40\1\0\5\40\1\0\2\40" + "\1\0\3\40\1\0\1\316\1\0\3\316\1\317\3\172" + "\1\316\1\0\1\316\1\317\1\172\1\316\1\0\2\316" + "\1\317\4\172\1\316\13\172\1\316\5\172\1\316\2\172" + "\1\320\4\172\15\0\1\321\6\0\1\322\34\0\1\316" + "\1\0\3\316\1\317\3\174\1\316\1\0\1\316\1\317" + "\1\174\1\316\1\0\2\316\1\317\4\174\1\316\13\174" + "\1\316\5\174\1\316\2\174\1\323\4\174\27\0\1\315" + "\33\0\1\324\60\0\1\325\64\0\3\57\4\0\1\57" + "\5\0\3\57\1\326\1\0\13\57\1\0\5\57\1\0" + "\2\57\1\0\4\57\6\0\3\57\4\0\1\57\5\0" + "\4\57\1\0\13\57\1\0\1\57\1\327\3\57\1\0" + "\2\57\1\0\4\57\1\206\1\0\7\206\1\210\47\206" + "\1\211\1\0\11\211\1\210\45\211\1\216\1\0\7\216" + "\1\220\47\216\1\221\1\0\11\221\1\220\45\221\1\225" + "\1\0\7\225\1\227\47\225\1\230\1\0\11\230\1\227" + "\45\230\30\0\1\330\25\0\1\330\35\0\1\331\12\0" + "\1\331\47\0\1\332\62\0\1\333\76\0\1\334\3\0" + "\1\243\1\0\7\243\1\245\47\243\1\246\1\0\1\247" + "\1\335\1\246\1\247\3\246\1\247\1\246\1\245\1\247" + "\5\246\1\247\36\246\1\247\1\0\11\247\1\245\45\247" + "\1\246\1\0\1\247\1\335\1\246\1\247\3\246\1\247" + "\1\246\1\336\1\247\5\246\1\247\36\246\13\0\1\337" + "\45\0\1\247\1\0\11\247\1\336\45\247\11\257\1\261" + "\47\257\13\262\1\261\45\262\1\267\1\0\7\267\1\271" + "\47\267\1\272\1\0\1\273\1\340\1\272\1\273\3\272" + "\1\273\1\272\1\271\1\273\5\272\1\273\36\272\1\273" + "\1\0\11\273\1\271\45\273\1\272\1\0\1\273\1\340" + "\1\272\1\273\3\272\1\273\1\272\1\341\1\273\5\272" + "\1\273\36\272\13\0\1\342\45\0\1\273\1\0\11\273" + "\1\341\45\273\2\152\1\0\24\152\1\343\31\152\1\302" + "\1\0\7\302\1\304\47\302\1\305\1\0\1\306\1\344" + "\1\305\1\306\3\305\1\306\1\305\1\304\1\306\5\305" + "\1\306\36\305\1\306\1\0\11\306\1\304\45\306\1\305" + "\1\0\1\306\1\344\1\305\1\306\3\305\1\306\1\305" + "\1\345\1\306\5\305\1\306\36\305\13\0\1\346\45\0" + "\1\306\1\0\11\306\1\345\45\306\2\161\1\0\24\161" + "\1\347\31\161\7\0\1\350\101\0\1\351\30\0\1\316" + "\1\0\10\316\1\0\4\316\1\0\34\316\1\0\5\316" + "\1\0\3\316\1\317\4\316\1\0\1\316\1\317\2\316" + "\1\0\2\316\1\317\31\316\1\352\4\316\15\0\1\321" + "\36\0\1\353\21\0\1\354\12\0\3\354\2\0\1\354" + "\11\0\2\354\1\0\1\354\2\0\2\354\10\0\3\57" + "\4\0\1\57\5\0\4\57\1\0\11\57\1\355\1\57" + "\1\0\5\57\1\0\2\57\1\0\4\57\33\0\1\356" + "\12\0\1\356\40\0\1\357\57\0\1\360\66\0\1\361" + "\12\0\1\361\40\0\1\362\35\0\2\363\1\0\3\363" + "\2\0\1\252\4\363\1\0\6\363\1\0\27\363\5\0" + "\2\364\1\0\3\364\2\0\1\276\4\364\1\0\6\364" + "\1\0\27\364\2\0\2\152\1\0\25\152\1\365\30\152" + "\3\0\2\366\1\0\3\366\2\0\1\311\4\366\1\0" + "\6\366\1\0\27\366\2\0\2\161\1\0\25\161\1\367" + "\30\161\31\0\1\370\103\0\1\352\21\0\1\354\12\0" + "\3\354\2\0\1\354\11\0\2\354\1\0\1\354\1\0" + "\1\353\2\354\10\0\3\57\4\0\1\57\5\0\4\57" + "\1\0\6\57\1\371\4\57\1\0\5\57\1\0\2\57" + "\1\0\4\57\44\0\1\372\54\0\1\373\55\0\1\374" + "\60\0\1\375\63\0\1\376\20\0\2\152\1\0\26\152" + "\1\377\27\152\2\161\1\0\26\161\1\u0100\27\161\32\0" + "\1\u0101\34\0\3\57\4\0\1\57\5\0\4\57\1\0" + "\3\57\1\u0102\7\57\1\0\2\57\1\u0102\2\57\1\0" + "\2\57\1\0\4\57\45\0\1\u0103\52\0\1\u0104\63\0" + "\1\u0105\43\0\1\u0106\63\0\1\u0107\25\0\1\u0107\2\0" + "\2\152\1\0\27\152\1\u0108\26\152\2\161\1\0\27\161" + "\1\u0109\26\161\33\0\1\u010a\32\0\1\u010b\3\57\3\0" + "\1\u010b\1\57\4\0\1\u010b\4\57\1\0\13\57\1\0" + "\5\57\1\0\2\57\1\0\4\57\35\0\1\u010c\56\0" + "\1\u010d\12\0\1\u010d\45\0\1\u010e\12\0\1\u010e\12\0" + "\2\152\1\0\30\152\1\u010f\25\152\2\161\1\0\30\161" + "\1\u0110\25\161\32\0\1\u0111\33\0\1\u010b\6\0\1\u010b" + "\5\0\1\u010b\36\0\2\152\1\0\27\152\1\u0112\26\152" + "\2\161\1\0\27\161\1\u0113\26\161\27\0\1\u0114\31\0" + "\2\152\1\0\24\152\1\u0115\31\152\2\161\1\0\24\161" + "\1\u0116\31\161"; /** * The transition table of the DFA */ final private static int yytrans[] = yy_unpack(yy_packed); /* error codes */ final private static int YY_UNKNOWN_ERROR = 0; // final private static int YY_ILLEGAL_STATE = 1; final private static int YY_NO_MATCH = 2; final private static int YY_PUSHBACK_2BIG = 3; /* error messages for the codes above */ final private static String YY_ERROR_MSG[] = { "Unkown internal scanner error", //$NON-NLS-1$ "Internal error: unknown state", //$NON-NLS-1$ "Error: could not match input", //$NON-NLS-1$ "Error: pushback value was too large" //$NON-NLS-1$ }; /** * YY_ATTRIBUTE[aState] contains the attributes of state <code>aState</code> */ private final static byte YY_ATTRIBUTE[] = { 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 9, 1, 9, 1, 1, 9, 9, 1, 1, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 9, 1, 1, 1, 1, 1, 1, 9, 9, 1, 9, 9, 1, 0, 1, 0, 1, 0, 0, 0, 9, 1, 1, 9, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 9, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 9, 0, 0, 0, 1, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 9, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 9, 0, 0, 9, 9, 9, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 9, 0, 1, 9, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 9, 9, 1, 1, 0, 1, 9, 9, 9, 1, 1, 0, 1, 1, 9, 1, 1 }; public static final String XML_ELEMENT_DECL_CONTENT = "1"; public static final String XML_ATTLIST_DECL_CONTENT = "2"; public static final String XML_TAG_OPEN = "3"; public static final String XML_END_TAG_OPEN = "4"; public static final String XML_TAG_NAME = "5"; public static final String WHITE_SPACE = "6"; public static final String BLOCK_TEXT = "7"; public static final String XML_COMMENT_TEXT = "8"; public static final String XML_CDATA_OPEN = "9"; public static final String XML_ELEMENT_DECLARATION = "10"; public static final String XML_ATTLIST_DECLARATION = "11"; public static final String XML_DOCTYPE_DECLARATION = "12"; public static final String XML_DOCTYPE_EXTERNAL_ID_PUBLIC = "13"; public static final String XML_DOCTYPE_EXTERNAL_ID_SYSTEM = "14"; public static final String XML_CHAR_REFERENCE = "15"; public static final String XML_COMMENT_OPEN = "16"; public static final String XML_COMMENT_CLOSE = "17"; public static final String XML_CDATA_CLOSE = "19"; public static final String XML_PE_REFERENCE = "20"; public static final String XML_ENTITY_REFERENCE = "21"; public static final String XML_DOCTYPE_INTERNAL_SUBSET = "22"; public static final String XML_EMPTY_TAG_CLOSE = "23"; public static final String XML_PI_CLOSE = "24"; public static final String XML_DECLARATION_OPEN = "25"; public static final String XML_TAG_ATTRIBUTE_VALUE = "26"; public static final String XML_TAG_ATTRIBUTE_EQUALS = "27"; public static final String XML_TAG_ATTRIBUTE_NAME = "28"; public static final String XML_PI_CONTENT = "29"; public static final String XML_CONTENT = "30"; public static final String UNDEFINED = "31"; public static final String XML_CDATA_TEXT = "32"; public static final String XML_TAG_CLOSE = "33"; public static final String XML_DECLARATION_CLOSE = "34"; public static final String XML_DOCTYPE_NAME = "35"; public static final String XML_DOCTYPE_EXTERNAL_ID_PUBREF = "36"; public static final String XML_DOCTYPE_EXTERNAL_ID_SYSREF = "37"; public static final String XML_ELEMENT_DECL_NAME = "38"; public static final String XML_ATTLIST_DECL_NAME = "39"; public static final String XML_PI_OPEN = "40"; /** the input device */ private java.io.Reader yy_reader; /** the current state of the DFA */ private int yy_state; /** the current lexical state */ private int yy_lexical_state = YYINITIAL; /** * this buffer contains the current text to be matched and is the source of * the yytext() string */ private char yy_buffer[] = new char[16384]; /** the textposition at the last accepting state */ private int yy_markedPos; /** the textposition at the last state to be included in yytext */ private int yy_pushbackPos; /** the current text position in the buffer */ private int yy_currentPos; /** startRead marks the beginning of the yytext() string in the buffer */ private int yy_startRead; /** * endRead marks the last character in the buffer, that has been read from * input */ private int yy_endRead; /** number of newlines encountered up to the start of the matched text */ private int yyline; /** the number of characters up to the start of the matched text */ private int yychar; /** * the number of characters from the last newline up to the start of the * matched text */ // private int yycolumn; /** * yy_atBOL == true <=> the scanner is currently at the beginning of a line */ // private boolean yy_atBOL; /** yy_atEOF == true <=> the scanner has returned a value for EOF */ private boolean yy_atEOF; /** denotes if the user-EOF-code has already been executed */ private boolean yy_eof_done; /* user code: */ private int fTokenCount = 0; // required holders for white-space compacting private boolean fShouldLoadBuffered = false; private String fBufferedContext = null; private int fBufferedStart = 1; private int fBufferedLength = 0; private String f_context = null; // state stack for handling embedded regions private IntStack fStateStack = new IntStack(); private String context = null; private int start = 0; private int textLength = 0; private int length = 0; // offset for tracking position specific block tags private int fOffset = 0; // the name of the current tag being opened private String fCurrentTagName = null; // the list of tag name BlockMarkers private List fBlockMarkers = new ArrayList(); // required to not seek text blocks on an end tag private boolean fIsBlockingEnabled = false; private boolean fIsCaseSensitiveBlocking = true; /* * user method */ public final void addBlockMarker(BlockMarker marker) { if (containsTagName(marker.getTagName())) return; fBlockMarkers.add(marker); } /** * user method */ public final void removeBlockMarker(BlockMarker marker) { fBlockMarkers.remove(marker); } /** * user method */ public final void removeBlockMarker(String tagname) { if (fBlockMarkers != null) { Iterator blocks = fBlockMarkers.iterator(); while (blocks.hasNext()) { if (((BlockMarker) blocks.next()).getTagName().equals(tagname)) blocks.remove(); } } } /* user method */ public final boolean isCaseSensitiveBlocking() { return fIsCaseSensitiveBlocking; } /* user method */ public final void setCaseSensitiveBlocking(boolean newValue) { fIsCaseSensitiveBlocking = newValue; } /* user method */ public boolean getBlockMarkerCaseSensitivity() { return getBlockMarkerCaseSensitivity(fCurrentTagName); } /* user method */ public boolean getBlockMarkerCaseSensitivity(String name) { Iterator iterator = fBlockMarkers.iterator(); while (iterator.hasNext()) { BlockMarker marker = (BlockMarker) iterator.next(); boolean casesensitive = marker.isCaseSensitive(); if (casesensitive && marker.getTagName().equals(name)) return casesensitive; else if (!casesensitive && marker.getTagName().equalsIgnoreCase(name)) return casesensitive; } return true; } /* user method */ public String getBlockMarkerContext() { return getBlockMarkerContext(fCurrentTagName); } /* user method */ public String getBlockMarkerContext(String name) { Iterator iterator = fBlockMarkers.iterator(); while (iterator.hasNext()) { BlockMarker marker = (BlockMarker) iterator.next(); if (marker.getTagName().equals(name)) return marker.getContext(); } return BLOCK_TEXT; } /* user method */ public List getBlockMarkers() { return fBlockMarkers; } /* user method */ public final int getOffset() { return fOffset + yychar; } private final boolean isBlockMarker() { return isBlockMarker(fCurrentTagName); } private final boolean isBlockMarker(String tagName) { if (!fIsBlockingEnabled) return false; return containsTagName(tagName); } /** * user method */ public final void beginBlockTagScan(String newTagName) { beginBlockMarkerScan(newTagName, BLOCK_TEXT); } /** * user method * * Special tokenizer setup. Allows tokenization to be initiated at the start * of a text block within a "newTagName" tag. * * Example: Tokenizer toker = new Tokenizer(); * toker.setCaseSensitiveBlocking(false); toker.reset(new * java.io.StringReader("afiuhqwkejhtasihgalkwhtq</scripter></scr></script>asgdasga")); * toker.beginBlockMarkerScan("script", BLOCK_TEXT); toker.getRegions(); * * Returns: BLOCK_TEXT: 0-40 XML_END_TAG_OPEN: 41-42 XML_TAG_NAME: 43-48 * XML_TAG_CLOSE: 49-49 XML_CONTENT: 50-57 * */ public final void beginBlockMarkerScan(String newTagName, String blockcontext) { yybegin(ST_BLOCK_TAG_SCAN); fCurrentTagName = newTagName; } /** * Method doScan. * * Returns a context region for all of the text from the current position * upto the end of input or to right *before* the first occurence of * searchString * * @param searchString - * target string to search for ex.: "-->", "</tagname" * @param requireTailSeparator - * whether the target must be immediately followed by whitespace * or '>' * @param context - * the context of the scanned region if non-zero length * @param exitState - * the state to go to if the region was of non-zero length * @param abortState - * the state to go to if the searchString was found immediately * @return String - the context found: the desired context on a non-zero * length match, the abortContext on immediate success * @throws IOException */ private final String doScan(String searchString, boolean requireTailSeparator, String searchContext, int exitState, int immediateFallbackState) throws IOException { boolean stillSearching = true; // Disable further block (probably) fIsBlockingEnabled = false; int searchStringLength = searchString.length(); int n = 0; char lastCheckChar; int i; boolean same = false; while (stillSearching) { n = 0; // Ensure that enough data from the input exists to compare against // the search String. n = yy_advance(); while (n != YYEOF && yy_currentPos < searchStringLength) n = yy_advance(); // If the input was too short or we've exhausted the input, stop // immediately. if (n == YYEOF) { stillSearching = false; } else { same = true; // Ensure that we've not encountered a complete block (<%%>) // that was *shorter* than the closeTagString and // thus found twice at current-targetLength [since the first // scan would have come out this far anyway]. // Check the characters in the target versus the last // targetLength characters read from the buffer // and see if it matches // safety check for array accesses (yy_currentPos is the *last* // character we can check against) if (yy_currentPos >= searchStringLength && yy_currentPos <= yy_buffer.length) { for (i = 0; i < searchStringLength; i++) { if (same && fIsCaseSensitiveBlocking) same = yy_buffer[i + yy_currentPos - searchStringLength] == searchString .charAt(i); else if (same && !fIsCaseSensitiveBlocking) same = Character.toLowerCase(yy_buffer[i + yy_currentPos - searchStringLength]) == Character .toLowerCase(searchString.charAt(i)); } } // safety check failed; no match is possible right now else { same = false; } if (same && requireTailSeparator && yy_currentPos < yy_buffer.length) { // Additional check for close tags to ensure that // targetString="</script" doesn't match // "</scriptS" lastCheckChar = yy_buffer[yy_currentPos]; // Succeed on "</script>" and "</script " if (lastCheckChar == '>' || Character.isWhitespace(lastCheckChar)) stillSearching = false; } else { stillSearching = !same || (yy_currentPos < yy_startRead + searchStringLength); } } } if (n != YYEOF || same) { // We've stopped short of the end or definitely found a match yy_markedPos = yy_currentPos - searchStringLength; yy_currentPos = yy_markedPos + 1; // If the searchString occurs at the very beginning of what would // have // been a Block, resume scanning normally immediately if (yy_markedPos == yy_startRead) { yybegin(immediateFallbackState); return primGetNextToken(); } } else { // We ran through the rest of the input yy_markedPos = yy_currentPos; yy_currentPos++; } yybegin(exitState); // If the ending occurs at the very beginning of what would have // been a Block, resume scanning normally immediately if (yy_markedPos == yy_startRead) return primGetNextToken(); return searchContext; } /** * user method * * A generic lookahead-like operation */ private final String doBlockScan(String target, String targetContext, int immediateFallbackState) throws IOException { return doScan(target, false, targetContext, immediateFallbackState, immediateFallbackState); } /** * user method does a lookahead for the current tag name */ private final String doBlockTagScan() throws IOException { fIsCaseSensitiveBlocking = getBlockMarkerCaseSensitivity(); return doScan("</" + fCurrentTagName, true, getBlockMarkerContext(fCurrentTagName), YYINITIAL, YYINITIAL); } /** * user method * * Converts the raw context String returned by the primGetNextToken() method * into a full ITextRegion by pulling in values for the current offset * within the scanning text. * * Returns null when EOF is encountered and attaches intermittently * discovered whitespace onto the end of useful regions. * * Note that this algorithm caches the token following the one being * returned so that whitespace can be collapsed. */ public final Token getNextToken() throws IOException { // load the starting non-whitespace token (assume that it is so) if (fShouldLoadBuffered) { context = fBufferedContext; start = fBufferedStart; textLength = length = fBufferedLength; fShouldLoadBuffered = false; } else { context = primGetNextToken(); if (context == XML_TAG_NAME) { if (containsTagName(yy_buffer, yy_startRead, yy_markedPos - yy_startRead)) fCurrentTagName = yytext(); else fCurrentTagName = null; } else if (context == XML_TAG_OPEN) { fIsBlockingEnabled = true; } else if (context == XML_END_TAG_OPEN) { fIsBlockingEnabled = false; } start = yychar; textLength = length = yylength(); if (yy_atEOF) { fTokenCount++; return null; } } // store the next token f_context = primGetNextToken(); if (f_context == XML_TAG_NAME) { if (containsTagName(yy_buffer, yy_startRead, yy_markedPos - yy_startRead)) fCurrentTagName = yytext(); else fCurrentTagName = null; } else if (f_context == XML_TAG_OPEN) { fIsBlockingEnabled = true; } else if (f_context == XML_END_TAG_OPEN) { fIsBlockingEnabled = false; } fBufferedContext = f_context; fBufferedStart = yychar; fBufferedLength = yylength(); fShouldLoadBuffered = true; if (fBufferedContext == WHITE_SPACE) { fShouldLoadBuffered = false; length += fBufferedLength; } if (context == null) { // EOF if (false) { System.out.println(getClass().getName() + " discovered " + fTokenCount + " tokens."); //$NON-NLS-2$//$NON-NLS-1$ } return null; } fTokenCount++; return new Token(context, start, textLength, length, fCurrentTagName); } /* user method */ public XMLTokenizer() { super(); } /* user method */ public XMLTokenizer(char[] charArray) { this(new CharArrayReader(charArray)); } /* user method */ public void reset(char[] charArray) { reset(new CharArrayReader(charArray), 0); } /* user method */ public void reset(char[] charArray, int newOffset) { reset(new CharArrayReader(charArray), newOffset); } /* user method */ public void reset(java.io.InputStream in) { reset(new java.io.InputStreamReader(in), 0); } /* user method */ public void reset(java.io.InputStream in, int newOffset) { reset(new java.io.InputStreamReader(in), newOffset); } /* user method */ public void reset(java.io.Reader in) { reset(in, 0); } /** * user method * * * Reset internal counters and vars to "newly created" values, in the hopes * that resetting a pre-existing tokenizer is faster than creating a new * one. * * This method contains code blocks that were essentially duplicated from * the <em>generated</em> output of this specification before this method * was added. Those code blocks were under the above copyright. */ public void reset(java.io.Reader in, int newOffset) { if (false) { System.out.println("resetting tokenizer");//$NON-NLS-1$ } fOffset = newOffset; /* the input device */ yy_reader = in; /* the current state of the DFA */ yy_state = 0; /* the current lexical state */ yy_lexical_state = YYINITIAL; /* * this buffer contains the current text to be matched and is the source * of the yytext() string */ java.util.Arrays.fill(yy_buffer, (char) 0); /* the textposition at the last accepting state */ yy_markedPos = 0; /* the textposition at the last state to be included in yytext */ yy_pushbackPos = 0; /* the current text position in the buffer */ yy_currentPos = 0; /* startRead marks the beginning of the yytext() string in the buffer */ yy_startRead = 0; /** * endRead marks the last character in the buffer, that has been read * from input */ yy_endRead = 0; /* number of newlines encountered up to the start of the matched text */ yyline = 0; /* the number of characters up to the start of the matched text */ yychar = 0; /* yy_atEOF == true <=> the scanner has returned a value for EOF */ yy_atEOF = false; /* denotes if the user-EOF-code has already been executed */ yy_eof_done = false; /* user vars: */ fTokenCount = 0; fShouldLoadBuffered = false; fBufferedContext = null; fBufferedStart = 1; fBufferedLength = 0; fStateStack = new IntStack(); context = null; start = 0; textLength = 0; length = 0; } /** * user method * */ public XMLTokenizer newInstance() { XMLTokenizer newInstance = new XMLTokenizer(); // global tagmarkers can be shared; they have no state and // are never destroyed (e.g. 'release') for (int i = 0; i < fBlockMarkers.size(); i++) { BlockMarker blockMarker = (BlockMarker) fBlockMarkers.get(i); if (blockMarker.isGlobal()) newInstance.addBlockMarker(blockMarker); } return null; } /* user method */ private final String scanXMLCommentText() throws IOException { // Scan for '-->' and return the text up to that point as // XML_COMMENT_TEXT unless the string occurs IMMEDIATELY, in which // case change to the ST_XML_COMMENT_END state and return the next // context as usual. return doScan("-->", false, XML_COMMENT_TEXT, ST_XML_COMMENT_END, ST_XML_COMMENT_END); } /** * Creates a new scanner There is also a java.io.InputStream version of this * constructor. * * @param in * the java.io.Reader to read input from. */ public XMLTokenizer(java.io.Reader in) { this.yy_reader = in; } /** * Creates a new scanner. There is also java.io.Reader version of this * constructor. * * @param in * the java.io.Inputstream to read input from. */ public XMLTokenizer(java.io.InputStream in) { this(new java.io.InputStreamReader(in)); } /** * Unpacks the compressed DFA transition table. * * @param packed * the packed transition table * @return the unpacked transition table */ private static int[] yy_unpack(String packed) { int[] trans = new int[9212]; int i = 0; /* index in packed string */ int j = 0; /* index in unpacked array */ while (i < 3220) { int count = packed.charAt(i++); int value = packed.charAt(i++); value--; do trans[j++] = value; while (--count > 0); } return trans; } /** * Unpacks the compressed character translation table. * * @param packed * the packed character translation table * @return the unpacked character translation table */ private static char[] yy_unpack_cmap(String packed) { char[] map = new char[0x10000]; int i = 0; /* index in packed string */ int j = 0; /* index in unpacked array */ while (i < 1372) { int count = packed.charAt(i++); char value = packed.charAt(i++); do map[j++] = value; while (--count > 0); } return map; } /** * Gets the next input character. * * @return the next character of the input stream, EOF if the end of the * stream is reached. * @exception IOException * if any I/O-Error occurs */ private int yy_advance() throws java.io.IOException { /* standard case */ if (yy_currentPos < yy_endRead) return yy_buffer[yy_currentPos++]; /* if the eof is reached, we don't need to work hard */ if (yy_atEOF) return YYEOF; /* otherwise: need to refill the buffer */ /* first: make room (if you can) */ if (yy_startRead > 0) { System.arraycopy(yy_buffer, yy_startRead, yy_buffer, 0, yy_endRead - yy_startRead); /* translate stored positions */ yy_endRead -= yy_startRead; yy_currentPos -= yy_startRead; yy_markedPos -= yy_startRead; yy_pushbackPos -= yy_startRead; yy_startRead = 0; } /* is the buffer big enough? */ if (yy_currentPos >= yy_buffer.length) { /* if not: blow it up */ char newBuffer[] = new char[yy_currentPos * 2]; System.arraycopy(yy_buffer, 0, newBuffer, 0, yy_buffer.length); yy_buffer = newBuffer; } /* finally: fill the buffer with new input */ int numRead = yy_reader.read(yy_buffer, yy_endRead, yy_buffer.length - yy_endRead); if (numRead == -1) return YYEOF; yy_endRead += numRead; return yy_buffer[yy_currentPos++]; } /** * Closes the input stream. */ final public void yyclose() throws java.io.IOException { yy_atEOF = true; /* indicate end of file */ yy_endRead = yy_startRead; /* invalidate buffer */ yy_reader.close(); } /** * Returns the current lexical state. */ final public int yystate() { return yy_lexical_state; } /** * Enters a new lexical state * * @param newState * the new lexical state */ final public void yybegin(int newState) { yy_lexical_state = newState; } /** * Returns the text matched by the current regular expression. */ final public String yytext() { return new String(yy_buffer, yy_startRead, yy_markedPos - yy_startRead); } /** * Returns the length of the matched text region. */ final public int yylength() { return yy_markedPos - yy_startRead; } /** * Reports an error that occured while scanning - from the SED JFlex * skeleton * * @param errorCode * the code of the errormessage to display */ private void yy_ScanError(int errorCode) { // DO NOT EXIT the VM on an error // System.exit(1); } /** * Pushes the specified amount of characters back into the input stream. * * They will be read again by then next call of the scanning method * * @param number * the number of characters to be read again. This number must * not be greater than yylength()! */ void yypushback(int number) { if (number > yylength()) yy_ScanError(YY_PUSHBACK_2BIG); yy_markedPos -= number; } /** * user method - skeleton.sed */ protected final boolean containsTagName(char[] markerTagName, int offset, int tagnameLength) { for (int j = 0; j < fBlockMarkers.size(); j++) { BlockMarker marker = (BlockMarker) fBlockMarkers.get(j); if (marker.getTagName().length() == tagnameLength) { boolean matchesSoFar = true; for (int i = 0; i < tagnameLength && matchesSoFar; i++) { if (marker.isCaseSensitive()) { if (marker.getTagName().charAt(i) != markerTagName[i + offset]) matchesSoFar = false; } else { if (Character .toLowerCase(marker.getTagName().charAt(i)) != Character .toLowerCase(markerTagName[i + offset])) matchesSoFar = false; } } if (matchesSoFar) return true; } } return false; } /** * user method - skeleton.sed * * Return ALL of the regions scannable within the remaining text Note: for * verification use */ public final List getRegions() { List tokens = new ArrayList(); Token region = null; try { region = getNextToken(); while (region != null) { if (region != null) { tokens.add(region); } region = getNextToken(); } } catch (StackOverflowError e) { throw e; } catch (Exception e) { // Since this is convenience method and NOT the recommended // way of getting tokens, many errors are simply hidden } return tokens; } /** * user method - skeleton.sed */ private final void dump(String s) { if (false) { System.out.println(s + " (" + yychar + "-" + //$NON-NLS-2$//$NON-NLS-1$ (yylength() + yychar) + "):\'" + //$NON-NLS-1$ yytext() + "\'");//$NON-NLS-1$ } } /* user method - skeleton.sed */ public final boolean isEOF() { return yy_atEOF; } /* user method - skeleton.sed */ protected final boolean containsTagName(String markerTagName) { Iterator blocks = fBlockMarkers.iterator(); while (blocks.hasNext()) { BlockMarker marker = (BlockMarker) blocks.next(); if (marker.isCaseSensitive()) { if (marker.getTagName().equals(markerTagName)) return true; } else { if (marker.getTagName().equalsIgnoreCase(markerTagName)) return true; } } return false; } /** * Contains user EOF-code, which will be executed exactly once, when the end * of file is reached */ private void yy_do_eof() { if (!yy_eof_done) { yy_eof_done = true; // do nothing, this is the downstream parser's job } } /** * Resumes scanning until the next regular expression is matched, the end of * input is encountered or an I/O-Error occurs. * * @return the next token * @exception IOException * if any I/O-Error occurs */ public String primGetNextToken() throws java.io.IOException { int yy_input; int yy_action; while (true) { yychar += yylength(); boolean yy_counted = false; for (yy_currentPos = yy_startRead; yy_currentPos < yy_markedPos; yy_currentPos++) { switch (yy_buffer[yy_currentPos]) { case '\r': yyline++; yy_counted = true; break; case '\n': if (yy_counted) yy_counted = false; else { yyline++; } break; default: yy_counted = false; } } if (yy_counted) { if (yy_advance() == '\n') yyline--; if (!yy_atEOF) yy_currentPos--; } yy_action = -1; yy_currentPos = yy_startRead = yy_markedPos; yy_state = yy_lexical_state; yy_forAction: { while (true) { yy_input = yy_advance(); if (yy_input == YYEOF) break yy_forAction; int yy_next = yytrans[yy_rowMap[yy_state] + yycmap[yy_input]]; if (yy_next == -1) break yy_forAction; yy_state = yy_next; int yy_attributes = YY_ATTRIBUTE[yy_state]; if ((yy_attributes & 1) > 0) { yy_action = yy_state; yy_markedPos = yy_currentPos; if ((yy_attributes & 8) > 0) break yy_forAction; } } } switch (yy_action) { case 275: case 276: case 277: { if (false) dump("\nCDATA start");//$NON-NLS-1$ fStateStack.push(yystate()); yybegin(ST_CDATA_TEXT); return XML_CDATA_OPEN; } case 279: break; case 269: { if (false) dump("element");//$NON-NLS-1$ yybegin(ST_XML_ELEMENT_DECLARATION); return XML_ELEMENT_DECLARATION; } case 280: break; case 268: { if (false) dump("attlist");//$NON-NLS-1$ yybegin(ST_XML_ATTLIST_DECLARATION); return XML_ATTLIST_DECLARATION; } case 281: break; case 267: { if (false) dump("doctype");//$NON-NLS-1$ yybegin(ST_XML_DOCTYPE_DECLARATION); return XML_DOCTYPE_DECLARATION; } case 282: break; case 262: { if (false) dump("doctype external id");//$NON-NLS-1$ yybegin(ST_XML_DOCTYPE_ID_PUBLIC); return XML_DOCTYPE_EXTERNAL_ID_PUBLIC; } case 283: break; case 261: { if (false) dump("doctype external id");//$NON-NLS-1$ yybegin(ST_XML_DOCTYPE_ID_SYSTEM); return XML_DOCTYPE_EXTERNAL_ID_SYSTEM; } case 284: break; case 257: case 266: { if (false) dump("DHTML processing instruction target");//$NON-NLS-1$ yybegin(ST_DHTML_ATTRIBUTE_NAME); return XML_TAG_NAME; } case 285: break; case 234: { if (false) dump("\nCharRef");//$NON-NLS-1$ return XML_CHAR_REFERENCE; } case 286: break; case 231: { if (false) dump("\ncomment start");//$NON-NLS-1$ yybegin(ST_XML_COMMENT); return XML_COMMENT_OPEN; } case 287: break; case 213: { if (false) dump("XML processing instruction target");//$NON-NLS-1$ yybegin(ST_XML_PI_ATTRIBUTE_NAME); return XML_TAG_NAME; } case 288: break; case 212: { if (false) dump("comment end");//$NON-NLS-1$ yybegin(YYINITIAL); return XML_COMMENT_CLOSE; } case 289: break; case 211: { if (false) dump("CDATA end");//$NON-NLS-1$ yybegin(fStateStack.pop()); return XML_CDATA_CLOSE; } case 290: break; case 210: { if (false) dump("\nPEReference");//$NON-NLS-1$ return XML_PE_REFERENCE; } case 291: break; case 207: { if (false) dump("\nEntityRef");//$NON-NLS-1$ return XML_ENTITY_REFERENCE; } case 292: break; case 158: case 172: case 180: { return XML_DOCTYPE_INTERNAL_SUBSET; } case 293: break; case 146: { yybegin(YYINITIAL); if (false) dump("empty tag close");//$NON-NLS-1$ return XML_EMPTY_TAG_CLOSE; } case 294: break; case 131: { if (false) dump("XML processing instruction end");//$NON-NLS-1$ yybegin(YYINITIAL); return XML_PI_CLOSE; } case 295: break; case 130: { // ended with nothing inside yybegin(YYINITIAL); return XML_PI_CLOSE; } case 296: break; case 127: { if (false) dump("processing instruction end");//$NON-NLS-1$ yybegin(YYINITIAL); return XML_PI_CLOSE; } case 297: break; case 119: { fStateStack.push(yystate()); if (false) dump("\ndeclaration start");//$NON-NLS-1$ yybegin(ST_XML_DECLARATION); return XML_DECLARATION_OPEN; } case 298: break; case 118: { if (false) dump("\nprocessing instruction start");//$NON-NLS-1$ yybegin(ST_PI); return XML_PI_OPEN; } case 299: break; case 62: { if (false) dump("DHTML processing instruction end");//$NON-NLS-1$ yybegin(YYINITIAL); return XML_PI_CLOSE; } case 300: break; case 56: case 58: case 59: case 60: case 135: { if (false) dump("XML processing instruction attribute value");//$NON-NLS-1$ yybegin(ST_XML_PI_ATTRIBUTE_NAME); return XML_TAG_ATTRIBUTE_VALUE; } case 301: break; case 55: { if (false) dump("XML processing instruction '='");//$NON-NLS-1$ yybegin(ST_XML_PI_ATTRIBUTE_VALUE); return XML_TAG_ATTRIBUTE_EQUALS; } case 302: break; case 54: { if (false) dump("XML processing instruction attribute name");//$NON-NLS-1$ yybegin(ST_XML_PI_EQUALS); return XML_TAG_ATTRIBUTE_NAME; } case 303: break; case 50: case 51: case 52: { // block scan until close is found return doScan("?>", false, XML_PI_CONTENT, ST_XML_PI_TAG_CLOSE, ST_XML_PI_TAG_CLOSE); } case 304: break; case 49: { yybegin(ST_PI_CONTENT); return WHITE_SPACE; } case 305: break; case 46: case 47: case 48: case 128: case 129: case 214: case 236: case 248: { if (false) dump("processing instruction target");//$NON-NLS-1$ yybegin(ST_PI_WS); return XML_TAG_NAME; } case 306: break; case 41: case 42: { if (false) dump("comment content");//$NON-NLS-1$ return scanXMLCommentText(); } case 307: break; case 40: { if (false) dump("LINE FEED");//$NON-NLS-1$ return WHITE_SPACE; } case 308: break; case 0: case 31: case 121: case 123: case 205: case 206: case 233: { if (false) dump("\nXML content");//$NON-NLS-1$ return XML_CONTENT; } case 309: break; case 5: case 8: case 9: case 10: case 12: case 13: case 14: case 15: case 17: case 18: case 19: case 20: case 21: case 22: case 23: case 24: case 25: case 26: case 28: case 45: { if (false) dump("white space");//$NON-NLS-1$ return WHITE_SPACE; } case 310: break; case 16: case 70: { if (false) dump("inappropriate tag name");//$NON-NLS-1$ yybegin(YYINITIAL); return XML_CONTENT; } case 311: break; case 27: case 105: case 106: case 191: case 226: case 244: case 254: case 263: case 270: case 273: { if (false) dump("elementdecl contentspec");//$NON-NLS-1$ return XML_ELEMENT_DECL_CONTENT; } case 312: break; case 29: case 112: case 113: case 202: case 230: case 246: case 255: case 264: case 271: case 274: { if (false) dump("attlist contentspec");//$NON-NLS-1$ return XML_ATTLIST_DECL_CONTENT; } case 313: break; case 32: case 71: case 82: { if (false) dump("\nstart tag open");//$NON-NLS-1$ yybegin(ST_XML_TAG_NAME); return XML_TAG_OPEN; } case 314: break; case 33: case 34: case 37: case 38: case 39: case 43: case 44: case 53: case 57: case 61: case 63: case 67: case 73: case 79: case 84: case 85: case 86: case 87: case 89: case 90: case 92: case 97: case 102: case 109: { if (false) System.out .println("!!!unexpected!!!: \"" + yytext() + "\":" + //$NON-NLS-2$//$NON-NLS-1$ yychar + "-" + (yychar + yylength()));//$NON-NLS-1$ return UNDEFINED; } case 315: break; case 35: case 36: { if (false) dump("CDATA text");//$NON-NLS-1$ String blockContext = doBlockScan( "]]>", XML_CDATA_TEXT, ST_CDATA_END);//$NON-NLS-1$ if (blockContext == XML_CDATA_TEXT) yybegin(ST_CDATA_END); return blockContext; } case 316: break; case 64: { if (false) dump("DHTML processing instruction attribute name");//$NON-NLS-1$ yybegin(ST_DHTML_EQUALS); return XML_TAG_ATTRIBUTE_NAME; } case 317: break; case 65: { if (false) dump("DHTML processing instruction '='");//$NON-NLS-1$ yybegin(ST_DHTML_ATTRIBUTE_VALUE); return XML_TAG_ATTRIBUTE_EQUALS; } case 318: break; case 66: case 68: case 69: case 143: { if (false) dump("DHTML processing instruction attribute value");//$NON-NLS-1$ yybegin(ST_DHTML_ATTRIBUTE_NAME); return XML_TAG_ATTRIBUTE_VALUE; } case 319: break; case 72: { if (false) dump("tag close");//$NON-NLS-1$ if (isBlockMarker()) { yybegin(ST_BLOCK_TAG_SCAN); } else yybegin(YYINITIAL); return XML_TAG_CLOSE; } case 320: break; case 74: case 75: { if (false) dump("tag name");//$NON-NLS-1$ yybegin(ST_XML_ATTRIBUTE_NAME); return XML_TAG_NAME; } case 321: break; case 76: { if (false) dump("attr name");//$NON-NLS-1$ yybegin(ST_XML_EQUALS); return XML_TAG_ATTRIBUTE_NAME; } case 322: break; case 77: { if (false) dump("equals");//$NON-NLS-1$ yybegin(ST_XML_ATTRIBUTE_VALUE); return XML_TAG_ATTRIBUTE_EQUALS; } case 323: break; case 78: case 80: case 81: case 150: { if (false) dump("attr value");//$NON-NLS-1$ yybegin(ST_XML_ATTRIBUTE_NAME); return XML_TAG_ATTRIBUTE_VALUE; } case 324: break; case 83: { if (false) dump("declaration end");//$NON-NLS-1$ if (false) { if (fStateStack.peek() != YYINITIAL) System.out.println("end embedded region");//$NON-NLS-1$ } yybegin(fStateStack.pop()); return XML_DECLARATION_CLOSE; } case 325: break; case 88: { if (false) dump("doctype type");//$NON-NLS-1$ yybegin(ST_XML_DOCTYPE_EXTERNAL_ID); return XML_DOCTYPE_NAME; } case 326: break; case 91: case 93: case 94: case 95: case 164: case 165: case 168: case 169: case 221: { if (false) dump("doctype public reference");//$NON-NLS-1$ yybegin(ST_XML_DOCTYPE_ID_SYSTEM); return XML_DOCTYPE_EXTERNAL_ID_PUBREF; } case 327: break; case 96: case 98: case 99: case 100: case 176: { if (false) dump("doctype system reference");//$NON-NLS-1$ yybegin(ST_XML_DECLARATION_CLOSE); return XML_DOCTYPE_EXTERNAL_ID_SYSREF; } case 328: break; case 101: case 103: case 104: case 184: case 185: case 188: case 189: case 224: { if (false) dump("elementdecl name");//$NON-NLS-1$ yybegin(ST_XML_ELEMENT_DECLARATION_CONTENT); return XML_ELEMENT_DECL_NAME; } case 329: break; case 107: { if (false) dump("elementdecl close");//$NON-NLS-1$ if (false) { if (fStateStack.peek() != YYINITIAL) System.out.println("end embedded region");//$NON-NLS-1$ } yybegin(fStateStack.pop()); return XML_DECLARATION_CLOSE; } case 330: break; case 108: case 110: case 111: case 195: case 196: case 199: case 200: case 228: { if (false) dump("attlist name");//$NON-NLS-1$ yybegin(ST_XML_ATTLIST_DECLARATION_CONTENT); return XML_ATTLIST_DECL_NAME; } case 331: break; case 114: { if (false) dump("attlist close");//$NON-NLS-1$ if (false) { if (fStateStack.peek() != YYINITIAL) System.out.println("end embedded region");//$NON-NLS-1$ } yybegin(fStateStack.pop()); return XML_DECLARATION_CLOSE; } case 332: break; case 117: { if (false) dump("\nend tag open");//$NON-NLS-1$ yybegin(ST_XML_TAG_NAME); return XML_END_TAG_OPEN; } case 333: break; case 115: case 116: { return doBlockTagScan(); } case 334: break; default: if (yy_input == YYEOF && yy_startRead == yy_currentPos) { yy_atEOF = true; yy_do_eof(); return null; } else { yy_ScanError(YY_NO_MATCH); } } } } }