/* The following code was generated by JFlex 1.4.3 on 6/2/14 11:40 PM */
package edu.stanford.nlp.international.arabic.process;
import java.io.Reader;
import java.util.Map;
import java.util.Properties;
import java.util.regex.Pattern;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.process.LexedTokenFactory;
import edu.stanford.nlp.util.Generics;
import edu.stanford.nlp.util.PropertiesUtils;
/**
* Tokenizer for UTF-8 Arabic. Supports raw text and both sections
* (vocalized and unvocalized) of the ATB.
*
* @author Spence Green
*/
class ArabicLexer {
/** This character denotes the end of file */
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
private static final int ZZ_BUFFERSIZE = 16384;
/** lexical states */
public static final int YYINITIAL = 0;
/**
* ZZ_LEXSTATE[l] is the state in the DFA for the lexical state l
* ZZ_LEXSTATE[l+1] is the state in the DFA for the lexical state l
* at the beginning of a line
* l is of the form l = 2*k, k a non negative integer
*/
private static final int ZZ_LEXSTATE[] = {
0, 0
};
/**
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
"\11\0\1\34\1\2\1\3\1\4\1\1\22\0\1\34\1\36\1\35"+
"\1\7\1\42\2\7\1\42\2\35\1\7\1\22\1\37\1\14\1\6"+
"\1\33\12\43\1\32\1\42\1\35\1\7\1\35\1\36\1\56\2\41"+
"\1\44\1\53\1\50\1\41\1\52\1\26\4\41\1\46\1\47\1\45"+
"\1\30\1\41\1\51\1\31\1\27\1\54\1\41\1\40\3\41\1\15"+
"\1\11\1\21\1\7\1\13\1\42\2\41\1\44\1\53\1\50\1\41"+
"\1\52\1\26\4\41\1\46\1\47\1\45\1\30\1\41\1\51\1\31"+
"\1\27\1\54\1\41\1\40\3\41\1\36\1\35\1\36\1\7\6\0"+
"\1\3\32\0\1\55\11\7\1\24\2\7\1\24\7\7\1\24\4\7"+
"\1\24\5\7\27\23\1\0\37\23\1\0\u013f\23\31\23\162\23\4\23"+
"\14\23\16\23\5\23\7\23\1\23\1\23\1\23\201\23\5\23\1\23"+
"\2\23\2\23\4\23\6\0\2\23\1\23\1\0\3\23\1\0\1\23"+
"\1\0\24\23\1\0\54\23\1\23\46\23\1\23\5\23\4\23\202\23"+
"\1\0\5\23\2\0\105\23\1\23\46\23\12\23\20\23\26\23\2\23"+
"\11\0\46\23\2\0\1\23\6\23\1\0\47\23\11\0\55\23\1\0"+
"\1\23\1\0\2\23\1\0\2\23\1\0\1\23\10\0\33\23\5\0"+
"\3\23\15\0\14\7\2\11\15\12\5\7\1\23\43\25\1\20\1\25"+
"\1\16\4\25\4\12\1\17\17\12\1\0\12\10\1\7\2\11\1\7"+
"\2\25\1\12\143\25\1\7\1\25\17\12\2\25\7\12\2\25\12\10"+
"\3\25\2\12\1\25\17\0\1\23\1\23\1\23\36\23\35\23\63\23"+
"\46\23\13\23\1\23\16\0\12\10\41\23\11\23\2\23\4\0\1\23"+
"\5\0\26\23\4\0\1\23\11\0\1\23\3\0\1\23\27\0\31\23"+
"\247\0\4\23\66\23\2\0\1\23\1\23\21\23\1\0\1\23\5\23"+
"\2\0\12\23\2\23\2\0\12\10\1\0\7\23\1\0\7\23\1\0"+
"\3\23\1\0\10\23\2\0\2\23\2\0\26\23\1\0\7\23\1\0"+
"\1\23\3\0\4\23\2\0\1\23\1\23\7\23\2\0\2\23\2\0"+
"\3\23\1\23\10\0\1\23\4\0\2\23\1\0\3\23\2\23\2\0"+
"\12\10\2\23\17\0\3\23\1\0\6\23\4\0\2\23\2\0\26\23"+
"\1\0\7\23\1\0\2\23\1\0\2\23\1\0\2\23\2\0\1\23"+
"\1\0\22\23\11\0\4\23\1\0\1\23\7\0\12\10\2\0\3\23"+
"\14\0\3\23\1\0\11\23\1\0\3\23\1\0\26\23\1\0\7\23"+
"\1\0\2\23\1\0\5\23\2\0\1\23\1\23\22\23\1\23\17\0"+
"\2\23\4\0\12\10\25\0\10\23\2\0\2\23\2\0\26\23\1\0"+
"\7\23\1\0\2\23\1\0\5\23\3\0\1\23\36\0\2\23\1\0"+
"\3\23\4\0\12\10\1\0\1\23\20\0\1\23\1\23\1\0\6\23"+
"\3\0\3\23\1\0\4\23\3\0\2\23\1\0\1\23\1\0\2\23"+
"\3\0\2\23\3\0\3\23\3\0\14\23\4\0\5\23\3\0\3\23"+
"\1\0\4\23\2\0\1\23\25\0\12\10\21\0\3\23\1\0\10\23"+
"\1\0\3\23\1\0\27\23\1\0\12\23\1\0\5\23\3\0\1\23"+
"\31\23\1\0\2\23\6\0\2\23\4\0\12\10\25\0\10\23\1\0"+
"\3\23\1\0\27\23\1\0\12\23\1\0\5\23\3\0\1\23\40\0"+
"\1\23\1\0\2\23\4\0\12\10\1\0\2\23\22\0\10\23\1\0"+
"\3\23\1\0\51\23\2\0\1\23\7\23\1\0\3\23\5\0\1\23"+
"\21\0\2\23\4\0\12\10\12\0\6\23\5\0\22\23\3\0\30\23"+
"\1\0\11\23\1\0\1\23\2\0\7\23\72\0\57\23\1\23\1\23"+
"\2\23\7\23\5\0\7\23\10\23\1\0\12\10\47\0\2\23\1\0"+
"\1\23\2\0\2\23\1\0\1\23\2\0\1\23\6\0\4\23\1\0"+
"\7\23\1\0\3\23\1\0\1\23\1\0\1\23\2\0\2\23\1\0"+
"\4\23\1\23\2\23\11\23\1\23\2\0\5\23\1\0\1\23\1\0"+
"\6\23\2\0\12\10\2\0\2\23\42\0\1\23\37\0\12\10\26\0"+
"\10\23\1\0\44\23\33\0\5\23\163\0\53\23\24\0\1\23\12\10"+
"\6\0\6\23\4\0\4\23\3\0\1\23\3\0\2\23\7\0\3\23"+
"\4\0\15\23\14\0\1\23\1\0\12\10\6\0\46\23\12\0\53\23"+
"\1\0\1\23\3\0\u0149\23\1\0\4\23\2\0\7\23\1\0\1\23"+
"\1\0\4\23\2\0\51\23\1\0\4\23\2\0\41\23\1\0\4\23"+
"\2\0\7\23\1\0\1\23\1\0\4\23\2\0\17\23\1\0\71\23"+
"\1\0\4\23\2\0\103\23\45\0\20\23\20\0\125\23\14\0\u026c\23"+
"\2\0\21\23\1\0\32\23\5\0\113\23\25\0\15\23\1\0\4\23"+
"\16\0\22\23\16\0\22\23\16\0\15\23\1\0\3\23\17\0\64\23"+
"\43\0\1\23\4\0\1\23\3\0\12\10\46\0\12\10\6\0\130\23"+
"\10\0\51\23\1\0\1\23\5\0\106\23\12\0\35\23\51\0\12\10"+
"\36\23\2\0\5\23\13\0\54\23\25\0\7\23\10\0\12\10\46\0"+
"\27\23\11\0\65\23\53\0\12\10\6\0\12\10\15\0\1\23\135\0"+
"\57\23\21\0\7\23\4\0\12\10\51\0\36\23\15\0\2\23\12\10"+
"\6\0\46\23\32\0\44\23\34\0\12\10\3\0\3\23\12\10\44\23"+
"\153\0\4\23\1\0\4\23\16\0\300\23\100\0\u0116\23\2\0\6\23"+
"\2\0\46\23\2\0\6\23\2\0\10\23\1\0\1\23\1\0\1\23"+
"\1\0\1\23\1\0\37\23\2\0\65\23\1\0\7\23\1\0\1\23"+
"\3\0\3\23\1\0\7\23\3\0\4\23\2\0\6\23\4\0\15\23"+
"\5\0\3\23\1\0\7\23\3\0\13\5\5\0\30\7\1\3\1\3"+
"\5\0\1\5\57\7\22\0\1\23\15\0\1\23\20\0\15\23\3\0"+
"\26\7\114\0\1\23\4\0\1\23\2\0\12\23\1\0\1\23\3\0"+
"\5\23\6\0\1\23\1\0\1\23\1\0\1\23\1\0\4\23\1\0"+
"\13\23\2\0\4\23\5\0\5\23\4\0\1\23\64\0\2\23\u0a7b\0"+
"\57\23\1\0\57\23\1\0\205\23\6\0\4\23\21\0\46\23\12\0"+
"\66\23\11\0\1\23\20\0\27\23\11\0\7\23\1\0\7\23\1\0"+
"\7\23\1\0\7\23\1\0\7\23\1\0\7\23\1\0\7\23\1\0"+
"\7\23\117\0\1\7\1\23\u01d0\0\1\5\4\0\2\23\52\0\5\23"+
"\5\0\2\23\4\0\126\23\6\0\3\23\1\0\132\23\1\0\4\23"+
"\5\0\51\23\3\0\136\23\21\0\33\23\65\0\20\23\u0200\0\u19b6\23"+
"\112\0\u51cc\23\64\0\u048d\23\103\0\56\23\2\0\u010d\23\3\0\20\23"+
"\12\10\2\23\24\0\57\23\20\0\31\23\10\0\106\23\61\0\11\23"+
"\2\0\147\23\2\0\4\23\1\0\2\23\16\0\12\23\120\0\10\23"+
"\1\0\3\23\1\0\4\23\1\0\27\23\35\0\64\23\16\0\62\23"+
"\34\0\12\10\30\0\6\23\3\0\1\23\4\0\12\10\34\23\12\0"+
"\27\23\31\0\35\23\7\0\57\23\34\0\1\23\12\10\46\0\51\23"+
"\27\0\3\23\1\0\10\23\4\0\12\10\6\0\27\23\3\0\1\23"+
"\5\0\60\23\1\0\1\23\3\0\2\23\2\0\5\23\2\0\1\23"+
"\1\0\1\23\30\0\3\23\43\0\6\23\2\0\6\23\2\0\6\23"+
"\11\0\7\23\1\0\7\23\221\0\43\23\15\0\12\10\6\0\u2ba4\23"+
"\14\0\27\23\4\0\61\23\u2104\0\u012e\23\2\0\76\23\2\0\152\23"+
"\46\0\7\23\14\0\5\23\5\0\1\23\1\0\12\23\1\0\15\23"+
"\1\0\5\23\1\0\1\23\1\0\2\23\1\0\2\23\1\0\154\23"+
"\41\0\u016b\23\22\0\100\23\2\0\66\23\50\0\14\23\164\0\5\23"+
"\1\0\207\23\23\0\12\10\7\0\32\23\6\0\32\23\13\0\131\23"+
"\3\0\6\23\2\0\6\23\2\0\6\23\2\0\3\23\43\0";
/**
* Translates characters to character classes
*/
private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
/**
* Translates DFA states to action switch labels.
*/
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
"\1\0\1\1\3\2\1\3\3\4\1\5\3\4\2\5"+
"\1\4\2\5\1\4\1\3\1\5\1\4\2\0\5\4"+
"\1\5\2\4\1\5\2\4\4\0\1\5\1\0\1\5"+
"\5\0\1\6\6\0\1\5\2\0\1\5\12\0\1\5"+
"\3\0\1\4\2\0\1\4\1\5\20\0\1\7\1\0"+
"\6\4\10\0\1\7\1\10\3\4\1\0\1\4\2\0"+
"\5\4\1\0\6\4";
private static int [] zzUnpackAction() {
int [] result = new int[130];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAction(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/**
* Translates a state to a row index in the transition table
*/
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
"\0\0\0\57\0\136\0\215\0\57\0\274\0\353\0\u011a"+
"\0\u0149\0\u0178\0\u01a7\0\u01d6\0\u0205\0\u0234\0\u0263\0\u0292"+
"\0\u02c1\0\u02f0\0\u031f\0\u034e\0\u037d\0\u03ac\0\57\0\u03db"+
"\0\u040a\0\u0439\0\u0468\0\u0497\0\u04c6\0\u04f5\0\u0524\0\u0553"+
"\0\u0582\0\u05b1\0\u05e0\0\u060f\0\u063e\0\u066d\0\u069c\0\u06cb"+
"\0\u06fa\0\u0729\0\u0758\0\u0787\0\u07b6\0\u07e5\0\u0814\0\u040a"+
"\0\u0843\0\u0872\0\u08a1\0\u08d0\0\u08ff\0\u092e\0\u095d\0\u098c"+
"\0\u09bb\0\u09ea\0\u0a19\0\u0a48\0\u0a77\0\u0aa6\0\u0ad5\0\u0b04"+
"\0\u0b33\0\u0b62\0\u0b91\0\u0bc0\0\u0bef\0\u0c1e\0\u0c4d\0\u0c7c"+
"\0\u0cab\0\u0cda\0\u0d09\0\u0d38\0\u0d67\0\u0d96\0\u0dc5\0\u0df4"+
"\0\u0e23\0\u0e52\0\u0e81\0\u0eb0\0\u0edf\0\u0f0e\0\u0f3d\0\u0f6c"+
"\0\u0f9b\0\u0fca\0\u0ff9\0\u1028\0\u1057\0\u1086\0\u10b5\0\u10e4"+
"\0\u1113\0\u1142\0\u1171\0\u11a0\0\u11cf\0\u11fe\0\u122d\0\u125c"+
"\0\u128b\0\u12ba\0\u12e9\0\u1318\0\u1347\0\u1376\0\215\0\u13a5"+
"\0\u13d4\0\u1403\0\u1432\0\u1461\0\u1490\0\u14bf\0\u1347\0\u098c"+
"\0\u09bb\0\u14ee\0\u151d\0\u154c\0\u157b\0\u15aa\0\u15d9\0\u1608"+
"\0\u1637\0\u1666";
private static int [] zzUnpackRowMap() {
int [] result = new int[130];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
}
private static int zzUnpackRowMap(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int high = packed.charAt(i++) << 16;
result[j++] = high | packed.charAt(i++);
}
return j;
}
/**
* The transition table of the DFA
*/
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
"\1\2\1\3\1\4\1\5\1\4\1\6\1\7\1\10"+
"\1\11\1\10\1\12\1\13\1\14\1\15\1\16\1\12"+
"\1\16\2\10\1\17\1\20\1\16\1\21\3\22\2\23"+
"\1\24\3\23\1\25\1\22\1\23\1\26\11\22\1\6"+
"\1\10\1\27\2\0\1\27\1\0\1\27\1\30\5\27"+
"\1\0\15\27\6\0\2\27\2\0\13\27\2\0\1\4"+
"\133\0\1\27\2\0\1\27\1\0\1\6\1\30\5\27"+
"\1\0\15\27\2\0\1\24\3\0\2\27\2\0\11\27"+
"\1\6\1\27\6\0\1\31\1\23\1\0\1\23\1\0"+
"\3\23\3\0\2\23\1\0\1\23\5\0\2\23\1\0"+
"\3\23\2\0\1\23\13\0\1\23\1\27\2\0\1\27"+
"\1\0\1\27\1\32\1\10\1\27\1\10\1\27\1\10"+
"\1\23\1\10\3\27\2\10\1\27\1\10\5\27\2\23"+
"\1\0\3\23\2\27\1\23\1\0\12\27\1\10\1\27"+
"\2\0\1\27\1\0\1\27\1\33\1\27\2\11\1\27"+
"\1\11\1\34\5\27\1\11\7\27\1\0\1\34\3\0"+
"\1\34\2\27\1\0\1\34\14\27\2\0\1\27\1\0"+
"\1\27\1\30\1\27\1\35\1\27\2\12\1\36\1\27"+
"\3\12\1\27\1\12\2\27\1\12\4\27\6\0\2\27"+
"\1\0\1\37\14\27\2\0\1\27\1\0\1\27\1\32"+
"\1\10\1\35\1\10\1\12\1\13\1\40\1\10\3\12"+
"\1\10\1\13\1\27\1\10\1\12\4\27\2\23\1\0"+
"\3\23\2\27\1\23\1\37\12\27\1\10\6\0\2\23"+
"\1\0\1\23\1\41\1\42\1\14\1\43\3\41\2\23"+
"\1\0\1\23\1\41\4\0\2\23\1\0\3\23\2\0"+
"\1\23\13\0\1\23\1\27\2\0\1\27\1\0\1\27"+
"\1\32\1\10\1\27\1\10\1\27\1\10\1\23\1\10"+
"\1\44\2\27\2\10\1\27\1\10\5\27\2\23\1\0"+
"\3\23\2\27\1\23\1\0\12\27\1\10\1\27\2\0"+
"\1\27\1\0\1\27\1\30\1\27\1\35\1\27\2\12"+
"\1\36\1\27\1\16\1\12\1\16\1\27\1\12\2\17"+
"\1\16\4\17\6\0\2\17\1\0\1\37\11\17\3\27"+
"\2\0\1\27\1\0\1\27\1\30\5\27\1\0\1\27"+
"\1\17\1\27\1\17\2\27\7\17\6\0\2\17\2\0"+
"\11\17\3\27\2\0\1\27\1\0\1\27\1\32\1\10"+
"\1\27\1\10\1\27\1\10\1\23\1\10\1\17\1\27"+
"\1\17\2\10\1\17\1\20\5\17\2\23\1\0\3\23"+
"\2\17\1\23\1\0\11\17\1\27\1\10\1\45\2\0"+
"\1\45\1\0\1\45\1\46\5\45\1\47\1\45\1\22"+
"\1\45\1\22\2\45\4\22\1\50\2\22\2\47\2\0"+
"\2\47\2\22\2\47\11\22\1\27\1\51\1\45\2\0"+
"\1\45\1\0\1\45\1\46\5\45\1\47\1\45\1\22"+
"\1\45\1\22\2\45\7\22\2\47\2\0\2\47\2\22"+
"\2\47\11\22\1\27\1\51\6\0\2\23\1\0\1\23"+
"\1\0\3\23\3\0\2\23\1\0\1\23\5\0\2\23"+
"\1\0\3\23\2\0\1\23\13\0\1\23\5\0\1\24"+
"\26\0\1\24\20\0\1\24\1\0\1\45\2\0\1\45"+
"\1\0\1\45\1\46\5\45\1\47\1\45\1\22\1\45"+
"\1\22\2\45\7\22\2\47\2\0\2\47\1\52\1\22"+
"\2\47\11\22\1\27\1\51\1\47\2\0\1\47\1\0"+
"\1\47\1\26\1\47\2\26\1\47\2\26\5\47\1\26"+
"\10\47\1\26\2\0\1\47\1\26\3\47\1\26\11\47"+
"\1\0\1\53\1\27\2\0\1\27\1\0\1\27\1\0"+
"\5\27\1\0\15\27\6\0\2\27\2\0\1\54\1\55"+
"\1\27\1\56\1\57\6\27\6\0\1\60\1\23\1\0"+
"\1\23\1\0\3\23\3\0\2\23\1\0\1\23\5\0"+
"\2\23\1\0\3\23\2\0\1\23\13\0\1\23\1\27"+
"\2\0\1\27\1\0\1\27\1\23\1\10\1\27\1\10"+
"\1\27\1\10\1\23\1\10\3\27\2\10\1\27\1\10"+
"\5\27\2\23\1\0\3\23\2\27\1\23\1\0\1\54"+
"\1\55\1\27\1\56\1\57\5\27\1\10\1\27\2\0"+
"\1\27\1\0\1\27\1\34\1\27\2\11\1\27\1\11"+
"\1\34\5\27\1\11\7\27\1\0\1\34\3\0\1\34"+
"\2\27\1\0\1\34\1\54\1\55\1\27\1\56\1\57"+
"\6\27\6\0\1\34\1\0\2\34\1\0\2\34\5\0"+
"\1\34\10\0\1\34\3\0\1\34\3\0\1\34\13\0"+
"\1\27\2\0\1\27\1\0\1\27\1\30\1\27\1\35"+
"\3\27\1\0\15\27\6\0\2\27\1\0\1\37\13\27"+
"\10\0\1\37\3\0\1\36\26\0\1\37\23\0\1\37"+
"\32\0\1\37\21\0\2\23\1\37\1\23\1\0\1\23"+
"\1\40\1\23\3\0\2\23\1\0\1\23\5\0\2\23"+
"\1\0\3\23\2\0\1\23\1\37\12\0\1\23\10\0"+
"\1\37\1\0\2\41\1\36\1\0\3\41\1\0\1\41"+
"\2\0\1\41\15\0\1\37\21\0\2\23\1\37\1\23"+
"\1\41\1\42\1\40\1\23\3\41\1\23\1\42\1\0"+
"\1\23\1\41\4\0\2\23\1\0\3\23\2\0\1\23"+
"\1\37\12\0\1\23\6\0\2\23\1\0\1\23\1\0"+
"\3\23\1\61\2\0\2\23\1\0\1\23\5\0\2\23"+
"\1\0\3\23\2\0\1\23\13\0\1\23\1\27\2\0"+
"\1\27\1\0\1\27\1\30\5\27\1\0\2\27\1\62"+
"\12\27\6\0\2\27\2\0\13\27\1\45\2\0\1\45"+
"\1\0\1\45\1\46\5\45\1\47\15\45\2\47\2\0"+
"\2\47\2\45\2\47\11\45\1\27\1\51\1\45\2\0"+
"\1\45\1\0\1\45\1\47\5\45\1\47\15\45\2\47"+
"\2\0\2\47\2\45\2\47\1\63\1\64\1\45\1\65"+
"\1\66\4\45\1\27\1\51\1\47\2\0\1\47\1\0"+
"\27\47\2\0\17\47\1\0\1\53\1\45\2\0\1\45"+
"\1\0\1\45\1\46\5\45\1\47\1\45\1\22\1\45"+
"\1\22\2\45\4\22\1\67\2\22\2\47\2\0\2\47"+
"\2\22\2\47\11\22\1\27\1\51\1\70\2\0\1\70"+
"\1\0\1\70\1\46\5\70\1\71\15\70\2\71\2\0"+
"\2\71\2\70\2\71\11\70\1\27\1\70\1\45\2\0"+
"\1\45\1\0\1\45\1\46\5\45\1\47\1\45\1\22"+
"\1\45\1\22\2\45\7\22\2\47\2\0\2\47\1\72"+
"\1\22\2\47\11\22\1\27\1\51\1\71\2\0\1\71"+
"\1\0\1\71\1\47\25\71\2\0\17\71\1\0\1\71"+
"\1\27\2\0\1\27\1\0\1\27\1\30\5\27\1\0"+
"\15\27\6\0\2\27\2\0\1\27\1\73\12\27\2\0"+
"\1\27\1\0\1\27\1\30\5\27\1\0\15\27\6\0"+
"\2\27\2\0\5\27\1\74\6\27\2\0\1\27\1\0"+
"\1\27\1\30\5\27\1\0\15\27\6\0\2\27\2\0"+
"\4\27\1\75\7\27\2\0\1\27\1\0\1\27\1\30"+
"\5\27\1\0\15\27\6\0\2\27\2\0\7\27\1\76"+
"\3\27\17\0\1\77\37\0\1\27\2\0\1\27\1\0"+
"\1\27\1\30\5\27\1\0\3\27\1\100\11\27\6\0"+
"\2\27\2\0\13\27\1\45\2\0\1\45\1\0\1\45"+
"\1\46\5\45\1\47\15\45\2\47\2\0\2\47\2\45"+
"\2\47\1\45\1\101\7\45\1\27\1\51\1\45\2\0"+
"\1\45\1\0\1\45\1\46\5\45\1\47\15\45\2\47"+
"\2\0\2\47\2\45\2\47\5\45\1\102\3\45\1\27"+
"\1\51\1\45\2\0\1\45\1\0\1\45\1\46\5\45"+
"\1\47\15\45\2\47\2\0\2\47\2\45\2\47\4\45"+
"\1\103\4\45\1\27\1\51\1\45\2\0\1\45\1\0"+
"\1\45\1\46\5\45\1\47\15\45\2\47\2\0\2\47"+
"\2\45\2\47\7\45\1\104\1\45\1\27\1\51\1\45"+
"\2\0\1\45\1\0\1\45\1\46\5\45\1\47\1\45"+
"\1\22\1\45\1\22\2\45\5\22\1\105\1\22\2\47"+
"\2\0\2\47\2\22\2\47\11\22\1\27\1\51\1\70"+
"\2\0\1\70\1\0\1\70\1\106\5\70\1\71\15\70"+
"\2\71\2\0\2\71\2\70\2\71\11\70\1\27\1\70"+
"\1\71\2\0\1\71\1\0\1\71\1\107\25\71\2\0"+
"\17\71\1\0\1\71\1\45\2\0\1\45\1\0\1\45"+
"\1\110\5\45\1\47\1\45\1\22\1\45\1\22\2\45"+
"\7\22\2\47\2\0\2\47\2\22\2\47\11\22\1\27"+
"\1\51\1\27\2\0\1\27\1\0\1\27\1\30\5\27"+
"\1\0\15\27\6\0\2\27\2\0\2\27\1\111\11\27"+
"\2\0\1\27\1\0\1\27\1\30\5\27\1\0\15\27"+
"\6\0\2\27\2\0\6\27\1\111\5\27\2\0\1\27"+
"\1\0\1\27\1\30\5\27\1\0\12\27\1\111\2\27"+
"\6\0\2\27\2\0\14\27\2\0\1\27\1\0\1\27"+
"\1\30\5\27\1\0\15\27\6\0\2\27\2\0\10\27"+
"\1\111\2\27\20\0\1\112\36\0\1\27\2\0\1\27"+
"\1\0\1\27\1\30\5\27\1\0\3\27\1\113\11\27"+
"\6\0\2\27\2\0\13\27\1\45\2\0\1\45\1\0"+
"\1\45\1\46\5\45\1\47\15\45\2\47\2\0\2\47"+
"\2\45\2\47\2\45\1\114\6\45\1\27\1\51\1\45"+
"\2\0\1\45\1\0\1\45\1\46\5\45\1\47\15\45"+
"\2\47\2\0\2\47\2\45\2\47\6\45\1\114\2\45"+
"\1\27\1\51\1\45\2\0\1\45\1\0\1\45\1\46"+
"\5\45\1\47\12\45\1\114\2\45\2\47\2\0\2\47"+
"\2\45\2\47\11\45\1\27\1\51\1\45\2\0\1\45"+
"\1\0\1\45\1\46\5\45\1\47\15\45\2\47\2\0"+
"\2\47\2\45\2\47\10\45\1\114\1\27\1\51\1\45"+
"\2\0\1\45\1\0\1\45\1\46\5\45\1\47\1\45"+
"\1\22\1\45\1\22\2\45\6\22\1\115\1\116\1\47"+
"\2\0\2\47\2\22\2\47\11\22\1\27\1\51\1\70"+
"\2\0\1\70\1\0\1\70\1\47\5\70\1\71\11\70"+
"\4\117\2\71\2\0\2\71\2\117\2\71\1\120\1\121"+
"\1\117\1\122\1\123\4\117\1\27\1\70\1\71\2\0"+
"\1\71\1\0\1\71\1\47\17\71\4\124\2\71\2\0"+
"\2\71\2\124\2\71\11\124\1\0\1\71\1\125\2\0"+
"\1\125\1\0\1\125\1\47\25\125\2\0\2\47\4\125"+
"\1\126\1\127\1\125\1\130\1\131\4\125\1\132\1\133"+
"\1\27\2\0\1\27\1\0\1\27\1\30\5\27\1\0"+
"\15\27\1\0\1\134\4\0\2\27\2\0\13\27\20\0"+
"\1\135\36\0\1\27\2\0\1\27\1\0\1\27\1\30"+
"\5\27\1\0\4\27\1\136\10\27\6\0\2\27\2\0"+
"\13\27\1\45\2\0\1\45\1\0\1\45\1\46\5\45"+
"\1\47\15\45\1\47\1\134\2\0\2\47\2\45\2\47"+
"\11\45\1\27\1\51\1\45\2\0\1\45\1\0\1\45"+
"\1\46\5\45\1\47\1\45\1\22\1\45\1\22\2\45"+
"\7\22\1\116\1\47\2\0\2\47\2\22\2\47\11\22"+
"\1\27\1\51\1\47\2\0\1\47\1\0\26\47\1\137"+
"\2\0\17\47\1\0\1\53\1\70\2\0\1\70\1\0"+
"\1\70\1\106\5\70\1\71\11\70\4\140\2\71\2\0"+
"\2\71\2\140\2\71\11\140\1\27\2\70\2\0\1\70"+
"\1\0\1\70\1\106\5\70\1\71\11\70\4\140\2\71"+
"\2\0\2\71\2\140\2\71\1\140\1\141\7\140\1\27"+
"\2\70\2\0\1\70\1\0\1\70\1\106\5\70\1\71"+
"\11\70\4\140\2\71\2\0\2\71\2\140\2\71\5\140"+
"\1\142\3\140\1\27\2\70\2\0\1\70\1\0\1\70"+
"\1\106\5\70\1\71\11\70\4\140\2\71\2\0\2\71"+
"\2\140\2\71\4\140\1\143\4\140\1\27\2\70\2\0"+
"\1\70\1\0\1\70\1\106\5\70\1\71\11\70\4\140"+
"\2\71\2\0\2\71\2\140\2\71\7\140\1\144\1\140"+
"\1\27\1\70\1\71\2\0\1\71\1\0\1\71\1\107"+
"\17\71\4\145\2\71\2\0\2\71\2\145\2\71\11\145"+
"\1\0\1\71\1\125\2\0\1\125\1\0\1\125\1\146"+
"\25\125\2\0\2\47\15\125\1\132\1\133\1\125\2\0"+
"\1\125\1\0\1\125\1\146\25\125\2\0\2\47\5\125"+
"\1\147\7\125\1\132\1\133\1\125\2\0\1\125\1\0"+
"\1\125\1\146\25\125\2\0\2\47\11\125\1\150\3\125"+
"\1\132\1\133\1\125\2\0\1\125\1\0\1\125\1\146"+
"\25\125\2\0\2\47\10\125\1\151\4\125\1\132\1\133"+
"\1\125\2\0\1\125\1\0\1\125\1\146\25\125\2\0"+
"\2\47\13\125\1\152\1\125\1\132\1\133\1\132\2\0"+
"\1\132\1\0\1\132\1\153\25\132\4\0\17\132\1\154"+
"\2\0\1\154\1\0\1\154\1\146\25\154\2\0\2\71"+
"\15\154\1\132\1\154\1\155\2\0\1\155\1\0\27\155"+
"\2\0\21\155\21\0\1\156\35\0\1\27\2\0\1\27"+
"\1\0\1\27\1\30\5\27\1\157\15\27\6\0\2\27"+
"\2\0\13\27\1\47\2\0\1\47\1\0\26\47\1\134"+
"\2\0\17\47\1\0\1\53\1\70\2\0\1\70\1\0"+
"\1\70\1\106\5\70\1\71\11\70\4\160\2\71\2\0"+
"\2\71\2\160\2\71\11\160\1\27\2\70\2\0\1\70"+
"\1\0\1\70\1\106\5\70\1\71\11\70\4\160\2\71"+
"\2\0\2\71\2\160\2\71\2\160\1\161\6\160\1\27"+
"\2\70\2\0\1\70\1\0\1\70\1\106\5\70\1\71"+
"\11\70\4\160\2\71\2\0\2\71\2\160\2\71\6\160"+
"\1\161\2\160\1\27\2\70\2\0\1\70\1\0\1\70"+
"\1\106\5\70\1\71\11\70\1\160\1\161\2\160\2\71"+
"\2\0\2\71\2\160\2\71\11\160\1\27\2\70\2\0"+
"\1\70\1\0\1\70\1\106\5\70\1\71\11\70\4\160"+
"\2\71\2\0\2\71\2\160\2\71\10\160\1\161\1\27"+
"\1\70\1\71\2\0\1\71\1\0\1\71\1\107\17\71"+
"\4\162\2\71\2\0\2\71\2\162\2\71\11\162\1\0"+
"\1\71\1\125\2\0\1\125\1\0\1\125\1\47\17\125"+
"\4\163\2\125\2\0\2\47\2\163\2\125\11\163\1\132"+
"\1\133\1\125\2\0\1\125\1\0\1\125\1\146\25\125"+
"\2\0\2\47\6\125\1\164\6\125\1\132\1\133\1\125"+
"\2\0\1\125\1\0\1\125\1\146\25\125\2\0\2\47"+
"\12\125\1\164\2\125\1\132\1\133\1\125\2\0\1\125"+
"\1\0\1\125\1\146\20\125\1\164\4\125\2\0\2\47"+
"\15\125\1\132\1\133\1\125\2\0\1\125\1\0\1\125"+
"\1\146\25\125\2\0\2\47\14\125\1\164\1\132\1\133"+
"\1\132\2\0\1\132\1\0\1\132\1\0\17\132\4\165"+
"\2\132\4\0\2\165\2\132\11\165\2\132\1\154\2\0"+
"\1\154\1\0\1\154\1\166\25\154\2\0\2\71\15\154"+
"\1\132\1\154\1\167\2\0\1\167\1\0\1\167\1\155"+
"\5\167\1\155\17\167\2\0\2\155\17\167\14\0\1\157"+
"\42\0\1\70\2\0\1\70\1\0\1\70\1\106\5\70"+
"\1\71\11\70\4\170\2\71\2\0\2\71\2\170\2\71"+
"\11\170\1\27\2\70\2\0\1\70\1\0\1\70\1\106"+
"\5\70\1\71\11\70\4\170\1\71\1\134\2\0\2\71"+
"\2\170\2\71\11\170\1\27\1\70\1\71\2\0\1\71"+
"\1\0\1\71\1\107\17\71\4\171\2\71\2\0\2\71"+
"\2\171\2\71\11\171\1\0\1\71\1\125\2\0\1\125"+
"\1\0\1\125\1\146\17\125\4\172\2\125\2\0\2\47"+
"\2\172\2\125\11\172\1\132\1\133\1\125\2\0\1\125"+
"\1\0\1\125\1\146\24\125\1\134\2\0\2\47\15\125"+
"\1\132\1\133\1\132\2\0\1\132\1\0\1\132\1\153"+
"\17\132\4\173\2\132\4\0\2\173\2\132\11\173\2\132"+
"\1\154\2\0\1\154\1\0\1\154\1\47\17\154\4\174"+
"\2\154\2\0\2\71\2\174\2\154\11\174\1\132\1\154"+
"\1\125\2\0\1\125\1\0\1\125\1\146\17\125\4\175"+
"\1\125\1\134\2\0\2\47\2\175\2\125\11\175\1\132"+
"\1\133\1\132\2\0\1\132\1\0\1\132\1\153\17\132"+
"\4\176\1\132\1\134\4\0\2\176\2\132\11\176\2\132"+
"\1\154\2\0\1\154\1\0\1\154\1\166\17\154\4\177"+
"\2\154\2\0\2\71\2\177\2\154\11\177\1\132\1\154"+
"\1\125\2\0\1\125\1\0\1\125\1\146\17\125\4\164"+
"\1\125\1\134\2\0\2\47\2\164\2\125\11\164\1\132"+
"\1\133\1\132\2\0\1\132\1\0\1\132\1\153\17\132"+
"\4\200\1\132\1\134\4\0\2\200\2\132\11\200\2\132"+
"\1\154\2\0\1\154\1\0\1\154\1\166\17\154\4\201"+
"\1\154\1\134\2\0\2\71\2\201\2\154\11\201\1\132"+
"\1\154\1\132\2\0\1\132\1\0\1\132\1\153\24\132"+
"\1\134\4\0\17\132\1\154\2\0\1\154\1\0\1\154"+
"\1\166\17\154\4\202\1\154\1\134\2\0\2\71\2\202"+
"\2\154\11\202\1\132\2\154\2\0\1\154\1\0\1\154"+
"\1\166\24\154\1\134\2\0\2\71\15\154\1\132\1\154";
private static int [] zzUnpackTrans() {
int [] result = new int[5781];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
}
private static int zzUnpackTrans(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
value--;
do result[j++] = value; while (--count > 0);
}
return j;
}
/* error codes */
private static final int ZZ_UNKNOWN_ERROR = 0;
private static final int ZZ_NO_MATCH = 1;
private static final int ZZ_PUSHBACK_2BIG = 2;
/* error messages for the codes above */
private static final String ZZ_ERROR_MSG[] = {
"Unkown internal scanner error",
"Error: could not match input",
"Error: pushback value was too large"
};
/**
* ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
*/
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
"\1\0\2\1\1\11\22\1\2\0\13\1\4\0\1\1"+
"\1\0\1\1\5\0\1\1\6\0\1\1\2\0\1\1"+
"\12\0\1\1\3\0\1\1\2\0\2\1\20\0\1\1"+
"\1\0\6\1\10\0\1\1\1\11\3\1\1\0\1\1"+
"\2\0\5\1\1\0\6\1";
private static int [] zzUnpackAttribute() {
int [] result = new int[130];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
}
private static int zzUnpackAttribute(String packed, int offset, int [] result) {
int i = 0; /* index in packed string */
int j = offset; /* index in unpacked array */
int l = packed.length();
while (i < l) {
int count = packed.charAt(i++);
int value = packed.charAt(i++);
do result[j++] = value; while (--count > 0);
}
return j;
}
/** the input device */
private java.io.Reader zzReader;
/** the current state of the DFA */
private int zzState;
/** the current lexical state */
private int zzLexicalState = YYINITIAL;
/** this buffer contains the current text to be matched and is
the source of the yytext() string */
private char zzBuffer[] = new char[ZZ_BUFFERSIZE];
/** the textposition at the last accepting state */
private int zzMarkedPos;
/** the current text position in the buffer */
private int zzCurrentPos;
/** startRead marks the beginning of the yytext() string in the buffer */
private int zzStartRead;
/** endRead marks the last character in the buffer, that has been read
from input */
private int zzEndRead;
/** number of newlines encountered up to the start of the matched text */
private int yyline;
/** the number of characters up to the start of the matched text */
private int yychar;
/**
* the number of characters from the last newline up to the start of the
* matched text
*/
private int yycolumn;
/**
* zzAtBOL == true <=> the scanner is currently at the beginning of a line
*/
private boolean zzAtBOL = true;
/** zzAtEOF == true <=> the scanner is at the EOF */
private boolean zzAtEOF;
/** denotes if the user-EOF-code has already been executed */
private boolean zzEOFDone;
/* user code: */
private LexedTokenFactory<?> tokenFactory;
private boolean invertible;
// Convert Arabic digits to ASCII digits
private boolean normArDigits;
// Convert Arabic punctuation to ASCII equivalents
private boolean normArPunc;
// Substitute newlines with newlineChar.
// Otherwise, treat them like whitespace
private boolean tokenizeNL;
private String newlineChar;
// Use \u2026 for ellipses
private boolean useUTF8Ellipsis;
// Arabic-specific orthographic normalization rules
private boolean normAlif;
private boolean normYa;
private boolean removeDiacritics;
private boolean removeTatweel;
private boolean removeQuranChars;
// Penn ATB vocalized section normalizations
private boolean removeProMarker;
private boolean removeSegMarker;
private boolean removeMorphMarker;
// Lengthening effects (yAAAAAAA): replace three or more of the same character with one
private boolean removeLengthening;
private final Pattern segmentationMarker = Pattern.compile("^-+|-+$");
// Escape parens for ATB parsing
private boolean atbEscaping;
// Normalize newlines to this token
public static final String NEWLINE_TOKEN = "*NL*";
private static final String SYSTEM_NEWLINE = System.getProperty("line.separator");
private Map<String,String> normMap;
public ArabicLexer(Reader r, LexedTokenFactory<?> tf, Properties props) {
this(r);
this.tokenFactory = tf;
tokenizeNL = PropertiesUtils.getBool(props, "tokenizeNLs", false);
useUTF8Ellipsis = PropertiesUtils.getBool(props, "useUTF8Ellipsis", false);
invertible = PropertiesUtils.getBool(props, "invertible", false);
normArDigits = PropertiesUtils.getBool(props, "normArDigits", false);
normArPunc = PropertiesUtils.getBool(props, "normArPunc", false);
normAlif = PropertiesUtils.getBool(props, "normAlif", false);
normYa = PropertiesUtils.getBool(props, "normYa", false);
removeDiacritics = PropertiesUtils.getBool(props, "removeDiacritics", false);
removeTatweel = PropertiesUtils.getBool(props, "removeTatweel", false);
removeQuranChars = PropertiesUtils.getBool(props, "removeQuranChars", false);
removeProMarker = PropertiesUtils.getBool(props, "removeProMarker", false);
removeSegMarker = PropertiesUtils.getBool(props, "removeSegMarker", false);
removeMorphMarker = PropertiesUtils.getBool(props, "removeMorphMarker", false);
removeLengthening = PropertiesUtils.getBool(props, "removeLengthening", false);
atbEscaping = PropertiesUtils.getBool(props, "atbEscaping", false);
setupNormalizationMap();
}
private void setupNormalizationMap() {
normMap = Generics.newHashMap(200);
// Junk characters that we always remove
normMap.put("\u0600","#");
normMap.put("\u0601","");
normMap.put("\u0602","");
normMap.put("\u0603","");
normMap.put("\u0606","\u221B");
normMap.put("\u0607","\u221C");
normMap.put("\u0608","");
normMap.put("\u0609","%");
normMap.put("\u060A","%");
normMap.put("\u060B","");
normMap.put("\u060E","");
normMap.put("\u060F","");
normMap.put("\u066E","\u0628");
normMap.put("\u066F","\u0642");
normMap.put("\u06CC","\u0649");
normMap.put("\u06D6","");
normMap.put("\u06D7","");
normMap.put("\u06D8","");
normMap.put("\u06D9","");
normMap.put("\u06DA","");
normMap.put("\u06DB","");
normMap.put("\u06DC","");
normMap.put("\u06DD","");
normMap.put("\u06DE","");
normMap.put("\u06DF","");
normMap.put("\u06E0","");
normMap.put("\u06E1","");
normMap.put("\u06E2","");
normMap.put("\u06E3","");
normMap.put("\u06E4","");
normMap.put("\u06E5","");
normMap.put("\u06E6","");
normMap.put("\u06E7","");
normMap.put("\u06E8","");
normMap.put("\u06E9","");
normMap.put("\u06EA","");
normMap.put("\u06EB","");
normMap.put("\u06EC","");
normMap.put("\u06ED","");
if (normArDigits) {
normMap.put("\u0660","0");
normMap.put("\u0661","1");
normMap.put("\u0662","2");
normMap.put("\u0663","3");
normMap.put("\u0664","4");
normMap.put("\u0665","5");
normMap.put("\u0666","6");
normMap.put("\u0667","7");
normMap.put("\u0668","8");
normMap.put("\u0669","9");
normMap.put("\u06F0","0");
normMap.put("\u06F1","1");
normMap.put("\u06F2","2");
normMap.put("\u06F3","3");
normMap.put("\u06F4","4");
normMap.put("\u06F5","5");
normMap.put("\u06F6","6");
normMap.put("\u06F7","7");
normMap.put("\u06F8","8");
normMap.put("\u06F9","9");
}
if (normArPunc) {
normMap.put("\u00BB","\"");
normMap.put("\u00AB","\"");
normMap.put("\u060C",",");
normMap.put("\u060D",",");
normMap.put("\u061B",";");
normMap.put("\u061E",".");
normMap.put("\u061F","?");
normMap.put("\u066A","%");
normMap.put("\u066B",",");
normMap.put("\u066C","\u0027");
normMap.put("\u066F","*");
normMap.put("\u06DF",".");
}
if (normAlif) {
normMap.put("\u0622","\u0627");
normMap.put("\u0623","\u0627");
normMap.put("\u0625","\u0627");
normMap.put("\u0671","\u0627");
normMap.put("\u0672","\u0627");
normMap.put("\u0673","\u0627");
}
if (normYa) {
normMap.put("\u064A","\u0649");
}
if (removeDiacritics) {
normMap.put("\u064B","");
normMap.put("\u064C","");
normMap.put("\u064D","");
normMap.put("\u064E","");
normMap.put("\u064F","");
normMap.put("\u0650","");
normMap.put("\u0651","");
normMap.put("\u0652","");
normMap.put("\u0653","");
normMap.put("\u0654","");
normMap.put("\u0655","");
normMap.put("\u0656","");
normMap.put("\u0657","");
normMap.put("\u0658","");
normMap.put("\u0659","");
normMap.put("\u065A","");
normMap.put("\u065B","");
normMap.put("\u065C","");
normMap.put("\u065D","");
normMap.put("\u065E","");
normMap.put("\u0670","");
}
if (removeTatweel) {
normMap.put("\u0640","");
normMap.put("_","");
}
if (removeQuranChars) {
// Arabic honorifics
normMap.put("\u0610","");
normMap.put("\u0611","");
normMap.put("\u0612","");
normMap.put("\u0613","");
normMap.put("\u0614","");
normMap.put("\u0615","");
normMap.put("\u0616","");
normMap.put("\u0617","");
normMap.put("\u0618","");
normMap.put("\u0619","");
normMap.put("\u061A","");
}
if (atbEscaping) {
normMap.put("(","-LRB-");
normMap.put(")","-RRB-");
}
}
private String normalizeToken(String text, boolean isWord) {
// Remove segmentation markers from the ATB
if (isWord && removeSegMarker) {
text = segmentationMarker.matcher(text).replaceAll("");
}
int len = text.length();
StringBuilder sb = new StringBuilder(len);
for (int i = 0; i < len; ++i) {
String thisChar = String.valueOf(text.charAt(i));
// Remove morpheme markers from the ATB vocalized section
if (isWord && removeMorphMarker && thisChar.equals("+")) {
continue;
}
if (removeLengthening && isLengthening(text, i)) {
continue;
}
if (normMap.containsKey(thisChar)) {
thisChar = normMap.get(thisChar);
}
if (thisChar.length() > 0) {
sb.append(thisChar);
}
}
return sb.toString();
}
private boolean isLengthening(String text, int pos) {
if (pos == 0) return false;
String thisChar = String.valueOf(text.charAt(pos));
if (!thisChar.equals(String.valueOf(text.charAt(pos - 1))))
return false;
if (pos < text.length() - 1 && thisChar.equals(String.valueOf(text.charAt(pos + 1))))
return true;
if (pos >= 2 && thisChar.equals(String.valueOf(text.charAt(pos - 2))))
return true;
return false;
}
/** Make the next token.
*
* @param txt What the token should be
* @param originalText The original String that got transformed into txt
*/
private Object getNext(String txt, String originalText) {
if (tokenFactory == null) {
throw new RuntimeException(this.getClass().getName() + ": Token factory is null.");
}
if (invertible) {
//String str = prevWordAfter.toString();
//prevWordAfter.setLength(0);
CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength());
word.set(CoreAnnotations.OriginalTextAnnotation.class, originalText);
//word.set(CoreAnnotations.BeforeAnnotation.class, str);
//prevWord.set(CoreAnnotations.AfterAnnotation.class, str);
//prevWord = word;
return word;
} else {
return tokenFactory.makeToken(txt, yychar, yylength());
}
}
private Object getNext(boolean isWord) {
String text = yytext();
String normText = normalizeToken(text, isWord);
return getNext(normText, text);
}
private Object getNewline() {
String nlString = tokenizeNL ? NEWLINE_TOKEN : SYSTEM_NEWLINE;
return getNext(nlString, yytext());
}
private Object getEllipsis() {
String ellipsisString = useUTF8Ellipsis ? "\u2026" : "...";
return getNext(ellipsisString, yytext());
}
/**
* Creates a new scanner
* There is also a java.io.InputStream version of this constructor.
*
* @param in the java.io.Reader to read input from.
*/
ArabicLexer(java.io.Reader in) {
this.zzReader = in;
}
/**
* Creates a new scanner.
* There is also java.io.Reader version of this constructor.
*
* @param in the java.io.Inputstream to read input from.
*/
ArabicLexer(java.io.InputStream in) {
this(new java.io.InputStreamReader(in));
}
/**
* Unpacks the compressed character translation table.
*
* @param packed the packed character translation table
* @return the unpacked character translation table
*/
private static char [] zzUnpackCMap(String packed) {
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
while (i < 1918) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
}
return map;
}
/**
* Refills the input buffer.
*
* @return <code>false</code>, iff there was new input.
*
* @exception java.io.IOException if any I/O-Error occurs
*/
private boolean zzRefill() throws java.io.IOException {
/* first: make room (if you can) */
if (zzStartRead > 0) {
System.arraycopy(zzBuffer, zzStartRead,
zzBuffer, 0,
zzEndRead-zzStartRead);
/* translate stored positions */
zzEndRead-= zzStartRead;
zzCurrentPos-= zzStartRead;
zzMarkedPos-= zzStartRead;
zzStartRead = 0;
}
/* is the buffer big enough? */
if (zzCurrentPos >= zzBuffer.length) {
/* if not: blow it up */
char newBuffer[] = new char[zzCurrentPos*2];
System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
zzBuffer = newBuffer;
}
/* finally: fill the buffer with new input */
int numRead = zzReader.read(zzBuffer, zzEndRead,
zzBuffer.length-zzEndRead);
if (numRead > 0) {
zzEndRead+= numRead;
return false;
}
// unlikely but not impossible: read 0 characters, but not at end of stream
if (numRead == 0) {
int c = zzReader.read();
if (c == -1) {
return true;
} else {
zzBuffer[zzEndRead++] = (char) c;
return false;
}
}
// numRead < 0
return true;
}
/**
* Closes the input stream.
*/
public final void yyclose() throws java.io.IOException {
zzAtEOF = true; /* indicate end of file */
zzEndRead = zzStartRead; /* invalidate buffer */
if (zzReader != null)
zzReader.close();
}
/**
* Resets the scanner to read from a new input stream.
* Does not close the old reader.
*
* All internal variables are reset, the old input stream
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
* @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) {
zzReader = reader;
zzAtBOL = true;
zzAtEOF = false;
zzEOFDone = false;
zzEndRead = zzStartRead = 0;
zzCurrentPos = zzMarkedPos = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
}
/**
* Returns the current lexical state.
*/
public final int yystate() {
return zzLexicalState;
}
/**
* Enters a new lexical state
*
* @param newState the new lexical state
*/
public final void yybegin(int newState) {
zzLexicalState = newState;
}
/**
* Returns the text matched by the current regular expression.
*/
public final String yytext() {
return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
}
/**
* Returns the character at position <tt>pos</tt> from the
* matched text.
*
* It is equivalent to yytext().charAt(pos), but faster
*
* @param pos the position of the character to fetch.
* A value from 0 to yylength()-1.
*
* @return the character at position pos
*/
public final char yycharat(int pos) {
return zzBuffer[zzStartRead+pos];
}
/**
* Returns the length of the matched text region.
*/
public final int yylength() {
return zzMarkedPos-zzStartRead;
}
/**
* Reports an error that occured while scanning.
*
* In a wellformed scanner (no or only correct usage of
* yypushback(int) and a match-all fallback rule) this method
* will only be called with things that "Can't Possibly Happen".
* If this method is called, something is seriously wrong
* (e.g. a JFlex bug producing a faulty scanner etc.).
*
* Usual syntax/scanner level error handling should be done
* in error fallback rules.
*
* @param errorCode the code of the errormessage to display
*/
private void zzScanError(int errorCode) {
String message;
try {
message = ZZ_ERROR_MSG[errorCode];
}
catch (ArrayIndexOutOfBoundsException e) {
message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
}
throw new Error(message);
}
/**
* Pushes the specified amount of characters back into the input stream.
*
* They will be read again by then next call of the scanning method
*
* @param number the number of characters to be read again.
* This number must not be greater than yylength()!
*/
public void yypushback(int number) {
if ( number > yylength() )
zzScanError(ZZ_PUSHBACK_2BIG);
zzMarkedPos -= number;
}
/**
* Resumes scanning until the next regular expression is matched,
* the end of input is encountered or an I/O-Error occurs.
*
* @return the next token
* @exception java.io.IOException if any I/O-Error occurs
*/
public Object next() throws java.io.IOException {
int zzInput;
int zzAction;
// cached fields:
int zzCurrentPosL;
int zzMarkedPosL;
int zzEndReadL = zzEndRead;
char [] zzBufferL = zzBuffer;
char [] zzCMapL = ZZ_CMAP;
int [] zzTransL = ZZ_TRANS;
int [] zzRowMapL = ZZ_ROWMAP;
int [] zzAttrL = ZZ_ATTRIBUTE;
while (true) {
zzMarkedPosL = zzMarkedPos;
zzAction = -1;
zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
zzState = ZZ_LEXSTATE[zzLexicalState];
zzForAction: {
while (true) {
if (zzCurrentPosL < zzEndReadL)
zzInput = zzBufferL[zzCurrentPosL++];
else if (zzAtEOF) {
zzInput = YYEOF;
break zzForAction;
}
else {
// store back cached positions
zzCurrentPos = zzCurrentPosL;
zzMarkedPos = zzMarkedPosL;
boolean eof = zzRefill();
// get translated positions and possibly new buffer
zzCurrentPosL = zzCurrentPos;
zzMarkedPosL = zzMarkedPos;
zzBufferL = zzBuffer;
zzEndReadL = zzEndRead;
if (eof) {
zzInput = YYEOF;
break zzForAction;
}
else {
zzInput = zzBufferL[zzCurrentPosL++];
}
}
int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
if (zzNext == -1) break zzForAction;
zzState = zzNext;
int zzAttributes = zzAttrL[zzState];
if ( (zzAttributes & 1) == 1 ) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
if ( (zzAttributes & 8) == 8 ) break zzForAction;
}
}
}
// store back cached position
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
case 6:
{ return getEllipsis();
}
case 9: break;
case 8:
{ if (removeProMarker) {
if ( ! removeSegMarker) {
return getNext("-", yytext());
}
} else {
return getNext(false);
}
}
case 10: break;
case 7:
{ if (! removeProMarker) return getNext(false);
}
case 11: break;
case 4:
{ return getNext(false);
}
case 12: break;
case 2:
{ return getNewline();
}
case 13: break;
case 5:
{ return getNext(true);
}
case 14: break;
case 3:
{
}
case 15: break;
case 1:
{ System.err.printf("Untokenizable: %s%n", yytext());
return getNext(true);
}
case 16: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
{
return null;
}
}
else {
zzScanError(ZZ_NO_MATCH);
}
}
}
}
}