/*
* Copyright (c) 2003, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/**
* This is a tool to generate categoryNames and categoryMap which are used in
* CharSet.java.
*/
package build.tools.generatebreakiteratordata;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.StringTokenizer;
class CharacterCategory {
/**
* A list of Unicode category names.
*/
static final String[] categoryNames = {
"Ll", /* Letter, Lowercase */
"Lu", /* Letter, Uppercase */
"Lt", /* Letter, Titlecase */
"Lo", /* Letter, Other */
"Lm", /* Letter, Modifier */
"Nd", /* Number, Decimal Digit */
"Nl", /* Number, Letter */
"No", /* Number, Other */
"Ps", /* Punctuation, Open */
"Pe", /* Punctuation, Close */
"Pi", /* Punctuation, Initial quote */
"Pf", /* Punctuation, Final quote */
"Pd", /* Punctuation, Dash */
"Pc", /* Punctuation, Connector */
"Po", /* Punctuation, Other */
"Sc", /* Symbol, Currency */
"Sm", /* Symbol, Math */
"So", /* Symbol, Other */
"Mn", /* Mark, Non-Spacing */
"Mc", /* Mark, Spacing Combining */
"Me", /* Mark, Enclosing */
"Zl", /* Separator, Line */
"Zp", /* Separator, Paragraph */
"Zs", /* Separator, Space */
"Cc", /* Other, Control */
"Cf", /* Other, Format */
"--", /* Dummy, ignored */
// Don't add anything after the Dummy entry!!
};
/**
* A array of Unicode code points for each category.
*/
private static int[][] categoryMap;
/**
* Generates CategoryMap for GenerateBreakIteratorData.
*/
static void makeCategoryMap(String filename) {
/* Overwrite specfile name */
specfile = filename;
/* Generate data in current format (1.5.0) */
generateNewData();
/* Copy generated data to cateogyMap */
categoryMap = new int[categoryNames.length-1][];
for (int i = 0; i < categoryNames.length-1; i++) {
int len = newListCount[BMP][i] + newListCount[nonBMP][i];
categoryMap[i] = new int[len];
System.arraycopy(newList[i], 0, categoryMap[i], 0, len);
}
}
/**
* Returns categoryMap for the given category.
*/
static int[] getCategoryMap(int category) {
return categoryMap[category];
}
/**
* Only used for debugging and generating a test program.
*/
public static void main(String[] args) {
/* Parses command-line options */
processArgs(args);
/* Generates data in current format (1.5.0) */
generateNewData();
/*
* Generates data in older format (1.4.X and earlier) and creates
* the old CategoryMap if "oldFilename" is not null.
*/
if (!oldDatafile.equals("")) {
generateOldData();
generateOldDatafile();
}
/* Displays summary of generated data */
showSummary();
/*
* Generates a test program which compares the new data and the return
* values of Character.getType().
* and the old data and the new data.
*/
generateTestProgram();
}
/**
* Spec (Unicode data file)
*/
private static String specfile = "UnicodeData.txt";
/**
* Output directory
*/
private static String outputDir = "";
/**
* Old data filename
*/
private static String oldDatafile = "";
/**
* Parses the specified arguments and sets up the variables.
*/
private static void processArgs(String[] args) {
for (int i = 0; i < args.length; i++) {
String arg =args[i];
if (arg.equals("-spec")) {
specfile = args[++i];
} else if (arg.equals("-old")) {
oldDatafile = args[++i];
} else if (arg.equals("-o")) {
outputDir = args[++i];
} else {
System.err.println("Usage: java CharacterCategory [-spec specfile]");
System.exit(1);
}
}
}
/**
* Displays summary of generated data
*/
private static void showSummary() {
int oldSum = 0;
int newSum = 0;
int oldSuppSum = 0;
int newSuppSum = 0;
for (int i = 0; i < categoryNames.length-1; i++) {
int newNum = newListCount[BMP][i] + newListCount[nonBMP][i];
if (oldTotalCount[i] != newNum) {
System.err.println("Error: The number of generated data is different between the new approach and the old approach.");
}
if (oldListCount[SURROGATE][i] != newListCount[nonBMP][i]) {
System.err.println("Error: The number of generated supplementarycharacters is different between the new approach and the old approach.");
}
System.out.println(" " + categoryNames[i] + ": " +
oldTotalCount[i] +
"(" + oldListCount[BEFORE][i] +
" + " + oldListCount[SURROGATE][i] +
" + " + oldListCount[AFTER][i] + ")" +
" --- " + newNum +
"(" + newListCount[BMP][i] +
" + " + newListCount[nonBMP][i] + ")");
oldSum += oldListCount[BEFORE][i] * 2 +
oldListCount[SURROGATE][i] * 4 +
oldListCount[AFTER][i] * 2;
newSum += newNum * 4 ;
oldSuppSum += oldListCount[SURROGATE][i] * 4;
newSuppSum += newListCount[nonBMP][i] * 4;
}
System.out.println("\nTotal buffer sizes are:\n " +
oldSum + "bytes(Including " + oldSuppSum +
"bytes for supplementary characters)\n " +
newSum + "bytes(Including " + newSuppSum +
"bytes for supplementary characters)");
if (!ignoredOld.toString().equals(ignoredNew.toString())) {
System.err.println("Ignored categories: Error: List mismatch: " +
ignoredOld + " vs. " + ignoredNew);
} else {
System.out.println("\nIgnored categories: " + ignoredOld);
System.out.println("Please confirm that they aren't used in BreakIteratorRules.");
}
}
private static final int HighSurrogate_CodeUnit_Start = 0xD800;
private static final int LowSurrogate_CodeUnit_Start = 0xDC00;
private static final int Supplementary_CodePoint_Start = 0x10000;
private static StringBuffer ignoredOld = new StringBuffer();
private static int[] oldTotalCount = new int[categoryNames.length];
private static int[][] oldListCount = new int[3][categoryNames.length];
private static int[][] oldListLen = new int[3][categoryNames.length];
private static StringBuffer[][] oldList = new StringBuffer[3][categoryNames.length];
private static final int BEFORE = 0;
private static final int SURROGATE = 1;
private static final int AFTER = 2;
/**
* Makes CategoryMap in ordler format which had been used by JDK 1.4.X and
* earlier versions.
*/
private static void generateOldData() {
/* Initialize arrays. */
for (int i = 0; i<categoryNames.length; i++) {
for (int j = BEFORE; j <= AFTER; j++) {
oldListCount[j][i] = 0;
oldList[j][i] = new StringBuffer();
oldListLen[j][i] = 17;
}
}
storeOldData();
if (oldTotalCount[categoryNames.length-1] != 1) {
System.err.println("This should not happen. Unicode data which belongs to an undefined category exists");
System.exit(1);
}
}
private static void storeOldData() {
try {
FileReader fin = new FileReader(specfile);
BufferedReader bin = new BufferedReader(fin);
String prevCode = "????";
String line;
int prevIndex = categoryNames.length - 1;
int prevCodeValue = -1;
int curCodeValue = 0;
boolean setFirst = false;
while ((line = bin.readLine()) != null) {
if (line.length() == 0) {
continue;
}
StringTokenizer st = new StringTokenizer(line, ";");
String code = st.nextToken();
char c = code.charAt(0);
if (c == '#' || c == '/') {
continue;
}
int i = Integer.valueOf(code, 16).intValue();
String characterName = st.nextToken();
String category = st.nextToken();
int index;
for (index = 0; index < categoryNames.length; index++) {
if (category.equals(categoryNames[index])) {
break;
}
}
if (index != categoryNames.length) {
curCodeValue = Integer.parseInt(code, 16);
if (prevIndex != index) {
appendOldChar(prevIndex, prevCodeValue, prevCode);
appendOldChar(index, curCodeValue, code);
prevIndex = index;
} else if (prevCodeValue != curCodeValue - 1) {
if (setFirst && characterName.endsWith(" Last>")) {
setFirst = false;
} else {
appendOldChar(prevIndex, prevCodeValue, prevCode);
appendOldChar(index, curCodeValue, code);
}
}
prevCodeValue = curCodeValue;
prevCode = code;
if (characterName.endsWith(" First>")) {
setFirst = true;
}
} else {
if (ignoredOld.indexOf(category) == -1) {
ignoredOld.append(category);
ignoredOld.append(' ');
}
}
}
appendOldChar(prevIndex, prevCodeValue, prevCode);
bin.close();
fin.close();
}
catch (Exception e) {
throw new InternalError(e.toString());
}
}
private static void appendOldChar(int index, int code, String s) {
int range;
if (code < HighSurrogate_CodeUnit_Start) {
range = BEFORE;
} else if (code < Supplementary_CodePoint_Start) {
range = AFTER;
} else {
range = SURROGATE;
}
if (oldListLen[range][index] > 64) {
oldList[range][index].append("\"\n + \"");
oldListLen[range][index] = 19;
}
if (code == 0x22 || code == 0x5c) {
oldList[range][index].append('\\');
oldList[range][index].append((char)code);
oldListLen[range][index] += 2;
} else if (code > 0x20 && code < 0x7F) {
oldList[range][index].append((char)code);
oldListLen[range][index] ++;
} else {
if (range == SURROGATE) {// Need to convert code point to code unit
oldList[range][index].append(toCodeUnit(code));
oldListLen[range][index] += 12;
} else {
oldList[range][index].append("\\u");
oldList[range][index].append(s);
oldListLen[range][index] += 6;
}
}
oldListCount[range][index] ++;
oldTotalCount[index]++;
}
private static String toCodeUnit(int i) {
StringBuffer sb = new StringBuffer();
sb.append("\\u");
sb.append(Integer.toString((i - Supplementary_CodePoint_Start) / 0x400 + HighSurrogate_CodeUnit_Start, 16).toUpperCase());
sb.append("\\u");
sb.append(Integer.toString(i % 0x400 + LowSurrogate_CodeUnit_Start, 16).toUpperCase());
return sb.toString();
}
private static int toCodePoint(String s) {
char c1 = s.charAt(0);
if (s.length() == 1 || !Character.isHighSurrogate(c1)) {
return (int)c1;
} else {
char c2 = s.charAt(1);
if (s.length() != 2 || !Character.isLowSurrogate(c2)) {
return -1;
}
return Character.toCodePoint(c1, c2);
}
}
private static StringBuffer ignoredNew = new StringBuffer();
private static int[] newTotalCount = new int[categoryNames.length];
private static int[][] newListCount = new int[2][categoryNames.length];
private static int[][] newList = new int[categoryNames.length][];
private static final int BMP = 0;
private static final int nonBMP = 1;
/**
* Makes CategoryMap in newer format which is used by JDK 1.5.0.
*/
private static void generateNewData() {
/* Initialize arrays. */
for (int i = 0; i<categoryNames.length; i++) {
newList[i] = new int[10];
}
storeNewData();
if (newListCount[BMP][categoryNames.length-1] != 1) {
System.err.println("This should not happen. Unicode data which belongs to an undefined category exists");
System.exit(1);
}
}
private static void storeNewData() {
try {
FileReader fin = new FileReader(specfile);
BufferedReader bin = new BufferedReader(fin);
String line;
int prevIndex = categoryNames.length - 1;
int prevCodeValue = -1;
int curCodeValue = 0;
boolean setFirst = false;
while ((line = bin.readLine()) != null) {
if (line.length() == 0) {
continue;
}
StringTokenizer st = new StringTokenizer(line, ";");
String code = st.nextToken();
char c = code.charAt(0);
if (c == '#' || c == '/') {
continue;
}
int i = Integer.valueOf(code, 16).intValue();
String characterName = st.nextToken();
String category = st.nextToken();
int index;
for (index = 0; index < categoryNames.length; index++) {
if (category.equals(categoryNames[index])) {
break;
}
}
if (index != categoryNames.length) {
curCodeValue = Integer.parseInt(code, 16);
if (prevIndex == index) {
if (setFirst) {
if (characterName.endsWith(" Last>")) {
setFirst = false;
} else {
System.err.println("*** Error 1 at " + code);
}
} else {
if (characterName.endsWith(" First>")) {
setFirst = true;
} else if (characterName.endsWith(" Last>")) {
System.err.println("*** Error 2 at " + code);
} else {
if (prevCodeValue != curCodeValue - 1) {
appendNewChar(prevIndex, prevCodeValue);
appendNewChar(index, curCodeValue);
}
}
}
} else {
if (setFirst) {
System.err.println("*** Error 3 at " + code);
} else if (characterName.endsWith(" First>")) {
setFirst = true;
} else if (characterName.endsWith(" Last>")) {
System.err.println("*** Error 4 at " + code);
}
appendNewChar(prevIndex, prevCodeValue);
appendNewChar(index, curCodeValue);
prevIndex = index;
}
prevCodeValue = curCodeValue;
} else {
if (ignoredNew.indexOf(category) == -1) {
ignoredNew.append(category);
ignoredNew.append(' ');
}
}
}
appendNewChar(prevIndex, prevCodeValue);
bin.close();
fin.close();
}
catch (Exception e) {
System.err.println("Error occurred on accessing " + specfile);
e.printStackTrace();
System.exit(1);
}
}
private static void appendNewChar(int index, int code) {
int bufLen = newList[index].length;
if (newTotalCount[index] == bufLen) {
int[] tmpBuf = new int[bufLen + 10];
System.arraycopy(newList[index], 0, tmpBuf, 0, bufLen);
newList[index] = tmpBuf;
}
newList[index][newTotalCount[index]++] = code;
if (code < 0x10000) {
newListCount[BMP][index]++;
} else {
newListCount[nonBMP][index]++;
}
}
/* Generates the old CategoryMap. */
private static void generateOldDatafile() {
try {
FileWriter fout = new FileWriter(oldDatafile);
BufferedWriter bout = new BufferedWriter(fout);
bout.write("\n //\n // The following String[][] can be used in CharSet.java as is.\n //\n\n private static final String[][] categoryMap = {\n");
for (int i = 0; i < categoryNames.length - 1; i++) {
if (oldTotalCount[i] != 0) {
bout.write(" { \"" + categoryNames[i] + "\",");
/* 0x0000-0xD7FF */
if (oldListCount[BEFORE][i] != 0) {
bout.write(" \"");
bout.write(oldList[BEFORE][i].toString() + "\"\n");
}
/* 0xD800-0xFFFF */
if (oldListCount[AFTER][i] != 0) {
if (oldListCount[BEFORE][i] != 0) {
bout.write(" + \"");
} else {
bout.write(" \"");
}
bout.write(oldList[AFTER][i].toString() + "\"\n");
}
/* 0xD800DC00(0x10000)-0xDBFF0xDFFFF(0x10FFFF) */
if (oldListCount[SURROGATE][i] != 0) {
if (oldListCount[BEFORE][i] != 0 || oldListCount[AFTER][i] != 0) {
bout.write(" + \"");
} else {
bout.write(" \"");
}
bout.write(oldList[SURROGATE][i].toString() + "\"\n");
}
bout.write(" },\n");
}
}
bout.write(" };\n\n");
bout.close();
fout.close();
}
catch (Exception e) {
System.err.println("Error occurred on accessing " + oldDatafile);
e.printStackTrace();
System.exit(1);
}
System.out.println("\n" + oldDatafile + " has been generated.");
}
/**
* Test program to be generated
*/
private static final String outfile = "CharacterCategoryTest.java";
/*
* Generates a test program which compare the generated date (newer one)
* with the return values of Characger.getType().
*/
private static void generateTestProgram() {
try {
FileWriter fout = new FileWriter(outfile);
BufferedWriter bout = new BufferedWriter(fout);
bout.write(collationMethod);
bout.write("\n //\n // The following arrays can be used in CharSet.java as is.\n //\n\n");
bout.write(" private static final String[] categoryNames = {");
for (int i = 0; i < categoryNames.length - 1; i++) {
if (i % 10 == 0) {
bout.write("\n ");
}
bout.write("\"" + categoryNames[i] + "\", ");
}
bout.write("\n };\n\n");
bout.write(" private static final int[][] categoryMap = {\n");
for (int i = 0; i < categoryNames.length - 1; i++) {
StringBuffer sb = new StringBuffer(" { /* Data for \"" + categoryNames[i] + "\" category */");
for (int j = 0; j < newTotalCount[i]; j++) {
if (j % 8 == 0) {
sb.append("\n ");
}
sb.append(" 0x");
sb.append(Integer.toString(newList[i][j], 16).toUpperCase());
sb.append(',');
}
sb.append("\n },\n");
bout.write(sb.toString());
}
bout.write(" };\n");
bout.write("\n}\n");
bout.close();
fout.close();
}
catch (Exception e) {
System.err.println("Error occurred on accessing " + outfile);
e.printStackTrace();
System.exit(1);
}
System.out.println("\n" + outfile + " has been generated.");
}
static String collationMethod =
"public class CharacterCategoryTest {\n\n" +
" static final int SIZE = 0x110000;\n" +
" static final String[] category = {\n" +
" \"Cn\", \"Lu\", \"Ll\", \"Lt\", \"Lm\", \"Lo\", \"Mn\", \"Me\",\n" +
" \"Mc\", \"Nd\", \"Nl\", \"No\", \"Zs\", \"Zl\", \"Zp\", \"Cc\",\n" +
" \"Cf\", \"\", \"Co\", \"Cs\", \"Pd\", \"Ps\", \"Pe\", \"Pc\",\n" +
" \"Po\", \"Sm\", \"Sc\", \"Sk\", \"So\", \"Pi\", \"Pf\"\n" +
" };\n\n" +
" public static void main(String[] args) {\n" +
" boolean err = false;\n" +
" byte[] b = new byte[SIZE];\n" +
" for (int i = 0; i < SIZE; i++) {\n" +
" b[i] = 0;\n" +
" }\n" +
" for (int i = 0; i < categoryMap.length; i++) {\n" +
" byte categoryNum = 0;\n" +
" String categoryName = categoryNames[i];\n" +
" for (int j = 0; j < category.length; j++) {\n" +
" if (categoryName.equals(category[j])) {\n" +
" categoryNum = (byte)j;\n" +
" break;\n" +
" }\n" +
" }\n" +
" int[] values = categoryMap[i];\n" +
" for (int j = 0; j < values.length;) {\n" +
" int firstChar = values[j++];\n" +
" int lastChar = values[j++];\n" +
" for (int k = firstChar; k <= lastChar; k++) {\n" +
" b[k] = categoryNum;\n" +
" }\n" +
" }\n" +
" }\n" +
" for (int i = 0; i < SIZE; i++) {\n" +
" int characterType = Character.getType(i);\n" +
" if (b[i] != characterType) {\n" +
" /* Co, Cs and Sk categories are ignored in CharacterCategory. */\n" +
" if (characterType == Character.PRIVATE_USE ||\n" +
" characterType == Character.SURROGATE ||\n" +
" characterType == Character.MODIFIER_SYMBOL) {\n" +
" continue;\n" +
" }\n" +
" err = true;\n" +
" System.err.println(\"Category conflict for a character(0x\" +\n" +
" Integer.toHexString(i) +\n" +
" \"). CharSet.categoryMap:\" +\n" +
" category[b[i]] +\n" +
" \" Character.getType():\" +\n" +
" category[characterType]);\n" +
" }\n" +
" }\n\n" +
" if (err) {\n" +
" throw new RuntimeException(\"Conflict occurred between Charset.categoryMap and Character.getType()\");\n" +
" }\n" +
" }\n";
}