/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.analysis.cn.smart.hhmm;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
import org.apache.lucene.analysis.cn.smart.Utility;
/**
* SmartChineseAnalyzer Word Dictionary
* @lucene.experimental
*/
class WordDictionary extends AbstractDictionary {
private WordDictionary() {
}
private static WordDictionary singleInstance;
/**
* Large prime number for hash function
*/
public static final int PRIME_INDEX_LENGTH = 12071;
/**
* wordIndexTable guarantees to hash all Chinese characters in Unicode into
* PRIME_INDEX_LENGTH array. There will be conflict, but in reality this
* program only handles the 6768 characters found in GB2312 plus some
* ASCII characters. Therefore in order to guarantee better precision, it is
* necessary to retain the original symbol in the charIndexTable.
*/
private short[] wordIndexTable;
private char[] charIndexTable;
/**
* To avoid taking too much space, the data structure needed to store the
* lexicon requires two multidimensional arrays to store word and frequency.
* Each word is placed in a char[]. Each char represents a Chinese char or
* other symbol. Each frequency is put into an int. These two arrays
* correspond to each other one-to-one. Therefore, one can use
* wordItem_charArrayTable[i][j] to look up word from lexicon, and
* wordItem_frequencyTable[i][j] to look up the corresponding frequency.
*/
private char[][][] wordItem_charArrayTable;
private int[][] wordItem_frequencyTable;
// static Logger log = Logger.getLogger(WordDictionary.class);
/**
* Get the singleton dictionary instance.
* @return singleton
*/
public synchronized static WordDictionary getInstance() {
if (singleInstance == null) {
singleInstance = new WordDictionary();
try {
singleInstance.load();
} catch (IOException e) {
String wordDictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
singleInstance.load(wordDictRoot);
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
}
return singleInstance;
}
/**
* Attempt to load dictionary from provided directory, first trying coredict.mem, failing back on coredict.dct
*
* @param dctFileRoot path to dictionary directory
*/
public void load(String dctFileRoot) {
String dctFilePath = dctFileRoot + "/coredict.dct";
File serialObj = new File(dctFileRoot + "/coredict.mem");
if (serialObj.exists() && loadFromObj(serialObj)) {
} else {
try {
wordIndexTable = new short[PRIME_INDEX_LENGTH];
charIndexTable = new char[PRIME_INDEX_LENGTH];
for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
charIndexTable[i] = 0;
wordIndexTable[i] = -1;
}
wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
// int total =
loadMainDataFromFile(dctFilePath);
expandDelimiterData();
mergeSameWords();
sortEachItems();
// log.info("load dictionary: " + dctFilePath + " total:" + total);
} catch (IOException e) {
throw new RuntimeException(e.getMessage());
}
saveToObj(serialObj);
}
}
/**
* Load coredict.mem internally from the jar file.
*
* @throws ClassNotFoundException
* @throws IOException
*/
public void load() throws IOException, ClassNotFoundException {
InputStream input = this.getClass().getResourceAsStream("coredict.mem");
loadFromObjectInputStream(input);
}
private boolean loadFromObj(File serialObj) {
try {
loadFromObjectInputStream(new FileInputStream(serialObj));
return true;
} catch (FileNotFoundException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
return false;
}
private void loadFromObjectInputStream(InputStream serialObjectInputStream)
throws IOException, ClassNotFoundException {
ObjectInputStream input = new ObjectInputStream(serialObjectInputStream);
wordIndexTable = (short[]) input.readObject();
charIndexTable = (char[]) input.readObject();
wordItem_charArrayTable = (char[][][]) input.readObject();
wordItem_frequencyTable = (int[][]) input.readObject();
// log.info("load core dict from serialization.");
input.close();
}
private void saveToObj(File serialObj) {
try {
ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(
serialObj));
output.writeObject(wordIndexTable);
output.writeObject(charIndexTable);
output.writeObject(wordItem_charArrayTable);
output.writeObject(wordItem_frequencyTable);
output.close();
// log.info("serialize core dict.");
} catch (Exception e) {
// log.warn(e.getMessage());
}
}
/**
* Load the datafile into this WordDictionary
*
* @param dctFilePath path to word dictionary (coredict.dct)
* @return number of words read
* @throws FileNotFoundException
* @throws IOException
* @throws UnsupportedEncodingException
*/
private int loadMainDataFromFile(String dctFilePath)
throws FileNotFoundException, IOException, UnsupportedEncodingException {
int i, cnt, length, total = 0;
// The file only counted 6763 Chinese characters plus 5 reserved slots 3756~3760.
// The 3756th is used (as a header) to store information.
int[] buffer = new int[3];
byte[] intBuffer = new byte[4];
String tmpword;
RandomAccessFile dctFile = new RandomAccessFile(dctFilePath, "r");
// GB2312 characters 0 - 6768
for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
// if (i == 5231)
// System.out.println(i);
dctFile.read(intBuffer);
// the dictionary was developed for C, and byte order must be converted to work with Java
cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
if (cnt <= 0) {
wordItem_charArrayTable[i] = null;
wordItem_frequencyTable[i] = null;
continue;
}
wordItem_charArrayTable[i] = new char[cnt][];
wordItem_frequencyTable[i] = new int[cnt];
total += cnt;
int j = 0;
while (j < cnt) {
// wordItemTable[i][j] = new WordItem();
dctFile.read(intBuffer);
buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
.getInt();// frequency
dctFile.read(intBuffer);
buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
.getInt();// length
dctFile.read(intBuffer);
buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN)
.getInt();// handle
// wordItemTable[i][j].frequency = buffer[0];
wordItem_frequencyTable[i][j] = buffer[0];
length = buffer[1];
if (length > 0) {
byte[] lchBuffer = new byte[length];
dctFile.read(lchBuffer);
tmpword = new String(lchBuffer, "GB2312");
// indexTable[i].wordItems[j].word = tmpword;
// wordItemTable[i][j].charArray = tmpword.toCharArray();
wordItem_charArrayTable[i][j] = tmpword.toCharArray();
} else {
// wordItemTable[i][j].charArray = null;
wordItem_charArrayTable[i][j] = null;
}
// System.out.println(indexTable[i].wordItems[j]);
j++;
}
String str = getCCByGB2312Id(i);
setTableIndex(str.charAt(0), i);
}
dctFile.close();
return total;
}
/**
* The original lexicon puts all information with punctuation into a
* chart (from 1 to 3755). Here it then gets expanded, separately being
* placed into the chart that has the corresponding symbol.
*/
private void expandDelimiterData() {
int i;
int cnt;
// Punctuation then treating index 3755 as 1,
// distribute the original punctuation corresponding dictionary into
int delimiterIndex = 3755 + GB2312_FIRST_CHAR;
i = 0;
while (i < wordItem_charArrayTable[delimiterIndex].length) {
char c = wordItem_charArrayTable[delimiterIndex][i][0];
int j = getGB2312Id(c);// the id value of the punctuation
if (wordItem_charArrayTable[j] == null) {
int k = i;
// Starting from i, count the number of the following worditem symbol from j
while (k < wordItem_charArrayTable[delimiterIndex].length
&& wordItem_charArrayTable[delimiterIndex][k][0] == c) {
k++;
}
// c is the punctuation character, j is the id value of c
// k-1 represents the index of the last punctuation character
cnt = k - i;
if (cnt != 0) {
wordItem_charArrayTable[j] = new char[cnt][];
wordItem_frequencyTable[j] = new int[cnt];
}
// Assign value for each wordItem.
for (k = 0; k < cnt; k++, i++) {
// wordItemTable[j][k] = new WordItem();
wordItem_frequencyTable[j][k] = wordItem_frequencyTable[delimiterIndex][i];
wordItem_charArrayTable[j][k] = new char[wordItem_charArrayTable[delimiterIndex][i].length - 1];
System.arraycopy(wordItem_charArrayTable[delimiterIndex][i], 1,
wordItem_charArrayTable[j][k], 0,
wordItem_charArrayTable[j][k].length);
}
setTableIndex(c, j);
}
}
// Delete the original corresponding symbol array.
wordItem_charArrayTable[delimiterIndex] = null;
wordItem_frequencyTable[delimiterIndex] = null;
}
/*
* since we aren't doing POS-tagging, merge the frequencies for entries of the same word (with different POS)
*/
private void mergeSameWords() {
int i;
for (i = 0; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
if (wordItem_charArrayTable[i] == null)
continue;
int len = 1;
for (int j = 1; j < wordItem_charArrayTable[i].length; j++) {
if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
wordItem_charArrayTable[i][j - 1], 0) != 0)
len++;
}
if (len < wordItem_charArrayTable[i].length) {
char[][] tempArray = new char[len][];
int[] tempFreq = new int[len];
int k = 0;
tempArray[0] = wordItem_charArrayTable[i][0];
tempFreq[0] = wordItem_frequencyTable[i][0];
for (int j = 1; j < wordItem_charArrayTable[i].length; j++) {
if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
tempArray[k], 0) != 0) {
k++;
// temp[k] = wordItemTable[i][j];
tempArray[k] = wordItem_charArrayTable[i][j];
tempFreq[k] = wordItem_frequencyTable[i][j];
} else {
// temp[k].frequency += wordItemTable[i][j].frequency;
tempFreq[k] += wordItem_frequencyTable[i][j];
}
}
// wordItemTable[i] = temp;
wordItem_charArrayTable[i] = tempArray;
wordItem_frequencyTable[i] = tempFreq;
}
}
}
private void sortEachItems() {
char[] tmpArray;
int tmpFreq;
for (int i = 0; i < wordItem_charArrayTable.length; i++) {
if (wordItem_charArrayTable[i] != null
&& wordItem_charArrayTable[i].length > 1) {
for (int j = 0; j < wordItem_charArrayTable[i].length - 1; j++) {
for (int j2 = j + 1; j2 < wordItem_charArrayTable[i].length; j2++) {
if (Utility.compareArray(wordItem_charArrayTable[i][j], 0,
wordItem_charArrayTable[i][j2], 0) > 0) {
tmpArray = wordItem_charArrayTable[i][j];
tmpFreq = wordItem_frequencyTable[i][j];
wordItem_charArrayTable[i][j] = wordItem_charArrayTable[i][j2];
wordItem_frequencyTable[i][j] = wordItem_frequencyTable[i][j2];
wordItem_charArrayTable[i][j2] = tmpArray;
wordItem_frequencyTable[i][j2] = tmpFreq;
}
}
}
}
}
}
/*
* Calculate character c's position in hash table,
* then initialize the value of that position in the address table.
*/
private boolean setTableIndex(char c, int j) {
int index = getAvaliableTableIndex(c);
if (index != -1) {
charIndexTable[index] = c;
wordIndexTable[index] = (short) j;
return true;
} else
return false;
}
private short getAvaliableTableIndex(char c) {
int hash1 = (int) (hash1(c) % PRIME_INDEX_LENGTH);
int hash2 = hash2(c) % PRIME_INDEX_LENGTH;
if (hash1 < 0)
hash1 = PRIME_INDEX_LENGTH + hash1;
if (hash2 < 0)
hash2 = PRIME_INDEX_LENGTH + hash2;
int index = hash1;
int i = 1;
while (charIndexTable[index] != 0 && charIndexTable[index] != c
&& i < PRIME_INDEX_LENGTH) {
index = (hash1 + i * hash2) % PRIME_INDEX_LENGTH;
i++;
}
// System.out.println(i - 1);
if (i < PRIME_INDEX_LENGTH
&& (charIndexTable[index] == 0 || charIndexTable[index] == c)) {
return (short) index;
} else
return -1;
}
private short getWordItemTableIndex(char c) {
int hash1 = (int) (hash1(c) % PRIME_INDEX_LENGTH);
int hash2 = hash2(c) % PRIME_INDEX_LENGTH;
if (hash1 < 0)
hash1 = PRIME_INDEX_LENGTH + hash1;
if (hash2 < 0)
hash2 = PRIME_INDEX_LENGTH + hash2;
int index = hash1;
int i = 1;
while (charIndexTable[index] != 0 && charIndexTable[index] != c
&& i < PRIME_INDEX_LENGTH) {
index = (hash1 + i * hash2) % PRIME_INDEX_LENGTH;
i++;
}
if (i < PRIME_INDEX_LENGTH && charIndexTable[index] == c) {
return (short) index;
} else
return -1;
}
/**
* Look up the text string corresponding with the word char array,
* and return the position of the word list.
*
* @param knownHashIndex already figure out position of the first word
* symbol charArray[0] in hash table. If not calculated yet, can be
* replaced with function int findInTable(char[] charArray).
* @param charArray look up the char array corresponding with the word.
* @return word location in word array. If not found, then return -1.
*/
private int findInTable(short knownHashIndex, char[] charArray) {
if (charArray == null || charArray.length == 0)
return -1;
char[][] items = wordItem_charArrayTable[wordIndexTable[knownHashIndex]];
int start = 0, end = items.length - 1;
int mid = (start + end) / 2, cmpResult;
// Binary search for the index of idArray
while (start <= end) {
cmpResult = Utility.compareArray(items[mid], 0, charArray, 1);
if (cmpResult == 0)
return mid;// find it
else if (cmpResult < 0)
start = mid + 1;
else if (cmpResult > 0)
end = mid - 1;
mid = (start + end) / 2;
}
return -1;
}
/**
* Find the first word in the dictionary that starts with the supplied prefix
*
* @see #getPrefixMatch(char[], int)
* @param charArray input prefix
* @return index of word, or -1 if not found
*/
public int getPrefixMatch(char[] charArray) {
return getPrefixMatch(charArray, 0);
}
/**
* Find the nth word in the dictionary that starts with the supplied prefix
*
* @see #getPrefixMatch(char[])
* @param charArray input prefix
* @param knownStart relative position in the dictionary to start
* @return index of word, or -1 if not found
*/
public int getPrefixMatch(char[] charArray, int knownStart) {
short index = getWordItemTableIndex(charArray[0]);
if (index == -1)
return -1;
char[][] items = wordItem_charArrayTable[wordIndexTable[index]];
int start = knownStart, end = items.length - 1;
int mid = (start + end) / 2, cmpResult;
// Binary search for the index of idArray
while (start <= end) {
cmpResult = Utility.compareArrayByPrefix(charArray, 1, items[mid], 0);
if (cmpResult == 0) {
// Get the first item which match the current word
while (mid >= 0
&& Utility.compareArrayByPrefix(charArray, 1, items[mid], 0) == 0)
mid--;
mid++;
return mid;// Find the first word that uses charArray as prefix.
} else if (cmpResult < 0)
end = mid - 1;
else
start = mid + 1;
mid = (start + end) / 2;
}
return -1;
}
/**
* Get the frequency of a word from the dictionary
*
* @param charArray input word
* @return word frequency, or zero if the word is not found
*/
public int getFrequency(char[] charArray) {
short hashIndex = getWordItemTableIndex(charArray[0]);
if (hashIndex == -1)
return 0;
int itemIndex = findInTable(hashIndex, charArray);
if (itemIndex != -1)
return wordItem_frequencyTable[wordIndexTable[hashIndex]][itemIndex];
return 0;
}
/**
* Return true if the dictionary entry at itemIndex for table charArray[0] is charArray
*
* @param charArray input word
* @param itemIndex item index for table charArray[0]
* @return true if the entry exists
*/
public boolean isEqual(char[] charArray, int itemIndex) {
short hashIndex = getWordItemTableIndex(charArray[0]);
return Utility.compareArray(charArray, 1,
wordItem_charArrayTable[wordIndexTable[hashIndex]][itemIndex], 0) == 0;
}
}