/*
* Created on 28-Oct-2004
*/
package org.apache.lucene.search.highlight;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
/**
* Hides implementation issues associated with obtaining a TokenStream for use
* with the higlighter - can obtain from TermFreqVectors with offsets and
* (optionally) positions or from Analyzer class reparsing the stored content.
*/
public class TokenSources {
/**
* A convenience method that tries to first get a TermPositionVector for the
* specified docId, then, falls back to using the passed in
* {@link org.apache.lucene.document.Document} to retrieve the TokenStream.
* This is useful when you already have the document, but would prefer to use
* the vector first.
*
* @param reader The {@link org.apache.lucene.index.IndexReader} to use to try
* and get the vector from
* @param docId The docId to retrieve.
* @param field The field to retrieve on the document
* @param document The document to fall back on
* @param analyzer The analyzer to use for creating the TokenStream if the
* vector doesn't exist
* @return The {@link org.apache.lucene.analysis.TokenStream} for the
* {@link org.apache.lucene.index.IndexableField} on the
* {@link org.apache.lucene.document.Document}
* @throws IOException if there was an error loading
*/
public static TokenStream getAnyTokenStream(IndexReader reader, int docId,
String field, StoredDocument document, Analyzer analyzer) throws IOException {
TokenStream ts = null;
Fields vectors = reader.getTermVectors(docId);
if (vectors != null) {
Terms vector = vectors.terms(field);
if (vector != null) {
ts = getTokenStream(vector);
}
}
// No token info stored so fall back to analyzing raw content
if (ts == null) {
ts = getTokenStream(document, field, analyzer);
}
return ts;
}
/**
* A convenience method that tries a number of approaches to getting a token
* stream. The cost of finding there are no termVectors in the index is
* minimal (1000 invocations still registers 0 ms). So this "lazy" (flexible?)
* approach to coding is probably acceptable
*
* @return null if field not stored correctly
* @throws IOException If there is a low-level I/O error
*/
public static TokenStream getAnyTokenStream(IndexReader reader, int docId,
String field, Analyzer analyzer) throws IOException {
TokenStream ts = null;
Fields vectors = reader.getTermVectors(docId);
if (vectors != null) {
Terms vector = vectors.terms(field);
if (vector != null) {
ts = getTokenStream(vector);
}
}
// No token info stored so fall back to analyzing raw content
if (ts == null) {
ts = getTokenStream(reader, docId, field, analyzer);
}
return ts;
}
public static TokenStream getTokenStream(Terms vector) throws IOException {
// assumes the worst and makes no assumptions about token position
// sequences.
return getTokenStream(vector, false);
}
/**
* Low level api. Returns a token stream generated from a {@link Terms}. This
* can be used to feed the highlighter with a pre-parsed token
* stream. The {@link Terms} must have offsets available.
*
* In my tests the speeds to recreate 1000 token streams using this method
* are: - with TermVector offset only data stored - 420 milliseconds - with
* TermVector offset AND position data stored - 271 milliseconds (nb timings
* for TermVector with position data are based on a tokenizer with contiguous
* positions - no overlaps or gaps) The cost of not using TermPositionVector
* to store pre-parsed content and using an analyzer to re-parse the original
* content: - reanalyzing the original content - 980 milliseconds
*
* The re-analyze timings will typically vary depending on - 1) The complexity
* of the analyzer code (timings above were using a
* stemmer/lowercaser/stopword combo) 2) The number of other fields (Lucene
* reads ALL fields off the disk when accessing just one document field - can
* cost dear!) 3) Use of compression on field storage - could be faster due to
* compression (less disk IO) or slower (more CPU burn) depending on the
* content.
*
* @param tokenPositionsGuaranteedContiguous true if the token position
* numbers have no overlaps or gaps. If looking to eek out the last
* drops of performance, set to true. If in doubt, set to false.
*
* @throws IllegalArgumentException if no offsets are available
*/
public static TokenStream getTokenStream(Terms tpv,
boolean tokenPositionsGuaranteedContiguous)
throws IOException {
if (!tpv.hasOffsets()) {
throw new IllegalArgumentException("Cannot create TokenStream from Terms without offsets");
}
if (!tokenPositionsGuaranteedContiguous && tpv.hasPositions()) {
return new TokenStreamFromTermPositionVector(tpv);
}
// an object used to iterate across an array of tokens
final class StoredTokenStream extends TokenStream {
Token tokens[];
int currentToken = 0;
CharTermAttribute termAtt;
OffsetAttribute offsetAtt;
PositionIncrementAttribute posincAtt;
PayloadAttribute payloadAtt;
StoredTokenStream(Token tokens[]) {
this.tokens = tokens;
termAtt = addAttribute(CharTermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
posincAtt = addAttribute(PositionIncrementAttribute.class);
payloadAtt = addAttribute(PayloadAttribute.class);
}
@Override
public boolean incrementToken() {
if (currentToken >= tokens.length) {
return false;
}
Token token = tokens[currentToken++];
clearAttributes();
termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
BytesRef payload = token.getPayload();
if (payload != null) {
payloadAtt.setPayload(payload);
}
posincAtt
.setPositionIncrement(currentToken <= 1
|| tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
.startOffset() ? 1 : 0);
return true;
}
}
boolean hasPayloads = tpv.hasPayloads();
// code to reconstruct the original sequence of Tokens
TermsEnum termsEnum = tpv.iterator(null);
int totalTokens = 0;
while(termsEnum.next() != null) {
totalTokens += (int) termsEnum.totalTermFreq();
}
Token tokensInOriginalOrder[] = new Token[totalTokens];
ArrayList<Token> unsortedTokens = null;
termsEnum = tpv.iterator(null);
BytesRef text;
DocsAndPositionsEnum dpEnum = null;
while ((text = termsEnum.next()) != null) {
dpEnum = termsEnum.docsAndPositions(null, dpEnum);
if (dpEnum == null) {
throw new IllegalArgumentException(
"Required TermVector Offset information was not found");
}
final String term = text.utf8ToString();
dpEnum.nextDoc();
final int freq = dpEnum.freq();
for(int posUpto=0;posUpto<freq;posUpto++) {
final int pos = dpEnum.nextPosition();
if (dpEnum.startOffset() < 0) {
throw new IllegalArgumentException(
"Required TermVector Offset information was not found");
}
final Token token = new Token(term,
dpEnum.startOffset(),
dpEnum.endOffset());
if (hasPayloads) {
// Must make a deep copy of the returned payload,
// since D&PEnum API is allowed to re-use on every
// call:
token.setPayload(BytesRef.deepCopyOf(dpEnum.getPayload()));
}
if (tokenPositionsGuaranteedContiguous && pos != -1) {
// We have positions stored and a guarantee that the token position
// information is contiguous
// This may be fast BUT wont work if Tokenizers used which create >1
// token in same position or
// creates jumps in position numbers - this code would fail under those
// circumstances
// tokens stored with positions - can use this to index straight into
// sorted array
tokensInOriginalOrder[pos] = token;
} else {
// tokens NOT stored with positions or not guaranteed contiguous - must
// add to list and sort later
if (unsortedTokens == null) {
unsortedTokens = new ArrayList<>();
}
unsortedTokens.add(token);
}
}
}
// If the field has been stored without position data we must perform a sort
if (unsortedTokens != null) {
tokensInOriginalOrder = unsortedTokens.toArray(new Token[unsortedTokens
.size()]);
ArrayUtil.timSort(tokensInOriginalOrder, new Comparator<Token>() {
@Override
public int compare(Token t1, Token t2) {
if (t1.startOffset() == t2.startOffset()) {
return t1.endOffset() - t2.endOffset();
} else {
return t1.startOffset() - t2.startOffset();
}
}
});
}
return new StoredTokenStream(tokensInOriginalOrder);
}
/**
* Returns a {@link TokenStream} with positions and offsets constructed from
* field termvectors. If the field has no termvectors, or positions or offsets
* are not included in the termvector, return null.
* @param reader the {@link IndexReader} to retrieve term vectors from
* @param docId the document to retrieve termvectors for
* @param field the field to retrieve termvectors for
* @return a {@link TokenStream}, or null if positions and offsets are not available
* @throws IOException If there is a low-level I/O error
*/
public static TokenStream getTokenStreamWithOffsets(IndexReader reader, int docId,
String field) throws IOException {
Fields vectors = reader.getTermVectors(docId);
if (vectors == null) {
return null;
}
Terms vector = vectors.terms(field);
if (vector == null) {
return null;
}
if (!vector.hasPositions() || !vector.hasOffsets()) {
return null;
}
return getTokenStream(vector);
}
// convenience method
public static TokenStream getTokenStream(IndexReader reader, int docId,
String field, Analyzer analyzer) throws IOException {
StoredDocument doc = reader.document(docId);
return getTokenStream(doc, field, analyzer);
}
public static TokenStream getTokenStream(StoredDocument doc, String field,
Analyzer analyzer) {
String contents = doc.get(field);
if (contents == null) {
throw new IllegalArgumentException("Field " + field
+ " in document is not stored and cannot be analyzed");
}
return getTokenStream(field, contents, analyzer);
}
// convenience method
public static TokenStream getTokenStream(String field, String contents,
Analyzer analyzer) {
try {
return analyzer.tokenStream(field, contents);
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}