/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.function;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.WeakHashMap;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.Term;
import org.apache.lucene.util.StringHelper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.RequestHandlerUtils;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.QParser;
import org.apache.solr.search.SolrIndexReader;
import org.apache.solr.update.processor.UpdateRequestProcessor;
import org.apache.solr.util.VersionedFile;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Obtains float field values from an external file.
* @version $Id: FileFloatSource.java 1147288 2011-07-15 19:17:05Z hossman $
*/
public class FileFloatSource extends ValueSource {
private SchemaField field;
private final SchemaField keyField;
private final float defVal;
private final String dataDir;
public FileFloatSource(SchemaField field, SchemaField keyField, float defVal, QParser parser) {
this.field = field;
this.keyField = keyField;
this.defVal = defVal;
this.dataDir = parser.getReq().getCore().getDataDir();
}
@Override
public String description() {
return "float(" + field + ')';
}
@Override
public DocValues getValues(Map context, IndexReader reader) throws IOException {
int offset = 0;
if (reader instanceof SolrIndexReader) {
SolrIndexReader r = (SolrIndexReader)reader;
while (r.getParent() != null) {
offset += r.getBase();
r = r.getParent();
}
reader = r;
}
final int off = offset;
final float[] arr = getCachedFloats(reader);
return new DocValues() {
@Override
public float floatVal(int doc) {
return arr[doc + off];
}
@Override
public int intVal(int doc) {
return (int)arr[doc + off];
}
@Override
public long longVal(int doc) {
return (long)arr[doc + off];
}
@Override
public double doubleVal(int doc) {
return (double)arr[doc + off];
}
@Override
public String strVal(int doc) {
return Float.toString(arr[doc + off]);
}
@Override
public String toString(int doc) {
return description() + '=' + floatVal(doc);
}
};
}
@Override
public boolean equals(Object o) {
if (o.getClass() != FileFloatSource.class) return false;
FileFloatSource other = (FileFloatSource)o;
return this.field.getName().equals(other.field.getName())
&& this.keyField.getName().equals(other.keyField.getName())
&& this.defVal == other.defVal
&& this.dataDir.equals(other.dataDir);
}
@Override
public int hashCode() {
return FileFloatSource.class.hashCode() + field.getName().hashCode();
};
@Override
public String toString() {
return "FileFloatSource(field="+field.getName()+",keyField="+keyField.getName()
+ ",defVal="+defVal+",dataDir="+dataDir+")";
}
public static void resetCache(){
floatCache.resetCache();
}
private final float[] getCachedFloats(IndexReader reader) {
return (float[])floatCache.get(reader, new Entry(this));
}
static Cache floatCache = new Cache() {
@Override
protected Object createValue(IndexReader reader, Object key) {
return getFloats(((Entry)key).ffs, reader);
}
};
/** Internal cache. (from lucene FieldCache) */
abstract static class Cache {
private final Map readerCache = new WeakHashMap();
protected abstract Object createValue(IndexReader reader, Object key);
public Object get(IndexReader reader, Object key) {
Map innerCache;
Object value;
synchronized (readerCache) {
innerCache = (Map) readerCache.get(reader);
if (innerCache == null) {
innerCache = new HashMap();
readerCache.put(reader, innerCache);
value = null;
} else {
value = innerCache.get(key);
}
if (value == null) {
value = new CreationPlaceholder();
innerCache.put(key, value);
}
}
if (value instanceof CreationPlaceholder) {
synchronized (value) {
CreationPlaceholder progress = (CreationPlaceholder) value;
if (progress.value == null) {
progress.value = createValue(reader, key);
synchronized (readerCache) {
innerCache.put(key, progress.value);
onlyForTesting = progress.value;
}
}
return progress.value;
}
}
return value;
}
public void resetCache(){
synchronized(readerCache){
// Map.clear() is optional and can throw UnsipportedOperationException,
// but readerCache is WeakHashMap and it supports clear().
readerCache.clear();
}
}
}
static Object onlyForTesting; // set to the last value
static final class CreationPlaceholder {
Object value;
}
/** Expert: Every composite-key in the internal cache is of this type. */
private static class Entry {
final FileFloatSource ffs;
public Entry(FileFloatSource ffs) {
this.ffs = ffs;
}
@Override
public boolean equals(Object o) {
if (!(o instanceof Entry)) return false;
Entry other = (Entry)o;
return ffs.equals(other.ffs);
}
@Override
public int hashCode() {
return ffs.hashCode();
}
}
private static float[] getFloats(FileFloatSource ffs, IndexReader reader) {
float[] vals = new float[reader.maxDoc()];
if (ffs.defVal != 0) {
Arrays.fill(vals, ffs.defVal);
}
InputStream is;
String fname = "external_" + ffs.field.getName();
try {
is = VersionedFile.getLatestFile(ffs.dataDir, fname);
} catch (IOException e) {
// log, use defaults
SolrCore.log.error("Error opening external value source file: " +e);
return vals;
}
BufferedReader r = new BufferedReader(new InputStreamReader(is));
String idName = StringHelper.intern(ffs.keyField.getName());
FieldType idType = ffs.keyField.getType();
boolean sorted=true; // assume sorted until we discover it's not
// warning: lucene's termEnum.skipTo() is not optimized... it simply does a next()
// because of this, simply ask the reader for a new termEnum rather than
// trying to use skipTo()
List<String> notFound = new ArrayList<String>();
int notFoundCount=0;
int otherErrors=0;
TermDocs termDocs = null;
Term protoTerm = new Term(idName, "");
TermEnum termEnum = null;
// Number of times to try termEnum.next() before resorting to skip
int numTimesNext = 10;
char delimiter='=';
String termVal;
boolean hasNext=true;
String prevKey="";
String lastVal="\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF\uFFFF";
try {
termDocs = reader.termDocs();
termEnum = reader.terms(protoTerm);
Term t = termEnum.term();
if (t != null && t.field() == idName) { // intern'd comparison
termVal = t.text();
} else {
termVal = lastVal;
}
for (String line; (line=r.readLine())!=null;) {
int delimIndex = line.lastIndexOf(delimiter);
if (delimIndex < 0) continue;
int endIndex = line.length();
/* EOLs should already be removed for BufferedReader.readLine()
for(int endIndex = line.length();endIndex>delimIndex+1; endIndex--) {
char ch = line.charAt(endIndex-1);
if (ch!='\n' && ch!='\r') break;
}
*/
String key = line.substring(0, delimIndex);
String val = line.substring(delimIndex+1, endIndex);
String internalKey = idType.toInternal(key);
float fval;
try {
fval=Float.parseFloat(val);
} catch (Exception e) {
if (++otherErrors<=10) {
SolrCore.log.error( "Error loading external value source + fileName + " + e
+ (otherErrors<10 ? "" : "\tSkipping future errors for this file.")
);
}
continue; // go to next line in file.. leave values as default.
}
if (sorted) {
// make sure this key is greater than the previous key
sorted = internalKey.compareTo(prevKey) >= 0;
prevKey = internalKey;
if (sorted) {
int countNext = 0;
for(;;) {
int cmp = internalKey.compareTo(termVal);
if (cmp == 0) {
termDocs.seek(termEnum);
while (termDocs.next()) {
vals[termDocs.doc()] = fval;
}
break;
} else if (cmp < 0) {
// term enum has already advanced past current key... we didn't find it.
if (notFoundCount<10) { // collect first 10 not found for logging
notFound.add(key);
}
notFoundCount++;
break;
} else {
// termEnum is less than our current key, so skip ahead
// try next() a few times to see if we hit or pass the target.
// Lucene's termEnum.skipTo() is currently unoptimized (it just does next())
// so the best thing is to simply ask the reader for a new termEnum(target)
// if we really need to skip.
if (++countNext > numTimesNext) {
termEnum = reader.terms(protoTerm.createTerm(internalKey));
t = termEnum.term();
} else {
hasNext = termEnum.next();
t = hasNext ? termEnum.term() : null;
}
if (t != null && t.field() == idName) { // intern'd comparison
termVal = t.text();
} else {
termVal = lastVal;
}
}
} // end for(;;)
}
}
if (!sorted) {
termEnum = reader.terms(protoTerm.createTerm(internalKey));
t = termEnum.term();
if (t != null && t.field() == idName // intern'd comparison
&& internalKey.equals(t.text()))
{
termDocs.seek (termEnum);
while (termDocs.next()) {
vals[termDocs.doc()] = fval;
}
} else {
if (notFoundCount<10) { // collect first 10 not found for logging
notFound.add(key);
}
notFoundCount++;
}
}
}
} catch (IOException e) {
// log, use defaults
SolrCore.log.error("Error loading external value source: " +e);
} finally {
// swallow exceptions on close so we don't override any
// exceptions that happened in the loop
if (termDocs!=null) try{termDocs.close();}catch(Exception e){}
if (termEnum!=null) try{termEnum.close();}catch(Exception e){}
try{r.close();}catch(Exception e){}
}
SolrCore.log.info("Loaded external value source " + fname
+ (notFoundCount==0 ? "" : " :"+notFoundCount+" missing keys "+notFound)
);
return vals;
}
public static class ReloadCacheRequestHandler extends RequestHandlerBase {
static final Logger log = LoggerFactory.getLogger(ReloadCacheRequestHandler.class);
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
FileFloatSource.resetCache();
log.debug("readerCache has been reset.");
UpdateRequestProcessor processor =
req.getCore().getUpdateProcessingChain(null).createProcessor(req, rsp);
try{
RequestHandlerUtils.handleCommit(processor, req.getParams(), true);
}
finally{
processor.finish();
}
}
@Override
public String getDescription() {
return "Reload readerCache request handler";
}
@Override
public String getSource() {
return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene_solr_3_5/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java $";
}
@Override
public String getSourceId() {
return "$Id: FileFloatSource.java 1147288 2011-07-15 19:17:05Z hossman $";
}
@Override
public String getVersion() {
return "$Revision: 1147288 $";
}
}
}