/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler;
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Date;
import java.util.List;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitSetIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LongValues;
import org.apache.solr.client.solrj.impl.BinaryResponseParser;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.MapWriter.EntryWriter;
import org.apache.solr.common.PushWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.QueryResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.BoolField;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import org.apache.solr.schema.TrieDateField;
import org.apache.solr.schema.TrieDoubleField;
import org.apache.solr.schema.TrieFloatField;
import org.apache.solr.schema.TrieIntField;
import org.apache.solr.schema.TrieLongField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SortSpec;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.apache.solr.common.util.Utils.makeMap;
public class ExportWriter implements SolrCore.RawWriter, Closeable {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private OutputStreamWriter respWriter;
final SolrQueryRequest req;
final SolrQueryResponse res;
FieldWriter[] fieldWriters;
int totalHits = 0;
FixedBitSet[] sets = null;
PushWriter writer;
private String wt;
ExportWriter(SolrQueryRequest req, SolrQueryResponse res, String wt) {
this.req = req;
this.res = res;
this.wt = wt;
}
@Override
public String getContentType() {
if ("javabin".equals(wt)) {
return BinaryResponseParser.BINARY_CONTENT_TYPE;
} else return "json";
}
@Override
public void close() throws IOException {
if (writer != null) writer.close();
if (respWriter != null) {
respWriter.flush();
respWriter.close();
}
}
protected void writeException(Exception e, PushWriter w, boolean log) throws IOException {
w.writeMap(mw -> {
mw.put("responseHeader", singletonMap("status", 400))
.put("response", makeMap(
"numFound", 0,
"docs", singletonList(singletonMap("EXCEPTION", e.getMessage()))));
});
if (log) {
SolrException.log(logger, e);
}
}
public void write(OutputStream os) throws IOException {
QueryResponseWriter rw = req.getCore().getResponseWriters().get(wt);
if (rw instanceof BinaryResponseWriter) {
//todo add support for other writers after testing
writer = new JavaBinCodec(os, null);
} else {
respWriter = new OutputStreamWriter(os, StandardCharsets.UTF_8);
writer = JSONResponseWriter.getPushWriter(respWriter, req, res);
}
Exception exception = res.getException();
if (exception != null) {
if (!(exception instanceof IgnoreException)) {
writeException(exception, writer, false);
}
return;
}
SolrRequestInfo info = SolrRequestInfo.getRequestInfo();
SortSpec sortSpec = info.getResponseBuilder().getSortSpec();
if(sortSpec == null) {
writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
return;
}
SolrIndexSearcher searcher = req.getSearcher();
Sort sort = searcher.weightSort(sortSpec.getSort());
if(sort == null) {
writeException((new IOException(new SyntaxError("No sort criteria was provided."))), writer, true);
return;
}
if(sort != null && sort.needsScores()) {
writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
return;
}
// There is a bailout in SolrIndexSearcher.getDocListNC when there are _no_ docs in the index at all.
// if (lastDocRequested <= 0) {
// That causes the totalHits and export entries in the context to _not_ get set.
// The only time that really matters is when we search against an _empty_ set. That's too obscure
// a condition to handle as part of this patch, if someone wants to pursue it it can be reproduced with:
// ant test -Dtestcase=StreamingTest -Dtests.method=testAllValidExportTypes -Dtests.seed=10F13879D0D1D6AD -Dtests.slow=true -Dtests.locale=es-PA -Dtests.timezone=America/Bahia_Banderas -Dtests.asserts=true -Dtests.file.encoding=ISO-8859-1
// You'll have to uncomment the if below to hit the null pointer exception.
// This is such an unusual case (i.e. an empty index) that catching this concdition here is probably OK.
// This came to light in the very artifical case of indexing a single doc to Cloud.
if (req.getContext().get("totalHits") != null) {
totalHits = ((Integer)req.getContext().get("totalHits")).intValue();
sets = (FixedBitSet[]) req.getContext().get("export");
if (sets == null) {
writeException((new IOException(new SyntaxError("xport RankQuery is required for xsort: rq={!xport}"))), writer, true);
return;
}
}
SolrParams params = req.getParams();
String fl = params.get("fl");
String[] fields = null;
if(fl == null) {
writeException((new IOException(new SyntaxError("export field list (fl) must be specified."))), writer, true);
return;
} else {
fields = fl.split(",");
for(int i=0;i<fields.length; i++) {
fields[i] = fields[i].trim();
if(fields[i].equals("score")) {
writeException((new IOException(new SyntaxError("Scoring is not currently supported with xsort."))), writer, true);
return;
}
}
}
try {
fieldWriters = getFieldWriters(fields, req.getSearcher());
} catch (Exception e) {
writeException(e, writer, true);
return;
}
writer.writeMap(m -> {
m.put("responseHeader", singletonMap("status", 0));
m.put("response", (MapWriter) mw -> {
mw.put("numFound", totalHits);
mw.put("docs", (IteratorWriter) iw -> writeDocs(req, iw, sort));
});
});
}
protected void writeDocs(SolrQueryRequest req, IteratorWriter.ItemWriter writer, Sort sort) throws IOException {
//Write the data.
List<LeafReaderContext> leaves = req.getSearcher().getTopReaderContext().leaves();
SortDoc sortDoc = getSortDoc(req.getSearcher(), sort.getSort());
int count = 0;
int queueSize = 30000;
SortQueue queue = new SortQueue(queueSize, sortDoc);
SortDoc[] outDocs = new SortDoc[queueSize];
while(count < totalHits) {
//long begin = System.nanoTime();
queue.reset();
SortDoc top = queue.top();
for(int i=0; i<leaves.size(); i++) {
sortDoc.setNextReader(leaves.get(i));
DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
int docId = -1;
while((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
sortDoc.setValues(docId);
if(top.lessThan(sortDoc)) {
top.setValues(sortDoc);
top = queue.updateTop();
}
}
}
int outDocsIndex = -1;
for(int i=0; i<queueSize; i++) {
SortDoc s = queue.pop();
if(s.docId > -1) {
outDocs[++outDocsIndex] = s;
}
}
//long end = System.nanoTime();
count += (outDocsIndex+1);
try {
for(int i=outDocsIndex; i>=0; --i) {
SortDoc s = outDocs[i];
writer.add((MapWriter) ew -> {
writeDoc(s, leaves, ew);
s.reset();
});
}
} catch(Throwable e) {
Throwable ex = e;
e.printStackTrace();
while(ex != null) {
String m = ex.getMessage();
if(m != null && m.contains("Broken pipe")) {
throw new IgnoreException();
}
ex = ex.getCause();
}
if(e instanceof IOException) {
throw ((IOException)e);
} else {
throw new IOException(e);
}
}
}
}
protected void writeDoc(SortDoc sortDoc,
List<LeafReaderContext> leaves,
EntryWriter ew) throws IOException {
int ord = sortDoc.ord;
FixedBitSet set = sets[ord];
set.clear(sortDoc.docId);
LeafReaderContext context = leaves.get(ord);
int fieldIndex = 0;
for (FieldWriter fieldWriter : fieldWriters) {
if (fieldWriter.write(sortDoc.docId, context.reader(), ew, fieldIndex)) {
++fieldIndex;
}
}
}
protected FieldWriter[] getFieldWriters(String[] fields, SolrIndexSearcher searcher) throws IOException {
IndexSchema schema = searcher.getSchema();
FieldWriter[] writers = new FieldWriter[fields.length];
for(int i=0; i<fields.length; i++) {
String field = fields[i];
SchemaField schemaField = null;
try {
schemaField = schema.getField(field);
} catch (Exception e) {
throw new IOException(e);
}
if(!schemaField.hasDocValues()) {
throw new IOException(field+" must have DocValues to use this feature.");
}
boolean multiValued = schemaField.multiValued();
FieldType fieldType = schemaField.getType();
if (fieldType instanceof TrieIntField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new IntFieldWriter(field);
}
} else if (fieldType instanceof TrieLongField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new LongFieldWriter(field);
}
} else if (fieldType instanceof TrieFloatField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new FloatFieldWriter(field);
}
} else if (fieldType instanceof TrieDoubleField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new DoubleFieldWriter(field);
}
} else if (fieldType instanceof StrField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
} else {
writers[i] = new StringFieldWriter(field, fieldType);
}
} else if (fieldType instanceof TrieDateField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, false);
} else {
writers[i] = new DateFieldWriter(field);
}
} else if (fieldType instanceof BoolField) {
if (multiValued) {
writers[i] = new MultiFieldWriter(field, fieldType, schemaField, true);
} else {
writers[i] = new BoolFieldWriter(field, fieldType);
}
} else {
throw new IOException("Export fields must either be one of the following types: int,float,long,double,string,date,boolean");
}
}
return writers;
}
private SortDoc getSortDoc(SolrIndexSearcher searcher, SortField[] sortFields) throws IOException {
SortValue[] sortValues = new SortValue[sortFields.length];
IndexSchema schema = searcher.getSchema();
for(int i=0; i<sortFields.length; ++i) {
SortField sf = sortFields[i];
String field = sf.getField();
boolean reverse = sf.getReverse();
SchemaField schemaField = schema.getField(field);
FieldType ft = schemaField.getType();
if(!schemaField.hasDocValues()) {
throw new IOException(field+" must have DocValues to use this feature.");
}
if(ft instanceof TrieIntField) {
if(reverse) {
sortValues[i] = new IntValue(field, new IntDesc());
} else {
sortValues[i] = new IntValue(field, new IntAsc());
}
} else if(ft instanceof TrieFloatField) {
if(reverse) {
sortValues[i] = new FloatValue(field, new FloatDesc());
} else {
sortValues[i] = new FloatValue(field, new FloatAsc());
}
} else if(ft instanceof TrieDoubleField) {
if(reverse) {
sortValues[i] = new DoubleValue(field, new DoubleDesc());
} else {
sortValues[i] = new DoubleValue(field, new DoubleAsc());
}
} else if(ft instanceof TrieLongField) {
if(reverse) {
sortValues[i] = new LongValue(field, new LongDesc());
} else {
sortValues[i] = new LongValue(field, new LongAsc());
}
} else if(ft instanceof StrField) {
LeafReader reader = searcher.getSlowAtomicReader();
SortedDocValues vals = reader.getSortedDocValues(field);
if(reverse) {
sortValues[i] = new StringValue(vals, field, new IntDesc());
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
}
} else if (ft instanceof TrieDateField) {
if (reverse) {
sortValues[i] = new LongValue(field, new LongDesc());
} else {
sortValues[i] = new LongValue(field, new LongAsc());
}
} else if (ft instanceof BoolField) {
// This is a bit of a hack, but since the boolean field stores ByteRefs, just like Strings
// _and_ since "F" happens to sort before "T" (thus false sorts "less" than true)
// we can just use the existing StringValue here.
LeafReader reader = searcher.getSlowAtomicReader();
SortedDocValues vals = reader.getSortedDocValues(field);
if (reverse) {
sortValues[i] = new StringValue(vals, field, new IntDesc());
} else {
sortValues[i] = new StringValue(vals, field, new IntAsc());
}
} else {
throw new IOException("Sort fields must be one of the following types: int,float,long,double,string,date,boolean");
}
}
if(sortValues.length == 1) {
return new SingleValueSortDoc(sortValues[0]);
} else if(sortValues.length == 2) {
return new DoubleValueSortDoc(sortValues[0], sortValues[1]);
} else if(sortValues.length == 3) {
return new TripleValueSortDoc(sortValues[0], sortValues[1], sortValues[2]);
} else if(sortValues.length == 4) {
return new QuadValueSortDoc(sortValues[0], sortValues[1], sortValues[2], sortValues[3]);
} else {
throw new IOException("A max of 4 sorts can be specified");
}
}
class SortQueue extends PriorityQueue<SortDoc> {
private SortDoc proto;
private Object[] cache;
public SortQueue(int len, SortDoc proto) {
super(len);
this.proto = proto;
}
protected boolean lessThan(SortDoc t1, SortDoc t2) {
return t1.lessThan(t2);
}
private void populate() {
Object[] heap = getHeapArray();
cache = new SortDoc[heap.length];
for (int i = 1; i < heap.length; i++) {
cache[i] = heap[i] = proto.copy();
}
size = maxSize;
}
private void reset() {
Object[] heap = getHeapArray();
if(cache != null) {
System.arraycopy(cache, 1, heap, 1, heap.length-1);
size = maxSize;
} else {
populate();
}
}
}
class SortDoc {
protected int docId = -1;
protected int ord = -1;
protected int docBase = -1;
private SortValue[] sortValues;
public SortDoc() {
}
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
for (SortValue value : sortValues) {
value.setNextReader(context);
}
}
public void reset() {
this.docId = -1;
}
public void setValues(int docId) throws IOException {
this.docId = docId;
for(SortValue sortValue : sortValues) {
sortValue.setCurrentValue(docId);
}
}
public void setValues(SortDoc sortDoc) throws IOException {
this.docId = sortDoc.docId;
this.ord = sortDoc.ord;
SortValue[] vals = sortDoc.sortValues;
for(int i=0; i<vals.length; i++) {
sortValues[i].setCurrentValue(vals[i]);
}
}
public SortDoc(SortValue[] sortValues) {
this.sortValues = sortValues;
}
public SortDoc copy() {
SortValue[] svs = new SortValue[sortValues.length];
for(int i=0; i<sortValues.length; i++) {
svs[i] = sortValues[i].copy();
}
return new SortDoc(svs);
}
public boolean lessThan(Object o) {
if(docId == -1) {
return true;
}
SortDoc sd = (SortDoc)o;
SortValue[] sortValues1 = sd.sortValues;
for(int i=0; i<sortValues.length; i++) {
int comp = sortValues[i].compareTo(sortValues1[i]);
if(comp < 0) {
return true;
} if(comp > 0) {
return false;
}
}
return docId+docBase < sd.docId+sd.docBase;
}
public String toString() {
return "";
}
}
class SingleValueSortDoc extends SortDoc {
protected SortValue value1;
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
value1.setNextReader(context);
}
public void reset() {
this.docId = -1;
this.value1.reset();
}
public void setValues(int docId) throws IOException {
this.docId = docId;
value1.setCurrentValue(docId);
}
public void setValues(SortDoc sortDoc) throws IOException {
this.docId = sortDoc.docId;
this.ord = sortDoc.ord;
value1.setCurrentValue(((SingleValueSortDoc)sortDoc).value1);
}
public SingleValueSortDoc(SortValue value1) {
super();
this.value1 = value1;
}
public SortDoc copy() {
return new SingleValueSortDoc(value1.copy());
}
public boolean lessThan(Object o) {
SingleValueSortDoc sd = (SingleValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
return docId+docBase > sd.docId+sd.docBase;
}
}
public int compareTo(Object o) {
SingleValueSortDoc sd = (SingleValueSortDoc)o;
return value1.compareTo(sd.value1);
}
public String toString() {
return docId+":"+value1.toString();
}
}
class DoubleValueSortDoc extends SingleValueSortDoc {
protected SortValue value2;
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
value1.setNextReader(context);
value2.setNextReader(context);
}
public void reset() {
this.docId = -1;
value1.reset();
value2.reset();
}
public void setValues(int docId) throws IOException {
this.docId = docId;
value1.setCurrentValue(docId);
value2.setCurrentValue(docId);
}
public void setValues(SortDoc sortDoc) throws IOException {
this.docId = sortDoc.docId;
this.ord = sortDoc.ord;
value1.setCurrentValue(((DoubleValueSortDoc)sortDoc).value1);
value2.setCurrentValue(((DoubleValueSortDoc)sortDoc).value2);
}
public DoubleValueSortDoc(SortValue value1, SortValue value2) {
super(value1);
this.value2 = value2;
}
public SortDoc copy() {
return new DoubleValueSortDoc(value1.copy(), value2.copy());
}
public boolean lessThan(Object o) {
DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value2.compareTo(sd.value2);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
return docId+docBase > sd.docId+sd.docBase;
}
}
}
public int compareTo(Object o) {
DoubleValueSortDoc sd = (DoubleValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == 0) {
return value2.compareTo(sd.value2);
} else {
return comp;
}
}
}
class TripleValueSortDoc extends DoubleValueSortDoc {
protected SortValue value3;
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
value1.setNextReader(context);
value2.setNextReader(context);
value3.setNextReader(context);
}
public void reset() {
this.docId = -1;
value1.reset();
value2.reset();
value3.reset();
}
public void setValues(int docId) throws IOException {
this.docId = docId;
value1.setCurrentValue(docId);
value2.setCurrentValue(docId);
value3.setCurrentValue(docId);
}
public void setValues(SortDoc sortDoc) throws IOException {
this.docId = sortDoc.docId;
this.ord = sortDoc.ord;
value1.setCurrentValue(((TripleValueSortDoc)sortDoc).value1);
value2.setCurrentValue(((TripleValueSortDoc)sortDoc).value2);
value3.setCurrentValue(((TripleValueSortDoc)sortDoc).value3);
}
public TripleValueSortDoc(SortValue value1, SortValue value2, SortValue value3) {
super(value1, value2);
this.value3 = value3;
}
public SortDoc copy() {
return new TripleValueSortDoc(value1.copy(), value2.copy(), value3.copy());
}
public boolean lessThan(Object o) {
TripleValueSortDoc sd = (TripleValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value2.compareTo(sd.value2);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value3.compareTo(sd.value3);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
return docId+docBase > sd.docId+sd.docBase;
}
}
}
}
public int compareTo(Object o) {
TripleValueSortDoc sd = (TripleValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == 0) {
comp = value2.compareTo(sd.value2);
if(comp == 0) {
return value3.compareTo(sd.value3);
} else {
return comp;
}
} else {
return comp;
}
}
}
class QuadValueSortDoc extends TripleValueSortDoc {
protected SortValue value4;
public void setNextReader(LeafReaderContext context) throws IOException {
this.ord = context.ord;
value1.setNextReader(context);
value2.setNextReader(context);
value3.setNextReader(context);
value4.setNextReader(context);
}
public void reset() {
this.docId = -1;
value1.reset();
value2.reset();
value3.reset();
value4.reset();
}
public void setValues(int docId) throws IOException {
this.docId = docId;
value1.setCurrentValue(docId);
value2.setCurrentValue(docId);
value3.setCurrentValue(docId);
value4.setCurrentValue(docId);
}
public void setValues(SortDoc sortDoc) throws IOException {
this.docId = sortDoc.docId;
this.ord = sortDoc.ord;
value1.setCurrentValue(((QuadValueSortDoc)sortDoc).value1);
value2.setCurrentValue(((QuadValueSortDoc)sortDoc).value2);
value3.setCurrentValue(((QuadValueSortDoc)sortDoc).value3);
value4.setCurrentValue(((QuadValueSortDoc)sortDoc).value4);
}
public QuadValueSortDoc(SortValue value1, SortValue value2, SortValue value3, SortValue value4) {
super(value1, value2, value3);
this.value4 = value4;
}
public SortDoc copy() {
return new QuadValueSortDoc(value1.copy(), value2.copy(), value3.copy(), value4.copy());
}
public boolean lessThan(Object o) {
QuadValueSortDoc sd = (QuadValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value2.compareTo(sd.value2);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value3.compareTo(sd.value3);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
comp = value4.compareTo(sd.value4);
if(comp == -1) {
return true;
} else if (comp == 1) {
return false;
} else {
return docId+docBase > sd.docId+sd.docBase;
}
}
}
}
}
public int compareTo(Object o) {
QuadValueSortDoc sd = (QuadValueSortDoc)o;
int comp = value1.compareTo(sd.value1);
if(comp == 0) {
comp = value2.compareTo(sd.value2);
if(comp == 0) {
comp = value3.compareTo(sd.value3);
if(comp == 0) {
return value4.compareTo(sd.value4);
} else {
return comp;
}
} else {
return comp;
}
} else {
return comp;
}
}
}
public interface SortValue extends Comparable<SortValue> {
public void setCurrentValue(int docId) throws IOException;
public void setNextReader(LeafReaderContext context) throws IOException;
public void setCurrentValue(SortValue value);
public void reset();
public SortValue copy();
}
class IntValue implements SortValue {
protected NumericDocValues vals;
protected String field;
protected int currentValue;
protected IntComp comp;
private int lastDocID;
public IntValue copy() {
return new IntValue(field, comp);
}
public IntValue(String field, IntComp comp) {
this.field = field;
this.comp = comp;
this.currentValue = comp.resetValue();
}
public void setNextReader(LeafReaderContext context) throws IOException {
this.vals = DocValues.getNumeric(context.reader(), field);
lastDocID = 0;
}
public void setCurrentValue(int docId) throws IOException {
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
}
lastDocID = docId;
int curDocID = vals.docID();
if (docId > curDocID) {
curDocID = vals.advance(docId);
}
if (docId == curDocID) {
currentValue = (int) vals.longValue();
} else {
currentValue = 0;
}
}
public int compareTo(SortValue o) {
IntValue iv = (IntValue)o;
return comp.compare(currentValue, iv.currentValue);
}
public void setCurrentValue (SortValue value) {
currentValue = ((IntValue)value).currentValue;
}
public void reset() {
currentValue = comp.resetValue();
}
}
public interface IntComp {
public int compare(int a, int b);
public int resetValue();
}
static class IntDesc implements IntComp {
public int resetValue() {
return Integer.MIN_VALUE;
}
public int compare(int a, int b) {
if(a < b) {
return -1;
} else if (a > b) {
return 1;
} else {
return 0;
}
}
}
static class IntAsc implements IntComp {
public int resetValue() {
return Integer.MAX_VALUE;
}
public int compare(int a, int b) {
if(a < b) {
return 1;
} else if (a > b) {
return -1;
} else {
return 0;
}
}
}
class LongValue implements SortValue {
protected NumericDocValues vals;
protected String field;
protected long currentValue;
protected LongComp comp;
private int lastDocID;
public LongValue(String field, LongComp comp) {
this.field = field;
this.comp = comp;
this.currentValue = comp.resetValue();
}
public LongValue copy() {
return new LongValue(field, comp);
}
public void setNextReader(LeafReaderContext context) throws IOException {
this.vals = DocValues.getNumeric(context.reader(), field);
lastDocID = 0;
}
public void setCurrentValue(int docId) throws IOException {
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
}
lastDocID = docId;
int curDocID = vals.docID();
if (docId > curDocID) {
curDocID = vals.advance(docId);
}
if (docId == curDocID) {
currentValue = vals.longValue();
} else {
currentValue = 0;
}
}
public void setCurrentValue(SortValue sv) {
LongValue lv = (LongValue)sv;
this.currentValue = lv.currentValue;
}
public int compareTo(SortValue o) {
LongValue l = (LongValue)o;
return comp.compare(currentValue, l.currentValue);
}
public void reset() {
this.currentValue = comp.resetValue();
}
}
interface LongComp {
public int compare(long a, long b);
public long resetValue();
}
static class LongDesc implements LongComp {
public long resetValue() {
return Long.MIN_VALUE;
}
public int compare(long a, long b) {
if(a < b) {
return -1;
} else if (a > b) {
return 1;
} else {
return 0;
}
}
}
static class LongAsc implements LongComp {
public long resetValue() {
return Long.MAX_VALUE;
}
public int compare(long a, long b) {
if(a < b) {
return 1;
} else if (a > b) {
return -1;
} else {
return 0;
}
}
}
class FloatValue implements SortValue {
protected NumericDocValues vals;
protected String field;
protected float currentValue;
protected FloatComp comp;
private int lastDocID;
public FloatValue(String field, FloatComp comp) {
this.field = field;
this.comp = comp;
this.currentValue = comp.resetValue();
}
public FloatValue copy() {
return new FloatValue(field, comp);
}
public void setNextReader(LeafReaderContext context) throws IOException {
this.vals = DocValues.getNumeric(context.reader(), field);
lastDocID = 0;
}
public void setCurrentValue(int docId) throws IOException {
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
}
lastDocID = docId;
int curDocID = vals.docID();
if (docId > curDocID) {
curDocID = vals.advance(docId);
}
if (docId == curDocID) {
currentValue = Float.intBitsToFloat((int)vals.longValue());
} else {
currentValue = 0f;
}
}
public void setCurrentValue(SortValue sv) {
FloatValue fv = (FloatValue)sv;
this.currentValue = fv.currentValue;
}
public void reset() {
this.currentValue = comp.resetValue();
}
public int compareTo(SortValue o) {
FloatValue fv = (FloatValue)o;
return comp.compare(currentValue, fv.currentValue);
}
}
interface FloatComp {
public int compare(float a, float b);
public float resetValue();
}
public static class FloatDesc implements FloatComp {
public float resetValue() {
return -Float.MAX_VALUE;
}
public int compare(float a, float b) {
if(a < b) {
return -1;
} else if (a > b) {
return 1;
} else {
return 0;
}
}
}
public static class FloatAsc implements FloatComp {
public float resetValue() {
return Float.MAX_VALUE;
}
public int compare(float a, float b) {
if(a < b) {
return 1;
} else if (a > b) {
return -1;
} else {
return 0;
}
}
}
class DoubleValue implements SortValue {
protected NumericDocValues vals;
protected String field;
protected double currentValue;
protected DoubleComp comp;
private int lastDocID;
private LeafReader reader;
public DoubleValue(String field, DoubleComp comp) {
this.field = field;
this.comp = comp;
this.currentValue = comp.resetValue();
}
public DoubleValue copy() {
return new DoubleValue(field, comp);
}
public void setNextReader(LeafReaderContext context) throws IOException {
this.reader = context.reader();
this.vals = DocValues.getNumeric(this.reader, this.field);
lastDocID = 0;
}
public void setCurrentValue(int docId) throws IOException {
if (docId < lastDocID) {
// TODO: can we enforce caller to go in order instead?
this.vals = DocValues.getNumeric(this.reader, this.field);
}
lastDocID = docId;
int curDocID = vals.docID();
if (docId > curDocID) {
curDocID = vals.advance(docId);
}
if (docId == curDocID) {
currentValue = Double.longBitsToDouble(vals.longValue());
} else {
currentValue = 0f;
}
}
public void setCurrentValue(SortValue sv) {
DoubleValue dv = (DoubleValue)sv;
this.currentValue = dv.currentValue;
}
public void reset() {
this.currentValue = comp.resetValue();
}
public int compareTo(SortValue o) {
DoubleValue dv = (DoubleValue)o;
return comp.compare(currentValue, dv.currentValue);
}
}
interface DoubleComp {
public int compare(double a, double b);
public double resetValue();
}
public static class DoubleDesc implements DoubleComp {
public double resetValue() {
return -Double.MAX_VALUE;
}
public int compare(double a, double b) {
if(a < b) {
return -1;
} else if (a > b) {
return 1;
} else {
return 0;
}
}
}
public static class DoubleAsc implements DoubleComp {
public double resetValue() {
return Double.MAX_VALUE;
}
public int compare(double a, double b) {
if(a < b) {
return 1;
} else if (a > b) {
return -1;
} else {
return 0;
}
}
}
class StringValue implements SortValue {
protected SortedDocValues vals;
protected MultiDocValues.OrdinalMap ordinalMap;
protected LongValues globalOrds;
protected SortedDocValues currentVals;
protected String field;
protected int segment;
protected int currentOrd;
protected IntComp comp;
protected int lastDocID;
public StringValue(SortedDocValues vals, String field, IntComp comp) {
this.vals = vals;
if(vals instanceof MultiDocValues.MultiSortedDocValues) {
this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) vals).mapping;
}
this.field = field;
this.comp = comp;
this.currentOrd = comp.resetValue();
}
public StringValue copy() {
return new StringValue(vals, field, comp);
}
public void setCurrentValue(int docId) throws IOException {
if (docId < lastDocID) {
throw new AssertionError("docs were sent out-of-order: lastDocID=" + lastDocID + " vs doc=" + docId);
}
lastDocID = docId;
if (docId > currentVals.docID()) {
currentVals.advance(docId);
}
if (docId == currentVals.docID()) {
int ord = currentVals.ordValue();
if(globalOrds != null) {
currentOrd = (int)globalOrds.get(ord);
} else {
currentOrd = ord;
}
} else {
currentOrd = -1;
}
}
public void setCurrentValue(SortValue sv) {
StringValue v = (StringValue)sv;
this.currentOrd = v.currentOrd;
}
public void setNextReader(LeafReaderContext context) throws IOException {
segment = context.ord;
if(ordinalMap != null) {
globalOrds = ordinalMap.getGlobalOrds(segment);
}
currentVals = DocValues.getSorted(context.reader(), field);
lastDocID = 0;
}
public void reset() {
this.currentOrd = comp.resetValue();
}
public int compareTo(SortValue o) {
StringValue sv = (StringValue)o;
return comp.compare(currentOrd, sv.currentOrd);
}
public String toString() {
return Integer.toString(this.currentOrd);
}
}
protected abstract class FieldWriter {
public abstract boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException;
}
class IntFieldWriter extends FieldWriter {
private String field;
public IntFieldWriter(String field) {
this.field = field;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val;
if (vals.advance(docId) == docId) {
val = (int) vals.longValue();
} else {
return false;
}
ew.put(this.field, val);
return true;
}
}
class MultiFieldWriter extends FieldWriter {
private String field;
private FieldType fieldType;
private SchemaField schemaField;
private boolean numeric;
private CharsRefBuilder cref = new CharsRefBuilder();
public MultiFieldWriter(String field, FieldType fieldType, SchemaField schemaField, boolean numeric) {
this.field = field;
this.fieldType = fieldType;
this.schemaField = schemaField;
this.numeric = numeric;
}
public boolean write(int docId, LeafReader reader, EntryWriter out, int fieldIndex) throws IOException {
SortedSetDocValues vals = DocValues.getSortedSet(reader, this.field);
if (vals.advance(docId) != docId) return false;
out.put(this.field,
(IteratorWriter) w -> {
long o;
while((o = vals.nextOrd()) != SortedSetDocValues.NO_MORE_ORDS) {
BytesRef ref = vals.lookupOrd(o);
fieldType.indexedToReadable(ref, cref);
IndexableField f = fieldType.createField(schemaField, cref.toString());
if (f == null) w.add(cref.toString());
else w.add(fieldType.toObject(f));
}
});
return true;
}
}
class LongFieldWriter extends FieldWriter {
private String field;
public LongFieldWriter(String field) {
this.field = field;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val;
if (vals.advance(docId) == docId) {
val = vals.longValue();
} else {
return false;
}
ew.put(field, val);
return true;
}
}
class DateFieldWriter extends FieldWriter {
private String field;
public DateFieldWriter(String field) {
this.field = field;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val;
if (vals.advance(docId) == docId) {
val = vals.longValue();
} else {
return false;
}
ew.put(this.field, new Date(val));
return true;
}
}
class BoolFieldWriter extends FieldWriter {
private String field;
private FieldType fieldType;
private CharsRefBuilder cref = new CharsRefBuilder();
public BoolFieldWriter(String field, FieldType fieldType) {
this.field = field;
this.fieldType = fieldType;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
if (vals.advance(docId) != docId) {
return false;
}
int ord = vals.ordValue();
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
ew.put(this.field, "true".equals(cref.toString()));
return true;
}
}
class FloatFieldWriter extends FieldWriter {
private String field;
public FloatFieldWriter(String field) {
this.field = field;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
int val;
if (vals.advance(docId) == docId) {
val = (int)vals.longValue();
} else {
return false;
}
ew.put(this.field, Float.intBitsToFloat(val));
return true;
}
}
class DoubleFieldWriter extends FieldWriter {
private String field;
public DoubleFieldWriter(String field) {
this.field = field;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
NumericDocValues vals = DocValues.getNumeric(reader, this.field);
long val;
if (vals.advance(docId) == docId) {
val = vals.longValue();
} else {
return false;
}
ew.put(this.field, Double.longBitsToDouble(val));
return true;
}
}
class StringFieldWriter extends FieldWriter {
private String field;
private FieldType fieldType;
private CharsRefBuilder cref = new CharsRefBuilder();
public StringFieldWriter(String field, FieldType fieldType) {
this.field = field;
this.fieldType = fieldType;
}
public boolean write(int docId, LeafReader reader, EntryWriter ew, int fieldIndex) throws IOException {
SortedDocValues vals = DocValues.getSorted(reader, this.field);
if (vals.advance(docId) != docId) {
return false;
}
int ord = vals.ordValue();
BytesRef ref = vals.lookupOrd(ord);
fieldType.indexedToReadable(ref, cref);
ew.put(this.field, cref.toString());
return true;
}
}
public abstract class PriorityQueue<T> {
protected int size = 0;
protected final int maxSize;
private final T[] heap;
public PriorityQueue(int maxSize) {
this(maxSize, true);
}
public PriorityQueue(int maxSize, boolean prepopulate) {
final int heapSize;
if (0 == maxSize) {
// We allocate 1 extra to avoid if statement in top()
heapSize = 2;
} else {
if (maxSize > ArrayUtil.MAX_ARRAY_LENGTH) {
// Don't wrap heapSize to -1, in this case, which
// causes a confusing NegativeArraySizeException.
// Note that very likely this will simply then hit
// an OOME, but at least that's more indicative to
// caller that this values is too big. We don't +1
// in this case, but it's very unlikely in practice
// one will actually insert this many objects into
// the PQ:
// Throw exception to prevent confusing OOME:
throw new IllegalArgumentException("maxSize must be <= " + ArrayUtil.MAX_ARRAY_LENGTH + "; got: " + maxSize);
} else {
// NOTE: we add +1 because all access to heap is
// 1-based not 0-based. heap[0] is unused.
heapSize = maxSize + 1;
}
}
// T is unbounded type, so this unchecked cast works always:
@SuppressWarnings("unchecked") final T[] h = (T[]) new Object[heapSize];
this.heap = h;
this.maxSize = maxSize;
if (prepopulate) {
// If sentinel objects are supported, populate the queue with them
T sentinel = getSentinelObject();
if (sentinel != null) {
heap[1] = sentinel;
for (int i = 2; i < heap.length; i++) {
heap[i] = getSentinelObject();
}
size = maxSize;
}
}
}
/** Determines the ordering of objects in this priority queue. Subclasses
* must define this one method.
* @return <code>true</code> iff parameter <tt>a</tt> is less than parameter <tt>b</tt>.
*/
protected abstract boolean lessThan(T a, T b);
protected T getSentinelObject() {
return null;
}
/**
* Adds an Object to a PriorityQueue in log(size) time. If one tries to add
* more objects than maxSize from initialize an
*
* @return the new 'top' element in the queue.
*/
public final T add(T element) {
size++;
heap[size] = element;
upHeap();
return heap[1];
}
/**
* Adds an Object to a PriorityQueue in log(size) time.
* It returns the object (if any) that was
* dropped off the heap because it was full. This can be
* the given parameter (in case it is smaller than the
* full heap's minimum, and couldn't be added), or another
* object that was previously the smallest value in the
* heap and now has been replaced by a larger one, or null
* if the queue wasn't yet full with maxSize elements.
*/
public T insertWithOverflow(T element) {
if (size < maxSize) {
add(element);
return null;
} else if (size > 0 && !lessThan(element, heap[1])) {
T ret = heap[1];
heap[1] = element;
updateTop();
return ret;
} else {
return element;
}
}
/** Returns the least element of the PriorityQueue in constant time. */
public final T top() {
// We don't need to check size here: if maxSize is 0,
// then heap is length 2 array with both entries null.
// If size is 0 then heap[1] is already null.
return heap[1];
}
/** Removes and returns the least element of the PriorityQueue in log(size)
time. */
public final T pop() {
if (size > 0) {
T result = heap[1]; // save first value
heap[1] = heap[size]; // move last to first
heap[size] = null; // permit GC of objects
size--;
downHeap(); // adjust heap
return result;
} else {
return null;
}
}
/**
* Should be called when the Object at top changes values. Still log(n) worst
* case, but it's at least twice as fast to
*
* <pre class="prettyprint">
* pq.top().change();
* pq.updateTop();
* </pre>
*
* instead of
*
* <pre class="prettyprint">
* o = pq.pop();
* o.change();
* pq.push(o);
* </pre>
*
* @return the new 'top' element.
*/
public final T updateTop() {
downHeap();
return heap[1];
}
/** Returns the number of elements currently stored in the PriorityQueue. */
public final int size() {
return size;
}
/** Removes all entries from the PriorityQueue. */
public final void clear() {
for (int i = 0; i <= size; i++) {
heap[i] = null;
}
size = 0;
}
private final void upHeap() {
int i = size;
T node = heap[i]; // save bottom node
int j = i >>> 1;
while (j > 0 && lessThan(node, heap[j])) {
heap[i] = heap[j]; // shift parents down
i = j;
j = j >>> 1;
}
heap[i] = node; // install saved node
}
private final void downHeap() {
int i = 1;
T node = heap[i]; // save top node
int j = i << 1; // find smaller child
int k = j + 1;
if (k <= size && lessThan(heap[k], heap[j])) {
j = k;
}
while (j <= size && lessThan(heap[j], node)) {
heap[i] = heap[j]; // shift up child
i = j;
j = i << 1;
k = j + 1;
if (k <= size && lessThan(heap[k], heap[j])) {
j = k;
}
}
heap[i] = node; // install saved node
}
/** This method returns the internal heap array as Object[].
* @lucene.internal
*/
public final Object[] getHeapArray() {
return (Object[]) heap;
}
}
public static class IgnoreException extends IOException {
public void printStackTrace(PrintWriter pw) {
pw.print("Early Client Disconnect");
}
public String getMessage() {
return "Early Client Disconnect";
}
}
}