/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.handler.component;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionQuery;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.queries.function.valuesource.FieldCacheSource;
import org.apache.lucene.queries.function.valuesource.QueryValueSource;
import org.apache.lucene.search.Query;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.DocValuesStats;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.NumberType;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocSet;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.QueryParsing;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.search.SyntaxError;
import org.apache.solr.util.hll.HLL;
import org.apache.solr.util.hll.HLLType;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
/**
* Models all of the information associated with a single {@link StatsParams#STATS_FIELD}
* instance.
*
* @see StatsComponent
*/
public class StatsField {
/**
* An enumeration representing the sumer set of all possible stat values that can be computed.
* Each of these enum values can be specified as a local param in a <code>stats.field</code>
* (eg: <code>stats.field={!min=true mean=true}my_field_name</code>) but not all enum values
* are valid for all field types (eg: <code>mean</code> is meaningless for String fields)
*
* @lucene.internal
* @lucene.experimental
*/
public static enum Stat {
min(true),
max(true),
missing(true),
sum(true),
count(true),
mean(false, sum, count),
sumOfSquares(true),
stddev(false, sum, count, sumOfSquares),
distinctValues(true),
countDistinct(false, distinctValues),
percentiles(true){
/** special for percentiles **/
boolean parseParams(StatsField sf) {
String percentileParas = sf.localParams.get(this.name());
if (percentileParas != null) {
List<Double> percentiles = new ArrayList<Double>();
try {
for (String percentile : StrUtils.splitSmart(percentileParas, ',')) {
percentiles.add(Double.parseDouble(percentile));
}
if (!percentiles.isEmpty()) {
sf.percentilesList.addAll(percentiles);
sf.tdigestCompression = sf.localParams.getDouble("tdigestCompression",
sf.tdigestCompression);
return true;
}
} catch (NumberFormatException e) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to parse "
+ StatsParams.STATS_FIELD + " local params: " + sf.localParams + " due to: "
+ e.getMessage(), e);
}
}
return false;
}
},
cardinality(true) {
/** special for percentiles **/
boolean parseParams(StatsField sf) {
try {
sf.hllOpts = HllOptions.parseHllOptions(sf.localParams, sf.schemaField);
return (null != sf.hllOpts);
} catch (Exception e) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to parse "
+ StatsParams.STATS_FIELD + " local params: " + sf.localParams + " due to: "
+ e.getMessage(), e);
}
}
};
private final List<Stat> distribDeps;
/**
* Sole constructor for Stat enum values
* @param deps the set of stat values, other then this one, which are a distributed
* dependency and must be computed and returned by each individual shards in
* order to compute <i>this</i> stat over the entire distributed result set.
* @param selfDep indicates that when computing this stat across a distributed result
* set, each shard must compute this stat <i>in addition to</i> any other
* distributed dependencies.
* @see #getDistribDeps
*/
Stat(boolean selfDep, Stat... deps) {
distribDeps = new ArrayList<Stat>(deps.length+1);
distribDeps.addAll(Arrays.asList(deps));
if (selfDep) {
distribDeps.add(this);
}
}
/**
* Given a String, returns the corrisponding Stat enum value if any, otherwise returns null.
*/
public static Stat forName(String paramKey) {
try {
return Stat.valueOf(paramKey);
} catch (IllegalArgumentException e) {
return null;
}
}
/**
* The stats that must be computed and returned by each shard involved in a distributed
* request in order to compute the overall value for this stat across the entire distributed
* result set. A Stat instance may include itself in the <code>getDistribDeps()</code> result,
* but that is not always the case.
*/
public EnumSet<Stat> getDistribDeps() {
return EnumSet.copyOf(this.distribDeps);
}
/**
* Called when the name of a stat is found as a local param on this {@link StatsField}
* @return true if the user is requesting this stat, else false
*/
boolean parseParams(StatsField sf) {
return sf.localParams.getBool(this.name(), false);
}
}
/**
* the equivilent stats if "calcdistinct" is specified
* @see Stat#countDistinct
* @see Stat#distinctValues
*/
private static final EnumSet<Stat> CALCDISTINCT_PSUEDO_STAT = EnumSet.of(Stat.countDistinct, Stat.distinctValues);
/**
* The set of stats computed by default when no localparams are used to specify explicit stats
*/
public final static Set<Stat> DEFAULT_STATS = Collections.<Stat>unmodifiableSet
(EnumSet.of(Stat.min, Stat.max, Stat.missing, Stat.sum, Stat.count, Stat.mean, Stat.sumOfSquares, Stat.stddev));
private final SolrIndexSearcher searcher;
private final ResponseBuilder rb;
private final String originalParam; // for error messages
private final SolrParams localParams;
private final ValueSource valueSource; // may be null if simple field stats
private final SchemaField schemaField; // may be null if function/query stats
private final String key;
private final boolean topLevelCalcDistinct;
private final String[] facets;
private final List<String> tagList;
private final List<String> excludeTagList;
private final EnumSet<Stat> statsToCalculate = EnumSet.noneOf(Stat.class);
private final EnumSet<Stat> statsInResponse = EnumSet.noneOf(Stat.class);
private final List<Double> percentilesList= new ArrayList<Double>();
private final boolean isShard;
private double tdigestCompression = 100.0D;
private HllOptions hllOpts;
/**
* @param rb the current request/response
* @param statsParam the raw {@link StatsParams#STATS_FIELD} string
*/
public StatsField(ResponseBuilder rb, String statsParam) {
this.rb = rb;
this.searcher = rb.req.getSearcher();
this.originalParam = statsParam;
SolrParams params = rb.req.getParams();
try {
isShard = params.getBool("isShard", false);
SolrParams localParams = QueryParsing.getLocalParams(originalParam, params);
if (null == localParams) {
// simplest possible input: bare string (field name)
ModifiableSolrParams customParams = new ModifiableSolrParams();
customParams.add(QueryParsing.V, originalParam);
localParams = customParams;
}
this.localParams = localParams;
String parserName = localParams.get(QueryParsing.TYPE);
SchemaField sf = null;
ValueSource vs = null;
if ( StringUtils.isBlank(parserName) ) {
// basic request for field stats
sf = searcher.getSchema().getField(localParams.get(QueryParsing.V));
} else {
// we have a non trivial request to compute stats over a query (or function)
// NOTE we could use QParser.getParser(...) here, but that would redundently
// reparse everything. ( TODO: refactor a common method in QParser ?)
QParserPlugin qplug = rb.req.getCore().getQueryPlugin(parserName);
QParser qp = qplug.createParser(localParams.get(QueryParsing.V),
localParams, params, rb.req);
// figure out what type of query we are dealing, get the most direct ValueSource
vs = extractValueSource(qp.parse());
// if this ValueSource directly corrisponds to a SchemaField, act as if
// we were asked to compute stats on it directly
// ie: "stats.field={!func key=foo}field(foo)" == "stats.field=foo"
sf = extractSchemaField(vs, searcher.getSchema());
if (null != sf) {
vs = null;
}
}
assert ( (null == vs) ^ (null == sf) ) : "exactly one of vs & sf must be null";
this.schemaField = sf;
this.valueSource = vs;
} catch (SyntaxError e) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Unable to parse " +
StatsParams.STATS_FIELD + ": " + originalParam + " due to: "
+ e.getMessage(), e);
}
// allow explicit setting of the response key via localparams...
this.key = localParams.get(CommonParams.OUTPUT_KEY,
// default to the main param value...
localParams.get(CommonParams.VALUE,
// default to entire original param str.
originalParam));
this.topLevelCalcDistinct = null == schemaField
? params.getBool(StatsParams.STATS_CALC_DISTINCT, false)
: params.getFieldBool(schemaField.getName(), StatsParams.STATS_CALC_DISTINCT, false);
populateStatsSets();
String[] facets = params.getFieldParams(key, StatsParams.STATS_FACET);
this.facets = (null == facets) ? new String[0] : facets;
String tagStr = localParams.get(CommonParams.TAG);
this.tagList = (null == tagStr)
? Collections.<String>emptyList()
: StrUtils.splitSmart(tagStr,',');
// figure out if we need a special base DocSet
String excludeStr = localParams.get(CommonParams.EXCLUDE);
this.excludeTagList = (null == excludeStr)
? Collections.<String>emptyList()
: StrUtils.splitSmart(excludeStr,',');
assert ( (null == this.valueSource) ^ (null == this.schemaField) )
: "exactly one of valueSource & schemaField must be null";
}
/**
* Inspects a {@link Query} to see if it directly maps to a {@link ValueSource},
* and if so returns it -- otherwise wraps it as needed.
*
* @param q Query whose scores we have been asked to compute stats of
* @returns a ValueSource to use for computing the stats
*/
private static ValueSource extractValueSource(Query q) {
return (q instanceof FunctionQuery) ?
// Common case: we're wrapping a func, so we can directly pull out ValueSource
((FunctionQuery) q).getValueSource() :
// asked to compute stats over a query, wrap it up as a ValueSource
new QueryValueSource(q, 0.0F);
}
/**
* Inspects a {@link ValueSource} to see if it directly maps to a {@link SchemaField},
* and if so returns it.
*
* @param vs ValueSource we've been asked to compute stats of
* @param schema The Schema to use
* @returns Corrisponding {@link SchemaField} or null if the ValueSource is more complex
* @see FieldCacheSource
*/
private static SchemaField extractSchemaField(ValueSource vs, IndexSchema schema) {
if (vs instanceof FieldCacheSource) {
String fieldName = ((FieldCacheSource)vs).getField();
return schema.getField(fieldName);
}
return null;
}
/**
* The key to be used when refering to this {@link StatsField} instance in the
* response tp clients.
*/
public String getOutputKey() {
return key;
}
/**
* Computes a base {@link DocSet} for the current request to be used
* when computing global stats for the local index.
*
* This is typically the same as the main DocSet for the {@link ResponseBuilder}
* unless {@link CommonParams#TAG tag}ged filter queries have been excluded using
* the {@link CommonParams#EXCLUDE ex} local param
*/
public DocSet computeBaseDocSet() throws IOException {
DocSet docs = rb.getResults().docSet;
Map<?,?> tagMap = (Map<?,?>) rb.req.getContext().get("tags");
if (excludeTagList.isEmpty() || null == tagMap) {
// either the exclude list is empty, or there
// aren't any tagged filters to exclude anyway.
return docs;
}
IdentityHashMap<Query,Boolean> excludeSet = new IdentityHashMap<Query,Boolean>();
for (String excludeTag : excludeTagList) {
Object olst = tagMap.get(excludeTag);
// tagMap has entries of List<String,List<QParser>>, but subject to change in the future
if (!(olst instanceof Collection)) continue;
for (Object o : (Collection<?>)olst) {
if (!(o instanceof QParser)) continue;
QParser qp = (QParser)o;
try {
excludeSet.put(qp.getQuery(), Boolean.TRUE);
} catch (SyntaxError e) {
// this shouldn't be possible since the request should have already
// failed when attempting to execute the query, but just in case...
throw new SolrException(ErrorCode.BAD_REQUEST, "Excluded query can't be parsed: " +
originalParam + " due to: " + e.getMessage(), e);
}
}
}
if (excludeSet.size() == 0) return docs;
List<Query> qlist = new ArrayList<Query>();
// add the base query
if (!excludeSet.containsKey(rb.getQuery())) {
qlist.add(rb.getQuery());
}
// add the filters
if (rb.getFilters() != null) {
for (Query q : rb.getFilters()) {
if (!excludeSet.containsKey(q)) {
qlist.add(q);
}
}
}
// get the new base docset for this facet
return searcher.getDocSet(qlist);
}
/**
* Computes the {@link StatsValues} for this {@link StatsField} relative to the
* specified {@link DocSet}
* @see #computeBaseDocSet
*/
public StatsValues computeLocalStatsValues(DocSet base) throws IOException {
if (statsToCalculate.isEmpty()) {
// perf optimization for the case where we compute nothing
// ie: stats.field={!min=$domin}myfield&domin=false
return StatsValuesFactory.createStatsValues(this);
}
if (null != schemaField && !schemaField.getType().isPointField()
&& (schemaField.multiValued() || schemaField.getType().multiValuedFieldCache())) {
// TODO: should this also be used for single-valued string fields? (should work fine)
return DocValuesStats.getCounts(searcher, this, base, facets);
} else {
// either a single valued field we pull from FieldCache, or an explicit
// function ValueSource
return computeLocalValueSourceStats(base);
}
}
private StatsValues computeLocalValueSourceStats(DocSet base) throws IOException {
IndexSchema schema = searcher.getSchema();
final StatsValues allstats = StatsValuesFactory.createStatsValues(this);
List<FieldFacetStats> facetStats = new ArrayList<>();
for( String facetField : facets ) {
SchemaField fsf = schema.getField(facetField);
if ( fsf.multiValued()) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Stats can only facet on single-valued fields, not: " + facetField );
}
facetStats.add(new FieldFacetStats(searcher, fsf, this));
}
final Iterator<LeafReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
LeafReaderContext ctx = null;
for (DocIterator docsIt = base.iterator(); docsIt.hasNext(); ) {
final int doc = docsIt.nextDoc();
if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
// advance
do {
ctx = ctxIt.next();
} while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
assert doc >= ctx.docBase;
// propagate the context among accumulators.
allstats.setNextReader(ctx);
for (FieldFacetStats f : facetStats) {
f.setNextReader(ctx);
}
}
// accumulate
allstats.accumulate(doc - ctx.docBase);
for (FieldFacetStats f : facetStats) {
f.facet(doc - ctx.docBase);
}
}
for (FieldFacetStats f : facetStats) {
allstats.addFacet(f.name, f.facetStatsValues);
}
return allstats;
}
/**
* The searcher that should be used for processing local stats
* @see SolrQueryRequest#getSearcher
*/
public SolrIndexSearcher getSearcher() {
// see AbstractStatsValues.setNextReader
return searcher;
}
/**
* The {@link SchemaField} whose results these stats are computed over, may be null
* if the stats are computed over the results of a function or query
*
* @see #getValueSource
*/
public SchemaField getSchemaField() {
return schemaField;
}
/**
* The {@link ValueSource} of a function or query whose results these stats are computed
* over, may be null if the stats are directly over a {@link SchemaField}
*
* @see #getValueSource
*/
public ValueSource getValueSource() {
return valueSource;
}
public List<String> getTagList() {
return tagList;
}
public String toString() {
return "StatsField<" + originalParam + ">";
}
/**
* A helper method which inspects the {@link #localParams} associated with this StatsField,
* and uses them to populate the {@link #statsInResponse} and {@link #statsToCalculate} data
* structures
*/
private void populateStatsSets() {
boolean statSpecifiedByLocalParam = false;
// local individual stat
Iterator<String> itParams = localParams.getParameterNamesIterator();
while (itParams.hasNext()) {
String paramKey = itParams.next();
Stat stat = Stat.forName(paramKey);
if (stat != null) {
statSpecifiedByLocalParam = true;
if (stat.parseParams(this)) {
statsInResponse.add(stat);
}
}
}
// if no individual stat setting use the default set
if ( ! ( statSpecifiedByLocalParam
// calcdistinct (as a local param) is a psuedo-stat, prevents default set
|| localParams.getBool("calcdistinct", false) ) ) {
statsInResponse.addAll(DEFAULT_STATS);
}
// calcDistinct is a psuedo-stat with optional top level param default behavior
// if not overridden by the specific individual stats
if (localParams.getBool("calcdistinct", topLevelCalcDistinct)) {
for (Stat stat : CALCDISTINCT_PSUEDO_STAT) {
// assume true, but don't include if specific stat overrides
if (localParams.getBool(stat.name(), true)) {
statsInResponse.add(stat);
}
}
}
for (Stat stat : statsInResponse) {
statsToCalculate.addAll(stat.getDistribDeps());
}
}
public boolean calculateStats(Stat stat) {
return statsToCalculate.contains(stat);
}
public boolean includeInResponse(Stat stat) {
if (isShard) {
return statsToCalculate.contains(stat);
}
if (statsInResponse.contains(stat)) {
return true;
}
return false;
}
public List<Double> getPercentilesList() {
return percentilesList;
}
public boolean getIsShard() {
return isShard;
}
public double getTdigestCompression() {
return tdigestCompression;
}
public HllOptions getHllOptions() {
return hllOpts;
}
/**
* Helper Struct for parsing and encapsulating all of the options relaed to building a {@link HLL}
*
* @see Stat#cardinality
* @lucene.internal
*/
public static final class HllOptions {
final HashFunction hasher;
// NOTE: this explanation linked to from the java-hll jdocs...
// https://github.com/aggregateknowledge/postgresql-hll/blob/master/README.markdown#explanation-of-parameters-and-tuning
// ..if i'm understanding the regwidth chart correctly, a value of 6 should be a enough
// to support any max cardinality given that we're always dealing with hashes and
// the cardinality of the set of all long values is 2**64 == 1.9e19
//
// But i guess that assumes a *perfect* hash and high log2m? ... if the hash algo is imperfect
// and/or log2m is low (ie: user is less concerned about accuracy), then many diff hash values
// might fall in the same register (ie: bucket) and having a wider register to count more of
// them may be useful
final int log2m;
final int regwidth;
final static String ERR = "cardinality must be specified as 'true' (for default tunning) or decimal number between 0 and 1 to adjust accuracy vs memory usage (large number is more memory and more accuracy)";
private HllOptions(int log2m, int regwidth, HashFunction hasher) {
this.log2m = log2m;
this.regwidth = regwidth;
this.hasher = hasher;
}
/**
* Creates an HllOptions based on the (local) params specified (if appropriate).
*
* @param localParams the LocalParams for this {@link StatsField}
* @param field the field corresponding to this {@link StatsField}, may be null if these stats are over a value source
* @return the {@link HllOptions} to use based on the params, or null if no {@link HLL} should be computed
* @throws SolrException if there are invalid options
*/
public static HllOptions parseHllOptions(SolrParams localParams, SchemaField field)
throws SolrException {
String cardinalityOpt = localParams.get(Stat.cardinality.name());
if (StringUtils.isBlank(cardinalityOpt)) {
return null;
}
final NumberType hashableNumType = getHashableNumericType(field);
// some sane defaults
int log2m = 13; // roughly equivilent to "cardinality='0.33'"
int regwidth = 6; // with decent hash, this is plenty for all valid long hashes
if (NumberType.FLOAT.equals(hashableNumType) || NumberType.INTEGER.equals(hashableNumType)) {
// for 32bit values, we can adjust our default regwidth down a bit
regwidth--;
// NOTE: EnumField uses LegacyNumericType.INT, and in theory we could be super conservative
// with it, but there's no point - just let the EXPLICIT HLL handle it
}
// TODO: we could attempt additional reductions in the default regwidth based on index
// statistics -- but thta doesn't seem worth the effort. for tiny indexes, the
// EXPLICIT and SPARSE HLL representations have us nicely covered, and in general we don't
// want to be too aggresive about lowering regwidth or we could really poor results if
// log2m is also low and there is heavy hashkey collision
try {
// NFE will short out here if it's not a number
final double accuracyOpt = Double.parseDouble(cardinalityOpt);
// if a float between 0 and 1 is specified, treat it as a prefrence of accuracy
// - 0 means accuracy is not a concern, save RAM
// - 1 means be as accurate as possible, using as much RAM as needed.
if (accuracyOpt < 0D || 1.0D < accuracyOpt) {
throw new SolrException(ErrorCode.BAD_REQUEST, ERR);
}
// use accuracyOpt as a scaling factor between min & max legal log2m values
log2m = HLL.MINIMUM_LOG2M_PARAM
+ (int) Math.round(accuracyOpt * (HLL.MAXIMUM_LOG2M_PARAM - HLL.MINIMUM_LOG2M_PARAM));
// use accuracyOpt as a scaling factor for regwidth as well, BUT...
// be more conservative -- HLL.MIN_REGWIDTH_PARAM is too absurdly low to be useful
// use previously computed (hashableNumType) default regwidth -1 as lower bound for scaling
final int MIN_HUERISTIC_REGWIDTH = regwidth-1;
regwidth = MIN_HUERISTIC_REGWIDTH
+ (int) Math.round(accuracyOpt * (HLL.MAXIMUM_REGWIDTH_PARAM - MIN_HUERISTIC_REGWIDTH));
} catch (NumberFormatException nfe) {
// param value isn't a number -- let's check for simple true/false
if (! localParams.getBool(Stat.cardinality.name(), false)) {
return null;
}
}
// let explicit params override both the default and/or any accuracy specification
log2m = localParams.getInt("hllLog2m", log2m);
regwidth = localParams.getInt("hllRegwidth", regwidth);
// validate legal values
if (log2m < HLL.MINIMUM_LOG2M_PARAM || HLL.MAXIMUM_LOG2M_PARAM < log2m) {
throw new SolrException(ErrorCode.BAD_REQUEST, "hllLog2m must be at least " +
HLL.MINIMUM_LOG2M_PARAM + " and at most " + HLL.MAXIMUM_LOG2M_PARAM
+ " (" + log2m +")");
}
if (regwidth < HLL.MINIMUM_REGWIDTH_PARAM || HLL.MAXIMUM_REGWIDTH_PARAM < regwidth) {
throw new SolrException(ErrorCode.BAD_REQUEST, "hllRegwidth must be at least " +
HLL.MINIMUM_REGWIDTH_PARAM + " and at most " + HLL.MAXIMUM_REGWIDTH_PARAM);
}
HashFunction hasher = localParams.getBool("hllPreHashed", false) ? null : Hashing.murmur3_128();
if (null == hasher) {
// if this is a function, or a non Long field, pre-hashed is invalid
// NOTE: we ignore hashableNumType - it's LONG for non numerics like Strings
if (null == field || !(NumberType.LONG.equals(field.getType().getNumberType()) || NumberType.DATE.equals(field.getType().getNumberType()))) {
throw new SolrException(ErrorCode.BAD_REQUEST, "hllPreHashed is only supported with Long based fields");
}
}
// if we're still here, then we need an HLL...
return new HllOptions(log2m, regwidth, hasher);
}
/** @see HLL */
public int getLog2m() {
return log2m;
}
/** @see HLL */
public int getRegwidth() {
return regwidth;
}
/** May be null if user has indicated that field values are pre-hashed */
public HashFunction getHasher() {
return hasher;
}
public HLL newHLL() {
// Although it (in theory) saves memory for "medium" size sets, the SPARSE type seems to have
// some nasty impacts on response time as it gets larger - particularly in distrib requests.
// Merging large SPARSE HLLs is much much slower then merging FULL HLLs with the same num docs
//
// TODO: add more tunning options for this.
return new HLL(getLog2m(), getRegwidth(), -1 /* auto explict threshold */,
false /* no sparse representation */, HLLType.EMPTY);
}
}
/**
* Returns the effective {@link NumberType} for the field for the purposes of hash values.
* ie: If the field has an explict NumberType that is returned; If the field has no explicit
* NumberType then {@link NumberType#LONG} is returned; If field is null, then
* {@link NumberType#FLOAT} is assumed for ValueSource.
*/
private static NumberType getHashableNumericType(SchemaField field) {
if (null == field) {
return NumberType.FLOAT;
}
final NumberType result = field.getType().getNumberType();
return null == result ? NumberType.LONG : result;
}
}