/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.LinkedHashMap;
import java.util.Map;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
import org.apache.lucene.util.Bits;
import org.apache.solr.common.MapSerializable;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.util.Hash;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/** @lucene.internal */
public class IndexFingerprint implements MapSerializable {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private long maxVersionSpecified;
private long maxVersionEncountered;
// this actually means max versions used in computing the hash.
// we cannot change this now because it changes back-compat
private long maxInHash;
private long versionsHash;
private long numVersions;
private long numDocs;
private long maxDoc;
public IndexFingerprint() {
// default constructor
}
public IndexFingerprint (long maxVersionSpecified) {
this.maxVersionSpecified = maxVersionSpecified;
}
public long getMaxVersionSpecified() {
return maxVersionSpecified;
}
public long getMaxVersionEncountered() {
return maxVersionEncountered;
}
public long getMaxInHash() {
return maxInHash;
}
public long getVersionsHash() {
return versionsHash;
}
public long getNumVersions() {
return numVersions;
}
public long getNumDocs() {
return numDocs;
}
public long getMaxDoc() {
return maxDoc;
}
/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
RTimer timer = new RTimer();
core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
try {
IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
final double duration = timer.stop();
log.info("IndexFingerprint millis:{} result:{}",duration, f);
return f;
} finally {
if (newestSearcher != null) {
newestSearcher.decref();
}
}
}
public static IndexFingerprint getFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, Long maxVersion)
throws IOException {
SchemaField versionField = VersionInfo.getAndCheckVersionField(searcher.getSchema());
ValueSource vs = versionField.getType().getValueSource(versionField, null);
Map funcContext = ValueSource.newContext(searcher);
vs.createWeight(funcContext, searcher);
IndexFingerprint f = new IndexFingerprint();
f.maxVersionSpecified = maxVersion;
f.maxDoc = ctx.reader().maxDoc();
f.numDocs = ctx.reader().numDocs();
int maxDoc = ctx.reader().maxDoc();
Bits liveDocs = ctx.reader().getLiveDocs();
FunctionValues fv = vs.getValues(funcContext, ctx);
for (int doc = 0; doc < maxDoc; doc++) {
if (liveDocs != null && !liveDocs.get(doc)) continue;
long v = fv.longVal(doc);
f.maxVersionEncountered = Math.max(v, f.maxVersionEncountered);
if (v <= f.maxVersionSpecified) {
f.maxInHash = Math.max(v, f.maxInHash);
f.versionsHash += Hash.fmix64(v);
f.numVersions++;
}
}
return f;
}
public static IndexFingerprint reduce(IndexFingerprint acc, IndexFingerprint f2) {
// acc should have maxVersionSpecified already set in it using IndexFingerprint(long maxVersionSpecified) constructor
acc.maxDoc = Math.max(acc.maxDoc, f2.maxDoc);
acc.numDocs += f2.numDocs;
acc.maxVersionEncountered = Math.max(acc.maxVersionEncountered, f2.maxVersionEncountered);
acc.maxInHash = Math.max(acc.maxInHash, f2.maxInHash);
acc.versionsHash += f2.versionsHash;
acc.numVersions += f2.numVersions;
return acc;
}
/** returns 0 for equal, negative if f1 is less recent than f2, positive if more recent */
public static int compare(IndexFingerprint f1, IndexFingerprint f2) {
int cmp;
// NOTE: some way want number of docs in index to take precedence over highest version (add-only systems for sure)
// if we're comparing all of the versions in the index, then go by the highest encountered.
if (f1.maxVersionSpecified == Long.MAX_VALUE) {
cmp = Long.compare(f1.maxVersionEncountered, f2.maxVersionEncountered);
if (cmp != 0) return cmp;
}
// Go by the highest version under the requested max.
cmp = Long.compare(f1.maxInHash, f2.maxInHash);
if (cmp != 0) return cmp;
// go by who has the most documents in the index
cmp = Long.compare(f1.numVersions, f2.numVersions);
if (cmp != 0) return cmp;
// both have same number of documents, so go by hash
cmp = Long.compare(f1.versionsHash, f2.versionsHash);
return cmp;
}
@Override
public Map<String, Object> toMap(Map<String, Object> map) {
map.put("maxVersionSpecified", maxVersionSpecified);
map.put("maxVersionEncountered", maxVersionEncountered);
map.put("maxInHash", maxInHash);
map.put("versionsHash", versionsHash);
map.put("numVersions", numVersions);
map.put("numDocs", numDocs);
map.put("maxDoc", maxDoc);
return map;
}
private static long getLong(Map m, String key, long def) {
Object oval = m.get(key);
return oval != null ? ((Number)oval).longValue() : def;
}
/**
* Create an IndexFingerprint object from a deserialized generic object (Map or NamedList)
*/
public static IndexFingerprint fromObject(Object o) {
if (o instanceof IndexFingerprint) return (IndexFingerprint) o;
Map map = null;
if (o instanceof Map) {
map = (Map) o;
} else if (o instanceof NamedList) {
map = ((NamedList) o).asShallowMap();
} else {
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type " + o);
}
IndexFingerprint f = new IndexFingerprint();
f.maxVersionSpecified = getLong(map, "maxVersionSpecified", Long.MAX_VALUE);
f.maxVersionEncountered = getLong(map, "maxVersionEncountered", -1);
f.maxInHash = getLong(map, "maxInHash", -1);
f.versionsHash = getLong(map, "versionsHash", -1);
f.numVersions = getLong(map, "numVersions", -1);
f.numDocs = getLong(map, "numDocs", -1);
f.maxDoc = getLong(map, "maxDoc", -1);
return f;
}
@Override
public String toString() {
return toMap(new LinkedHashMap<>()).toString();
}
}