/**
* ServerShard
* Copyright 2013 by Michael Peter Christen
* First released 13.02.2013 at http://yacy.net
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package net.yacy.cora.federate.solr.instance;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.concurrent.ConcurrentLinkedQueue;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.StreamingResponseCallback;
import org.apache.solr.client.solrj.SolrRequest.METHOD;
import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.client.solrj.response.UpdateResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import net.yacy.cora.federate.solr.connector.ShardSelection;
public class ServerShard extends SolrClient {
private static final long serialVersionUID = -3524175189049786312L;
private static final UpdateResponse _dummyOKResponse = new UpdateResponse();
static {
_dummyOKResponse.setElapsedTime(0);
_dummyOKResponse.setResponse(new NamedList<Object>());
}
private final ShardSelection shards;
private final boolean writeEnabled;
public ServerShard(ArrayList<SolrClient> server, final ShardSelection.Method method, final boolean writeEnabled) {
this.shards = new ShardSelection(server, method);
this.writeEnabled = writeEnabled;
}
/**
* Adds a collection of documents
* @param docs the collection of documents
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrInputDocument doc: docs) ur = this.shards.server4write(doc).add(doc);
return ur; // TODO: this accumlation of update responses is wrong, but sufficient (because we do not evaluate it)
}
/**
* Adds a collection of documents, specifying max time before they become committed
* @param docs the collection of documents
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since solr 3.5
*/
@Override
public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrInputDocument doc: docs) ur = this.shards.server4write(doc).add(doc, commitWithinMs);
return ur;
}
/**
* Adds a collection of beans
* @param beans the collection of beans
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse addBeans(Collection<?> beans ) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.addBeans(beans);
return ur;
}
/**
* Adds a collection of beans specifying max time before they become committed
* @param beans the collection of beans
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since solr 3.5
*/
@Override
public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.addBeans(beans, commitWithinMs);
return ur;
}
/**
* Adds a single document
* @param doc the input document
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
return this.shards.server4write(doc).add(doc);
}
/**
* Adds a single document specifying max time before it becomes committed
* @param doc the input document
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since solr 3.5
*/
@Override
public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
return this.shards.server4write(doc).add(doc, commitWithinMs);
}
/**
* Adds a single bean
* @param obj the input bean
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.addBean(obj);
return ur;
}
/**
* Adds a single bean specifying max time before it becomes committed
* @param obj the input bean
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since solr 3.5
*/
@Override
public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.addBean(obj, commitWithinMs);
return ur;
}
/**
* Performs an explicit commit, causing pending documents to be committed for indexing
* <p>
* waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse commit() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.commit();
return ur;
}
/**
* Performs an explicit optimize, causing a merge of all segments to one.
* <p>
* waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
* <p>
* Note: In most cases it is not required to do explicit optimize
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse optimize() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.optimize();
return ur;
}
/**
* Performs an explicit commit, causing pending documents to be committed for indexing
* @param waitFlush block until index changes are flushed to disk
* @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.commit(waitFlush, waitSearcher);
return ur;
}
/**
* Performs an explicit commit, causing pending documents to be committed for indexing
* @param waitFlush block until index changes are flushed to disk
* @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
* @param softCommit makes index changes visible while neither fsync-ing index files nor writing a new index descriptor
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.commit(waitFlush, waitSearcher, softCommit);
return ur;
}
/**
* Performs an explicit optimize, causing a merge of all segments to one.
* <p>
* Note: In most cases it is not required to do explicit optimize
* @param waitFlush block until index changes are flushed to disk
* @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.optimize(waitFlush, waitSearcher);
return ur;
}
/**
* Performs an explicit optimize, causing a merge of all segments to one.
* <p>
* Note: In most cases it is not required to do explicit optimize
* @param waitFlush block until index changes are flushed to disk
* @param waitSearcher block until a new searcher is opened and registered as the main query searcher, making the changes visible
* @param maxSegments optimizes down to at most this number of segments
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.optimize(waitFlush, waitSearcher, maxSegments);
return ur;
}
/**
* Performs a rollback of all non-committed documents pending.
* <p>
* Note that this is not a true rollback as in databases. Content you have previously
* added may have been committed due to autoCommit, buffer full, other client performing
* a commit etc.
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse rollback() throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards) ur = s.rollback();
return ur;
}
/**
* Deletes a single document by unique ID
* @param id the ID of the document to delete
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(id);
return ur;
}
/**
* Deletes a single document by unique ID, specifying max time before commit
* @param id the ID of the document to delete
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since 3.6
*/
@Override
public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(id, commitWithinMs);
return ur;
}
/**
* Deletes a list of documents by unique ID
* @param ids the list of document IDs to delete
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(ids);
return ur;
}
/**
* Deletes a list of documents by unique ID, specifying max time before commit
* @param ids the list of document IDs to delete
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since 3.6
*/
@Override
public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteById(ids, commitWithinMs);
return ur;
}
/**
* Deletes documents from the index based on a query
* @param query the query expressing what documents to delete
* @throws IOException If there is a low-level I/O error.
*/
@Override
public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteByQuery(query);
return ur;
}
/**
* Deletes documents from the index based on a query, specifying max time before commit
* @param query the query expressing what documents to delete
* @param commitWithinMs max time (in ms) before a commit will happen
* @throws IOException If there is a low-level I/O error.
* @since 3.6
*/
@Override
public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
if (!this.writeEnabled) return _dummyOKResponse;
UpdateResponse ur = null;
for (SolrClient s: this.shards.server4read()) ur = s.deleteByQuery(query, commitWithinMs);
return ur;
}
/**
* Issues a ping request to check if the server is alive
* @throws IOException If there is a low-level I/O error.
*/
@Override
public SolrPingResponse ping() throws SolrServerException, IOException {
for (SolrClient s: this.shards) {
SolrPingResponse spr = s.ping();
if (spr != null) return spr;
}
return null;
}
/**
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @throws IOException
*/
@Override
public QueryResponse query(final SolrParams params) throws SolrServerException, IOException {
List<SolrClient> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params);
}
// concurrently call all shards
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
List<Thread> t = new ArrayList<Thread>();
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
@Override
public void run() {
this.setName("ServerShard.query/1(" + params.toString() + ")");
QueryResponse rsp;
try {
rsp = s.query(params);
} catch (final Throwable e) {return;}
qrl.add(rsp);
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) {
try {t0.join();} catch (final InterruptedException e) {}
}
// prepare combined response
return ResponseAccumulator.combineResponses(qrl);
}
/**
* Performs a query to the Solr server
* @param params an object holding all key/value parameters to send along the request
* @param method specifies the HTTP method to use for the request, such as GET or POST
* @throws IOException
*/
@Override
public QueryResponse query(final SolrParams params, final METHOD method) throws SolrServerException, IOException {
List<SolrClient> qs = this.shards.server4read();
if (qs.size() == 1) {
return qs.get(0).query(params, method);
}
final Collection<QueryResponse> qrl = new ConcurrentLinkedQueue<QueryResponse>();
// concurrently call all shards
List<Thread> t = new ArrayList<Thread>();
for (final SolrClient s: qs) {
Thread t0 = new Thread() {
@Override
public void run() {
this.setName("ServerShard.query/2(" + params.toString() + ")");
QueryResponse rsp;
try {
rsp = s.query(params, method);
} catch (final Throwable e) {return;}
qrl.add(rsp);
}
};
t0.start();
t.add(t0);
}
for (Thread t0: t) {
try {t0.join();} catch (final InterruptedException e) {}
}
// prepare combined response
return ResponseAccumulator.combineResponses(qrl);
}
/**
* Query solr, and stream the results. Unlike the standard query, this will
* send events for each Document rather then add them to the QueryResponse.
*
* Although this function returns a 'QueryResponse' it should be used with care
* since it excludes anything that was passed to callback. Also note that
* future version may pass even more info to the callback and may not return
* the results in the QueryResponse.
*
* @since solr 4.0
*/
@Override
public QueryResponse queryAndStreamResponse( SolrParams params, StreamingResponseCallback callback ) throws SolrServerException, IOException {
throw new UnsupportedOperationException("stream response not possible with shards");
}
/**
* SolrServer implementations need to implement how a request is actually processed
*/
@Override
public NamedList<Object> request(@SuppressWarnings("rawtypes") SolrRequest request, String collection) throws SolrServerException, IOException {
ResponseAccumulator acc = new ResponseAccumulator();
for (SolrClient s: this.shards.server4read()) acc.addResponse(s.request(request, collection));
return acc.getAccumulatedResponse();
}
@Override
public DocumentObjectBinder getBinder() {
DocumentObjectBinder db;
for (SolrClient s: this.shards) {
db = s.getBinder();
if (db != null) return db;
}
return null;
}
@Override
public void shutdown() {
for (SolrClient s: this.shards) {
try {
s.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}