/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.search.stats;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.CompositeIdRouter;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestDistribIDF extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private MiniSolrCloudCluster solrCluster;
@Override
public void setUp() throws Exception {
if (random().nextBoolean()) {
System.setProperty("solr.statsCache", ExactStatsCache.class.getName());
} else {
System.setProperty("solr.statsCache", LRUStatsCache.class.getName());
}
super.setUp();
solrCluster = new MiniSolrCloudCluster(3, createTempDir(), buildJettyConfig("/solr"));
// set some system properties for use by tests
System.setProperty("solr.test.sys.prop1", "propone");
System.setProperty("solr.test.sys.prop2", "proptwo");
solrCluster.uploadConfigSet(TEST_PATH().resolve("collection1/conf"), "conf1");
solrCluster.uploadConfigSet(configset("configset-2"), "conf2");
}
@Override
public void tearDown() throws Exception {
solrCluster.shutdown();
System.clearProperty("solr.statsCache");
System.clearProperty("solr.test.sys.prop1");
System.clearProperty("solr.test.sys.prop2");
super.tearDown();
}
@Test
public void testSimpleQuery() throws Exception {
//3 shards. 3rd shard won't have any data.
createCollection("onecollection", "conf1", ImplicitDocRouter.NAME);
createCollection("onecollection_local", "conf2", ImplicitDocRouter.NAME);
SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", 1);
doc.setField("cat", "football");
doc.addField(ShardParams._ROUTE_, "a");
solrCluster.getSolrClient().add("onecollection", doc);
solrCluster.getSolrClient().add("onecollection_local", doc);
doc = new SolrInputDocument();
doc.setField("id", 2);
doc.setField("cat", "football");
doc.addField(ShardParams._ROUTE_, "b");
solrCluster.getSolrClient().add("onecollection", doc);
solrCluster.getSolrClient().add("onecollection_local", doc);
int nDocs = TestUtil.nextInt(random(), 10, 100);
for (int i=0; i<nDocs; i++) {
doc = new SolrInputDocument();
doc.setField("id", 3 + i);
String cat = TestUtil.randomSimpleString(random());
if (!cat.equals("football")) { //Making sure no other document has the query term in it.
doc.setField("cat", cat);
if (rarely()) { //Put most documents in shard b so that 'football' becomes 'rare' in shard b
doc.addField(ShardParams._ROUTE_, "a");
} else {
doc.addField(ShardParams._ROUTE_, "b");
}
solrCluster.getSolrClient().add("onecollection", doc);
solrCluster.getSolrClient().add("onecollection_local", doc);
}
}
solrCluster.getSolrClient().commit("onecollection");
solrCluster.getSolrClient().commit("onecollection_local");
//Test against all nodes
for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) {
try (SolrClient solrClient = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) {
try (SolrClient solrClient_local = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) {
SolrQuery query = new SolrQuery("cat:football");
query.setFields("*,score");
QueryResponse queryResponse = solrClient.query("onecollection", query);
assertEquals(2, queryResponse.getResults().getNumFound());
float score1 = (float) queryResponse.getResults().get(0).get("score");
float score2 = (float) queryResponse.getResults().get(1).get("score");
assertEquals("Doc1 score=" + score1 + " Doc2 score=" + score2, 0, Float.compare(score1, score2));
query = new SolrQuery("cat:football");
query.setShowDebugInfo(true);
query.setFields("*,score");
queryResponse = solrClient_local.query("onecollection_local", query);
assertEquals(2, queryResponse.getResults().getNumFound());
assertEquals(2, queryResponse.getResults().get(0).get("id"));
assertEquals(1, queryResponse.getResults().get(1).get("id"));
float score1_local = (float) queryResponse.getResults().get(0).get("score");
float score2_local = (float) queryResponse.getResults().get(1).get("score");
assertEquals("Doc1 score=" + score1_local + " Doc2 score=" + score2_local, 1,
Float.compare(score1_local, score2_local));
}
}
}
}
@Test
public void testMultiCollectionQuery() throws Exception {
// collection1 and collection2 are collections which have distributed idf enabled
// collection1_local and collection2_local don't have distributed idf available
// Only one doc has cat:football in each collection
// When doing queries across collections we want to test that the query takes into account
// distributed idf for the collection=collection1,collection2 query.
// The way we verify is that score should be the same when querying across collection1 and collection2
// But should be different when querying across collection1_local and collection2_local
// since the idf is calculated per shard
createCollection("collection1", "conf1");
createCollection("collection1_local", "conf2");
createCollection("collection2", "conf1");
createCollection("collection2_local", "conf2");
addDocsRandomly();
//Test against all nodes
for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) {
try (SolrClient solrClient = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) {
try (SolrClient solrClient_local = getHttpSolrClient(jettySolrRunner.getBaseUrl().toString())) {
SolrQuery query = new SolrQuery("cat:football");
query.setFields("*,score").add("collection", "collection1,collection2");
QueryResponse queryResponse = solrClient.query("collection1", query);
assertEquals(2, queryResponse.getResults().getNumFound());
float score1 = (float) queryResponse.getResults().get(0).get("score");
float score2 = (float) queryResponse.getResults().get(1).get("score");
assertEquals("Doc1 score=" + score1 + " Doc2 score=" + score2, 0, Float.compare(score1, score2));
query = new SolrQuery("cat:football");
query.setFields("*,score").add("collection", "collection1_local,collection2_local");
queryResponse = solrClient_local.query("collection1_local", query);
assertEquals(2, queryResponse.getResults().getNumFound());
assertEquals(2, queryResponse.getResults().get(0).get("id"));
assertEquals(1, queryResponse.getResults().get(1).get("id"));
float score1_local = (float) queryResponse.getResults().get(0).get("score");
float score2_local = (float) queryResponse.getResults().get(1).get("score");
assertEquals("Doc1 score=" + score1_local + " Doc2 score=" + score2_local, 1,
Float.compare(score1_local, score2_local));
}
}
}
}
private void createCollection(String name, String config) throws Exception {
createCollection(name, config, CompositeIdRouter.NAME);
}
private void createCollection(String name, String config, String router) throws Exception {
CollectionAdminResponse response;
if (router.equals(ImplicitDocRouter.NAME)) {
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setConfigName(config);
create.setCollectionName(name);
create.setReplicationFactor(1);
create.setMaxShardsPerNode(1);
create.setRouterName(router);
create.setShards("a,b,c");
response = create.process(solrCluster.getSolrClient());
} else {
CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
create.setConfigName(config);
create.setCollectionName(name);
create.setNumShards(2);
create.setReplicationFactor(1);
create.setMaxShardsPerNode(1);
response = create.process(solrCluster.getSolrClient());
}
if (response.getStatus() != 0 || response.getErrorMessages() != null) {
fail("Could not create collection. Response" + response.toString());
}
ZkStateReader zkStateReader = solrCluster.getSolrClient().getZkStateReader();
AbstractDistribZkTestBase.waitForRecoveriesToFinish(name, zkStateReader, false, true, 100);
}
private void addDocsRandomly() throws IOException, SolrServerException {
SolrInputDocument doc = new SolrInputDocument();
doc.setField("id", 1);
doc.setField("cat", "football");
solrCluster.getSolrClient().add("collection1", doc);
solrCluster.getSolrClient().add("collection1_local", doc);
doc = new SolrInputDocument();
doc.setField("id", 2);
doc.setField("cat", "football");
solrCluster.getSolrClient().add("collection2", doc);
solrCluster.getSolrClient().add("collection2_local", doc);
int nDocs = TestUtil.nextInt(random(), 10, 100);
int collection1Count = 1;
int collection2Count = 1;
for (int i=0; i<nDocs; i++) {
doc = new SolrInputDocument();
doc.setField("id", 3 + i);
String cat = TestUtil.randomSimpleString(random());
if (!cat.equals("football")) { //Making sure no other document has the query term in it.
doc.setField("cat", cat);
if (rarely()) { //Put most documents in collection2* so that 'football' becomes 'rare' in collection2*
solrCluster.getSolrClient().add("collection1", doc);
solrCluster.getSolrClient().add("collection1_local", doc);
collection1Count++;
} else {
solrCluster.getSolrClient().add("collection2", doc);
solrCluster.getSolrClient().add("collection2_local", doc);
collection2Count++;
}
}
}
log.info("numDocs={}. collection1Count={} collection2Count={}", nDocs, collection1Count, collection2Count);
solrCluster.getSolrClient().commit("collection1");
solrCluster.getSolrClient().commit("collection2");
solrCluster.getSolrClient().commit("collection1_local");
solrCluster.getSolrClient().commit("collection2_local");
}
}