/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.blur.slur;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import java.util.Collection;
import java.util.List;
import org.apache.blur.MiniCluster;
import org.apache.blur.thirdparty.thrift_0_9_0.TException;
import org.apache.blur.thrift.BlurClient;
import org.apache.blur.thrift.generated.Blur;
import org.apache.blur.thrift.generated.Blur.Iface;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.blur.thrift.generated.BlurQuery;
import org.apache.blur.thrift.generated.BlurResults;
import org.apache.blur.thrift.generated.Query;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.thrift.generated.TableStats;
import org.apache.blur.utils.BlurConstants;
import org.apache.blur.utils.GCWatcher;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.Lists;
public class SolrLookingBlurServerTest {
private static final File TMPDIR = new File(System.getProperty("blur.tmp.dir", "./target/tmp_BlurClusterTest"));
private static MiniCluster miniCluster;
private static String connectionStr;
@BeforeClass
public static void startCluster() throws IOException {
GCWatcher.init(0.60);
LocalFileSystem localFS = FileSystem.getLocal(new Configuration());
File testDirectory = new File(TMPDIR, "blur-cluster-test").getAbsoluteFile();
testDirectory.mkdirs();
Path directory = new Path(testDirectory.getPath());
FsPermission dirPermissions = localFS.getFileStatus(directory).getPermission();
FsAction userAction = dirPermissions.getUserAction();
FsAction groupAction = dirPermissions.getGroupAction();
FsAction otherAction = dirPermissions.getOtherAction();
StringBuilder builder = new StringBuilder();
builder.append(userAction.ordinal());
builder.append(groupAction.ordinal());
builder.append(otherAction.ordinal());
String dirPermissionNum = builder.toString();
System.setProperty("dfs.datanode.data.dir.perm", dirPermissionNum);
testDirectory.delete();
miniCluster = new MiniCluster();
miniCluster.startBlurCluster(new File(testDirectory, "cluster").getAbsolutePath(), 2, 3, true);
connectionStr = miniCluster.getControllerConnectionStr();
}
@AfterClass
public static void shutdownCluster() {
miniCluster.shutdownBlurCluster();
}
@Test
public void childDocsShouldBecomeRecordsOfRow() throws Exception {
String table = "childDocsShouldBecomeRecordsOfRow";
TestTableCreator.newTable(table)
.withRowCount(1).withRecordsPerRow(100)
.withRecordColumns("fam.value").create();
TableStats stats = client().tableStats(table);
assertEquals("We should have one record.", 100, stats.recordCount);
assertEquals("We should have one row.", 1, stats.rowCount);
assertTotalResults(table, "fam.value:value0-0", 1l);
assertTotalRecordResults(table, "recordid:99", 1l);
removeTable(table);
}
@Test
public void docShouldBeDiscoverableWithMultiValuedFields() throws SolrServerException, IOException, BlurException,
TException {
String table = "docShouldBeDiscoverableWithMultiValuedFields";
createTable(table);
SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
SolrInputDocument doc = new SolrInputDocument();
doc.addField("rowid", "1");
SolrInputDocument child = new SolrInputDocument();
child.addField("recordid", "1");
child.addField("fam.value", "123");
child.addField("fam.value", "124");
doc.addChildDocument(child);
server.add(doc);
assertTotalResults(table, "fam.value:123", 1l);
assertTotalResults(table, "fam.value:124", 1l);
assertTotalResults(table, "fam.value:justincase", 0l);
removeTable(table);
}
@Test
public void documentsShouldBeAbleToBeIndexedInBatch() throws Exception,
TException {
String table = "multipleDocumentsShouldBeIndexed";
TestTableCreator.newTable(table)
.withRowCount(100).withRecordsPerRow(1)
.withRecordColumns("fam.value").create();
assertTotalResults(table, "fam.value:value0-0", 1l);
assertTotalResults(table, "fam.value:value1-0", 1l);
assertTotalResults(table, "rowid:1", 1l);
assertTotalResults(table, "rowid:2", 1l);
assertTotalResults(table, "fam.value:value99-0", 1l);
assertTotalResults(table, "fam.value:justincase", 0l);
removeTable(table);
}
@Test
public void weShouldBeAbleToDeleteARowById() throws Exception,
TException {
String table = "weShouldBeAbleToDeleteARowById";
SolrServer server = TestTableCreator.newTable(table).withRowCount(2).withRecordsPerRow(1)
.withRecordColumns("fam.value").create();
assertTotalResults(table, "rowid:0", 1l);
assertTotalResults(table, "rowid:1", 1l);
server.deleteById("1");
assertTotalResults(table, "rowid:0", 1l);
assertTotalResults(table, "rowid:1", 0l);
removeTable(table);
}
@Test
public void weShouldBeAbleToDeleteARowByAListOfIds() throws Exception,
TException {
String table = "weShouldBeAbleToDeleteARowByAListOfIds";
SolrServer server = TestTableCreator.newTable(table).withRowCount(20).withRecordsPerRow(1)
.withRecordColumns("fam.value").create();
assertTotalResults(table, "rowid:1", 1l);
assertTotalResults(table, "rowid:2", 1l);
List<String> ids = Lists.newArrayList("1", "2", "3", "4", "5");
server.deleteById(ids);
for (String id : ids) {
assertTotalResults(table, "rowid:" + id, 0l);
}
removeTable(table);
}
@Test
public void basicFullTextQuery() throws Exception {
String table = "basicFullTextQuery";
SolrServer server = TestTableCreator.newTable(table)
.withRowCount(1).withRecordsPerRow(2)
.withRecordColumns("fam.value", "fam.mvf", "fam.mvf").create();
SolrQuery query = new SolrQuery("value0-0");
QueryResponse response = server.query(query);
assertEquals("We should get our doc back.", 1l, response.getResults().getNumFound());
SolrDocument docResult = response.getResults().get(0);
assertEquals("0", docResult.getFieldValue("recordid"));
assertEquals("value0-0", docResult.getFieldValue("fam.value"));
Collection<Object> mvfVals = docResult.getFieldValues("fam.mvf");
assertTrue("We should get all our values back[" + mvfVals + "]",
CollectionUtils.isEqualCollection(mvfVals, Lists.newArrayList("value0-0", "value0-0")));
removeTable(table);
}
@Test
public void fieldsRequestsShouldTurnIntoSelectors() throws Exception,
TException {
String table = "fieldsRequestsShouldTurnIntoSelectors";
SolrServer server = TestTableCreator.newTable(table)
.withRowCount(1).withRecordsPerRow(2)
.withRecordColumns("fam.value", "fam.mvf").create();
SolrQuery query = new SolrQuery("value0-0");
query.setFields("fam.value");
QueryResponse response = server.query(query);
assertEquals("We should get our doc back for a valid test.", 1l, response.getResults().getNumFound());
SolrDocument docResult = response.getResults().get(0);
assertEquals("value0-0", docResult.getFieldValue("fam.value"));
assertNull("We shouldn't get this one back since it wasnt in our fields.", docResult.getFieldValues("fam.mvf"));
removeTable(table);
}
@Test
public void weShouldBeAbleToPageResults() throws SolrServerException, IOException, BlurException, TException {
String table = "weShouldBeAbleToPageResults";
SolrServer server = createServerAndTableWithSimpleTestDoc(table);
SolrQuery query = new SolrQuery("123");
query.setFields("fam.value");
QueryResponse response = server.query(query);
assertEquals("We should get our doc back for a valid test.", 1l, response.getResults().getNumFound());
SolrDocument docResult = response.getResults().get(0);
assertEquals("123", docResult.getFieldValue("fam.value"));
assertNull("We shouldn't get this one back since it wasnt in our fields.", docResult.getFieldValues("fam.mvf"));
removeTable(table);
}
private SolrServer createServerAndTableWithSimpleTestDoc(String table) throws BlurException, TException, IOException,
SolrServerException {
createTable(table);
SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), table);
SolrInputDocument doc;
doc = createSimpleTestDoc();
server.add(doc);
return server;
}
private SolrInputDocument createSimpleTestDoc() {
SolrInputDocument doc;
doc = new SolrInputDocument();
doc.addField("rowid", "1");
SolrInputDocument child = new SolrInputDocument();
child.addField("recordid", "1");
child.addField("fam.value", "123");
child.addField("fam.mvf", "aaa");
child.addField("fam.mvf", "bbb");
doc.addChildDocument(child);
return doc;
}
private void assertTotalResults(String table, String q, long expected) throws BlurException, TException {
BlurQuery bquery = new BlurQuery();
Query query = new Query();
query.setQuery(q);
bquery.setQuery(query);
BlurResults results = client().query(table, bquery);
assertEquals("Should find our row.", expected, results.getTotalResults());
}
private void assertTotalRecordResults(String table, String q, long expected) throws BlurException, TException {
BlurQuery bquery = new BlurQuery();
Query query = new Query();
query.setQuery(q);
query.setRowQuery(false);
bquery.setQuery(query);
BlurResults results = client().query(table, bquery);
assertEquals("Should find our record.", expected, results.getTotalResults());
}
private void removeTable(String table) {
try {
client().disableTable(table);
} catch (Exception e) {
}
try {
client().removeTable(table, true);
} catch (Exception e) {
}
}
private static void createTable(String tableName) throws BlurException, TException, IOException {
Blur.Iface client = client();
TableDescriptor tableDescriptor = new TableDescriptor();
tableDescriptor.setName(tableName);
tableDescriptor.setShardCount(5);
tableDescriptor.setTableUri(miniCluster.getFileSystemUri().toString() + "/blur/" + tableName);
client.createTable(tableDescriptor);
List<String> tableList = client.tableList();
assertTrue(tableList.contains(tableName));
}
private static Iface client() {
return BlurClient.getClient(connectionStr);
}
private static class TestTableCreator {
private int rows = 1;
private int recordsPerRow = 10;
private String tableName;
private String[] columns = { "fam.value" };
private TestTableCreator(String table) {
this.tableName = table;
}
public static TestTableCreator newTable(String table) {
return new TestTableCreator(table);
}
public TestTableCreator withRowCount(int rows) {
this.rows = rows;
return this;
}
public TestTableCreator withRecordsPerRow(int count) {
this.recordsPerRow = count;
return this;
}
public TestTableCreator withRecordColumns(String... cols) {
this.columns = cols;
return this;
}
public SolrServer create() throws Exception {
createTable(tableName);
SolrServer server = new SolrLookingBlurServer(miniCluster.getControllerConnectionStr(), tableName);
for (int i = 0; i < rows; i++) {
SolrInputDocument parent = new SolrInputDocument();
parent.addField(BlurConstants.ROW_ID, i);
for (int j = 0; j < recordsPerRow; j++) {
SolrInputDocument child = new SolrInputDocument();
child.addField(BlurConstants.RECORD_ID, j);
for (String colName : columns) {
child.addField(colName, "value" + i + "-" + j);
}
parent.addChildDocument(child);
}
server.add(parent);
}
return server;
}
}
}