/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.update.processor;
import java.io.IOException;
import java.io.StringWriter;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import javax.xml.xpath.XPathExpressionException;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.LocalSolrQueryRequest;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.request.SolrRequestHandler;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.servlet.DirectSolrConnection;
import org.apache.solr.update.AddUpdateCommand;
import org.apache.solr.util.BaseTestHarness;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.xml.sax.SAXException;
public class TolerantUpdateProcessorTest extends UpdateProcessorTestBase {
/**
* List of valid + invalid documents
*/
private static List<SolrInputDocument> docs = null;
/**
* IDs of the invalid documents in <code>docs</code>
*/
private static String[] badIds = null;
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig-update-processor-chains.xml", "schema12.xml");
}
@AfterClass
public static void tearDownClass() {
docs = null;
badIds = null;
}
@Override
public void setUp() throws Exception {
super.setUp();
//expected exception messages
ignoreException("Error adding field");
ignoreException("Document is missing mandatory uniqueKey field");
if (docs == null) {
docs = new ArrayList<>(20);
badIds = new String[10];
for(int i = 0; i < 10;i++) {
// a valid document
docs.add(doc(field("id", String.valueOf(2*i)), field("weight", i)));
// ... and an invalid one
docs.add(doc(field("id", String.valueOf(2*i+1)), field("weight", "b")));
badIds[i] = String.valueOf(2*i+1);
}
}
}
@Override
public void tearDown() throws Exception {
resetExceptionIgnores();
assertU(delQ("*:*"));
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='0']");
super.tearDown();
}
/**
* future proof TolerantUpdateProcessor against new default method impls being added to UpdateProcessor
* to ensure that every method involved in a processor chain life cycle is overridden with
* exception catching/tracking.
*/
public void testReflection() {
for (Method method : TolerantUpdateProcessor.class.getMethods()) {
if (method.getDeclaringClass().equals(Object.class) || method.getName().equals("close")) {
continue;
}
assertEquals("base class(es) has changed, TolerantUpdateProcessor needs updated to ensure it " +
"overrides all solr update lifcycle methods with exception tracking: " + method.toString(),
TolerantUpdateProcessor.class, method.getDeclaringClass());
}
}
@Test
public void testValidAdds() throws IOException {
SolrInputDocument validDoc = doc(field("id", "1"), field("text", "the quick brown fox"));
add("tolerant-chain-max-errors-10", null, validDoc);
validDoc = doc(field("id", "2"), field("text", "the quick brown fox"));
add("tolerant-chain-max-errors-not-set", null, validDoc);
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='2']");
assertQ(req("q","id:1")
,"//result[@numFound='1']");
assertQ(req("q","id:2")
,"//result[@numFound='1']");
}
@Test
public void testInvalidAdds() throws IOException {
SolrInputDocument invalidDoc = doc(field("text", "the quick brown fox")); //no id
try {
// This doc should fail without being tolerant
add("not-tolerant", null, invalidDoc);
fail("Expecting exception");
} catch (Exception e) {
//expected
assertTrue(e.getMessage().contains("Document is missing mandatory uniqueKey field"));
}
assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc}), null, "(unknown)");
//a valid doc
SolrInputDocument validDoc = doc(field("id", "1"), field("text", "the quick brown fox"));
try {
// This batch should fail without being tolerant
add("not-tolerant", null, Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc}));
fail("Expecting exception");
} catch (Exception e) {
//expected
assertTrue(e.getMessage().contains("Document is missing mandatory uniqueKey field"));
}
assertU(commit());
assertQ(req("q","id:1")
,"//result[@numFound='0']");
assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc}), null, "(unknown)");
assertU(commit());
// verify that the good document made it in.
assertQ(req("q","id:1")
,"//result[@numFound='1']");
invalidDoc = doc(field("id", "2"), field("weight", "aaa"));
validDoc = doc(field("id", "3"), field("weight", "3"));
try {
// This batch should fail without being tolerant
add("not-tolerant", null, Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc})); //no id
fail("Expecting exception");
} catch (Exception e) {
//expected
assertTrue(e.getMessage().contains("Error adding field"));
}
assertU(commit());
assertQ(req("q","id:3")
,"//result[@numFound='0']");
assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", Arrays.asList(new SolrInputDocument[]{invalidDoc, validDoc}), null, "2");
assertU(commit());
// The valid document was indexed
assertQ(req("q","id:3")
,"//result[@numFound='1']");
// The invalid document was NOT indexed
assertQ(req("q","id:2")
,"//result[@numFound='0']");
}
@Test
public void testMaxErrorsDefault() throws IOException {
try {
// by default the TolerantUpdateProcessor accepts all errors, so this batch should succeed with 10 errors.
assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, null, badIds);
} catch(Exception e) {
fail("Shouldn't get an exception for this batch: " + e.getMessage());
}
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='10']");
}
public void testMaxErrorsSucceed() throws IOException {
ModifiableSolrParams requestParams = new ModifiableSolrParams();
requestParams.add("maxErrors", "10");
// still OK
assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, requestParams, badIds);
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='10']");
}
@Test
public void testMaxErrorsThrowsException() throws IOException {
ModifiableSolrParams requestParams = new ModifiableSolrParams();
requestParams.add("maxErrors", "5");
try {
// should fail
assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, requestParams, badIds);
fail("Expecting exception");
} catch (SolrException e) {
assertTrue(e.getMessage(),
e.getMessage().contains("ERROR: [doc=1] Error adding field 'weight'='b' msg=For input string: \"b\""));
}
//the first good documents made it to the index
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='6']");
}
@Test
public void testMaxErrorsInfinite() throws IOException {
ModifiableSolrParams requestParams = new ModifiableSolrParams();
requestParams.add("maxErrors", "-1");
try {
assertAddsSucceedWithErrors("tolerant-chain-max-errors-not-set", docs, null, badIds);
} catch(Exception e) {
fail("Shouldn't get an exception for this batch: " + e.getMessage());
}
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='10']");
}
@Test
public void testMaxErrors0() throws IOException {
//make the TolerantUpdateProcessor intolerant
List<SolrInputDocument> smallBatch = docs.subList(0, 2);
ModifiableSolrParams requestParams = new ModifiableSolrParams();
requestParams.add("maxErrors", "0");
try {
// should fail
assertAddsSucceedWithErrors("tolerant-chain-max-errors-10", smallBatch, requestParams, "1");
fail("Expecting exception");
} catch (SolrException e) {
assertTrue(e.getMessage().contains("ERROR: [doc=1] Error adding field 'weight'='b' msg=For input string: \"b\""));
}
//the first good documents made it to the index
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='1']");
}
@Test
public void testInvalidDelete() throws XPathExpressionException, SAXException {
ignoreException("undefined field invalidfield");
String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox"));
assertNull(BaseTestHarness.validateXPath(response,
"//int[@name='status']=0",
"//arr[@name='errors']",
"count(//arr[@name='errors']/lst)=0"));
response = update("tolerant-chain-max-errors-10", delQ("invalidfield:1"));
assertNull(BaseTestHarness.validateXPath
(response,
"//int[@name='status']=0",
"count(//arr[@name='errors']/lst)=1",
"//arr[@name='errors']/lst/str[@name='type']/text()='DELQ'",
"//arr[@name='errors']/lst/str[@name='id']/text()='invalidfield:1'",
"//arr[@name='errors']/lst/str[@name='message']/text()='undefined field invalidfield'"));
}
@Test
public void testValidDelete() throws XPathExpressionException, SAXException {
ignoreException("undefined field invalidfield");
String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox"));
assertNull(BaseTestHarness.validateXPath(response,
"//int[@name='status']=0",
"//arr[@name='errors']",
"count(//arr[@name='errors']/lst)=0"));
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='1']");
response = update("tolerant-chain-max-errors-10", delQ("id:1"));
assertNull(BaseTestHarness.validateXPath(response,
"//int[@name='status']=0",
"//arr[@name='errors']",
"count(//arr[@name='errors']/lst)=0"));
assertU(commit());
assertQ(req("q","*:*")
,"//result[@numFound='0']");
}
@Test
public void testResponse() throws SAXException, XPathExpressionException, IOException {
String response = update("tolerant-chain-max-errors-10", adoc("id", "1", "text", "the quick brown fox"));
assertNull(BaseTestHarness.validateXPath(response,
"//int[@name='status']=0",
"//arr[@name='errors']",
"count(//arr[@name='errors']/lst)=0"));
response = update("tolerant-chain-max-errors-10", adoc("text", "the quick brown fox"));
assertNull(BaseTestHarness.validateXPath(response, "//int[@name='status']=0",
"//int[@name='maxErrors']/text()='10'",
"count(//arr[@name='errors']/lst)=1",
"//arr[@name='errors']/lst/str[@name='id']/text()='(unknown)'",
"//arr[@name='errors']/lst/str[@name='message']/text()='Document is missing mandatory uniqueKey field: id'"));
response = update("tolerant-chain-max-errors-10", adoc("text", "the quick brown fox"));
StringWriter builder = new StringWriter();
builder.append("<add>");
for (SolrInputDocument doc:docs) {
ClientUtils.writeXML(doc, builder);
}
builder.append("</add>");
response = update("tolerant-chain-max-errors-10", builder.toString());
assertNull(BaseTestHarness.validateXPath(response, "//int[@name='status']=0",
"//int[@name='maxErrors']/text()='10'",
"count(//arr[@name='errors']/lst)=10",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='0')",
"//arr[@name='errors']/lst/str[@name='id']/text()='1'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='2')",
"//arr[@name='errors']/lst/str[@name='id']/text()='3'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='4')",
"//arr[@name='errors']/lst/str[@name='id']/text()='5'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='6')",
"//arr[@name='errors']/lst/str[@name='id']/text()='7'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='8')",
"//arr[@name='errors']/lst/str[@name='id']/text()='9'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='10')",
"//arr[@name='errors']/lst/str[@name='id']/text()='11'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='12')",
"//arr[@name='errors']/lst/str[@name='id']/text()='13'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='14')",
"//arr[@name='errors']/lst/str[@name='id']/text()='15'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='16')",
"//arr[@name='errors']/lst/str[@name='id']/text()='17'",
"not(//arr[@name='errors']/lst/str[@name='id']/text()='18')",
"//arr[@name='errors']/lst/str[@name='id']/text()='19'"));
// spot check response when effective maxErrors is unlimited
response = update("tolerant-chain-max-errors-not-set", builder.toString());
assertNull(BaseTestHarness.validateXPath(response, "//int[@name='maxErrors']/text()='-1'"));
}
public String update(String chain, String xml) {
DirectSolrConnection connection = new DirectSolrConnection(h.getCore());
SolrRequestHandler handler = h.getCore().getRequestHandler("/update");
ModifiableSolrParams params = new ModifiableSolrParams();
params.add("update.chain", chain);
try {
return connection.request(handler, params, xml);
} catch (SolrException e) {
throw (SolrException)e;
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
}
private void assertAddsSucceedWithErrors(String chain,
final Collection<SolrInputDocument> docs,
SolrParams requestParams,
String... idsShouldFail) throws IOException {
SolrQueryResponse response = add(chain, requestParams, docs);
@SuppressWarnings("unchecked")
List<SimpleOrderedMap<String>> errors = (List<SimpleOrderedMap<String>>)
response.getResponseHeader().get("errors");
assertNotNull(errors);
assertEquals("number of errors", idsShouldFail.length, errors.size());
Set<String> addErrorIdsExpected = new HashSet<String>(Arrays.asList(idsShouldFail));
for (SimpleOrderedMap<String> err : errors) {
assertEquals("this method only expects 'add' errors", "ADD", err.get("type"));
String id = err.get("id");
assertNotNull("null err id", id);
assertTrue("unexpected id", addErrorIdsExpected.contains(id));
}
}
protected SolrQueryResponse add(final String chain, SolrParams requestParams, final SolrInputDocument doc) throws IOException {
return add(chain, requestParams, Arrays.asList(new SolrInputDocument[]{doc}));
}
protected SolrQueryResponse add(final String chain, SolrParams requestParams, final Collection<SolrInputDocument> docs) throws IOException {
SolrCore core = h.getCore();
UpdateRequestProcessorChain pc = core.getUpdateProcessingChain(chain);
assertNotNull("No Chain named: " + chain, pc);
SolrQueryResponse rsp = new SolrQueryResponse();
rsp.add("responseHeader", new SimpleOrderedMap<Object>());
if(requestParams == null) {
requestParams = new ModifiableSolrParams();
}
SolrQueryRequest req = new LocalSolrQueryRequest(core, requestParams);
UpdateRequestProcessor processor = null;
try {
processor = pc.createProcessor(req, rsp);
for(SolrInputDocument doc:docs) {
AddUpdateCommand cmd = new AddUpdateCommand(req);
cmd.solrDoc = doc;
processor.processAdd(cmd);
}
processor.finish();
} finally {
IOUtils.closeQuietly(processor);
req.close();
}
return rsp;
}
}