/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.client.solrj;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.lang.invoke.MethodHandles;
import java.net.HttpURLConnection;
import java.net.Socket;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.IOUtils;
import org.apache.solr.SolrJettyTestBase;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.impl.BinaryRequestWriter;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.RequestWriter;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776")
public class TestSolrJErrorHandling extends SolrJettyTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
List<Throwable> unexpected = new CopyOnWriteArrayList<>();
@BeforeClass
public static void beforeTest() throws Exception {
createJetty(legacyExampleCollection1SolrHome());
}
@Override
public void setUp() throws Exception {
super.setUp();
unexpected.clear();
}
public String getChain(Throwable th) {
StringBuilder sb = new StringBuilder(40);
Throwable lastCause = null;
do {
if (lastCause != null) sb.append("->");
sb.append(th.getClass().getSimpleName());
lastCause = th;
th = th.getCause();
} while(th != null);
sb.append("(" + lastCause.getMessage() + ")");
return sb.toString();
}
public void showExceptions() throws Exception {
if (unexpected.isEmpty()) return;
Map<String,Integer> counts = new HashMap<>();
// dedup in case there are many clients or many exceptions
for (Throwable e : unexpected) {
String chain = getChain(e);
Integer prev = counts.put(chain, 1);
if (prev != null) {
counts.put(chain, prev+1);
}
}
StringBuilder sb = new StringBuilder("EXCEPTION LIST:");
for (Map.Entry<String,Integer> entry : counts.entrySet()) {
sb.append("\n\t").append(entry.getValue()).append(") ").append(entry.getKey());
}
log.error(sb.toString());
}
@Test
public void testWithXml() throws Exception {
HttpSolrClient client = (HttpSolrClient) getSolrClient();
client.setRequestWriter(new RequestWriter());
client.deleteByQuery("*:*"); // delete everything!
doIt(client);
}
@Test
public void testWithBinary() throws Exception {
HttpSolrClient client = (HttpSolrClient) getSolrClient();
client.setRequestWriter(new BinaryRequestWriter());
client.deleteByQuery("*:*"); // delete everything!
doIt(client);
}
Iterator<SolrInputDocument> manyDocs(final int base, final int numDocs) {
return new Iterator<SolrInputDocument>() {
int count = 0;
@Override
public boolean hasNext() {
return count < numDocs;
}
@Override
public SolrInputDocument next() {
int id = base + count++;
if (count == 1) { // first doc is legit, and will increment a counter
return sdoc("id","test", "count_i", map("inc",1));
}
// include "ignore_exception" so the log doesn't fill up with known exceptions, and change the values for each doc
// so binary format won't compress too much
return sdoc("id",Integer.toString(id),"ignore_exception_field_does_not_exist_"+id,"fieldval"+id);
}
@Override
public void remove() {
}
};
};
void doThreads(final HttpSolrClient client, final int numThreads, final int numRequests) throws Exception {
final AtomicInteger tries = new AtomicInteger(0);
List<Thread> threads = new ArrayList<>();
for (int i=0; i<numThreads; i++) {
final int threadNum = i;
threads.add( new Thread() {
int reqLeft = numRequests;
@Override
public void run() {
try {
while (--reqLeft >= 0) {
tries.incrementAndGet();
doSingle(client, threadNum);
}
} catch (Throwable e) {
// Allow thread to exit, we should have already recorded the exception.
}
}
}
);
}
for (Thread thread : threads) {
thread.start();
}
for (Thread thread : threads) {
thread.join();
}
showExceptions();
int count = getCount(client);
if (count > tries.get()) {
fail("Number of requests was " + tries.get() + " but final count was " + count);
}
assertEquals(tries.get(), getCount(client));
assertTrue("got unexpected exceptions. ", unexpected.isEmpty() );
}
int getCount(HttpSolrClient client) throws IOException, SolrServerException {
client.commit();
QueryResponse rsp = client.query(params("q", "id:test", "fl", "count_i", "wt", "json"));
int count = ((Number)rsp.getResults().get(0).get("count_i")).intValue();
return count;
}
// this always failed with the Jetty 9.3 snapshot
void doIt(HttpSolrClient client) throws Exception {
client.deleteByQuery("*:*");
doThreads(client,10,100);
// doSingle(client, 1);
}
void doSingle(HttpSolrClient client, int threadNum) {
try {
client.add(manyDocs(threadNum*1000000, 1000));
}
catch (HttpSolrClient.RemoteSolrException e) {
String msg = e.getMessage();
assertTrue(msg, msg.contains("field_does_not_exist"));
}
catch (Throwable e) {
unexpected.add(e);
log.error("unexpected exception:", e);
fail("FAILING unexpected exception: " + e);
}
}
/***
@Test
public void testLive() throws Exception {
HttpSolrClient client = new HttpSolrClient("http://localhost:8983/techproducts/solr/");
client.add( sdoc() );
doiIt(client);
}
***/
String getJsonDocs(int numDocs) {
StringBuilder sb = new StringBuilder(numDocs * 20);
sb.append("[");
for (int i = 0; i < numDocs; i++) {
sb.append("{ id : '" + i + "' , unknown_field_" + i + " : 'unknown field value' }");
}
sb.append("]");
return sb.toString();
}
byte[] whitespace(int n) {
byte[] arr = new byte[n];
Arrays.fill(arr, (byte) ' ');
return arr;
}
String getResponse(InputStream is) throws Exception {
StringBuilder sb = new StringBuilder();
byte[] buf = new byte[100000];
for (;;) {
int n = 0;
try {
n = is.read(buf);
} catch (IOException e) {
// a real HTTP client probably wouldn't try to read past the end and would thus
// not get an exception until the *next* http request.
log.error("CAUGHT IOException, but already read " + sb.length() + " : " + getChain(e));
}
if (n <= 0) break;
sb.append(new String(buf, 0, n, StandardCharsets.UTF_8));
log.info("BUFFER=" + sb.toString());
break; // for now, assume we got whole response in one read... otherwise we could block when trying to read again
}
return sb.toString();
}
@Test
public void testHttpURLConnection() throws Exception {
String bodyString = getJsonDocs(200000); // sometimes succeeds with this size, but larger can cause OOM from command line
HttpSolrClient client = (HttpSolrClient) getSolrClient();
String urlString = client.getBaseURL() + "/update";
HttpURLConnection conn = null;
URL url = new URL(urlString);
conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(), StandardCharsets.UTF_8);
writer.write(bodyString);
writer.flush();
int code = 1;
try {
code = conn.getResponseCode();
} catch (Throwable th) {
log.error("ERROR DURING conn.getResponseCode():",th);
}
/***
java.io.IOException: Error writing to server
at __randomizedtesting.SeedInfo.seed([2928C6EE314CD076:947A81A74F582526]:0)
at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:665)
at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:677)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1533)
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1440)
at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480)
*/
log.info("CODE=" + code);
InputStream is;
if (code == 200) {
is = conn.getInputStream();
} else {
log.info("Attempting to get error stream.");
is = conn.getErrorStream();
if (is == null) {
log.info("Can't get error stream... try input stream?");
is = conn.getInputStream();
}
}
String rbody = IOUtils.toString(is, StandardCharsets.UTF_8);
log.info("RESPONSE BODY:" + rbody);
}
@Test
public void testRawSocket() throws Exception {
String hostName = "127.0.0.1";
int port = jetty.getLocalPort();
try (Socket socket = new Socket(hostName, port);
OutputStream out = new BufferedOutputStream(socket.getOutputStream());
InputStream in = socket.getInputStream();
) {
byte[] body = getJsonDocs(100000).getBytes(StandardCharsets.UTF_8);
int bodyLen = body.length;
// bodyLen *= 10; // make server wait for more
byte[] whitespace = whitespace(1000000);
bodyLen += whitespace.length;
String headers = "POST /solr/collection1/update HTTP/1.1\n" +
"Host: localhost:" + port + "\n" +
// "User-Agent: curl/7.43.0\n" +
"Accept: */*\n" +
"Content-type:application/json\n" +
"Content-Length: " + bodyLen + "\n" +
"Connection: Keep-Alive\n";
// Headers of HTTP connection are defined to be ASCII only:
out.write(headers.getBytes(StandardCharsets.US_ASCII));
out.write('\n'); // extra newline separates headers from body
out.write(body);
out.flush();
// Now what if I try to write more? This doesn't seem to throw an exception!
Thread.sleep(1000);
out.write(whitespace); // whitespace
out.flush();
String rbody = getResponse(in); // This will throw a connection reset exception if you try to read past the end of the HTTP response
log.info("RESPONSE BODY:" + rbody);
assertTrue(rbody.contains("unknown_field"));
/***
// can I reuse now?
// writing another request doesn't actually throw an exception, but the following read does
out.write(headers);
out.write("\n"); // extra newline separates headers from body
out.write(body);
out.flush();
rbody = getResponse(in);
log.info("RESPONSE BODY:" + rbody);
assertTrue(rbody.contains("unknown_field"));
***/
}
}
}