/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.solr.servlet;
import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Vector;
import javax.servlet.ReadListener;
import javax.servlet.ServletInputStream;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.MultiMapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ContentStream;
import org.apache.solr.core.SolrCore;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.servlet.SolrRequestParsers.MultipartRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.FormDataRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.RawRequestParser;
import org.apache.solr.servlet.SolrRequestParsers.StandardRequestParser;
import static org.mockito.Mockito.*;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SolrRequestParserTest extends SolrTestCaseJ4 {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
@BeforeClass
public static void beforeClass() throws Exception {
initCore("solrconfig.xml", "schema.xml");
parser = new SolrRequestParsers( h.getCore().getSolrConfig() );
}
static SolrRequestParsers parser;
@AfterClass
public static void afterClass() {
parser = null;
}
@Test
public void testStreamBody() throws Exception
{
String body1 = "AMANAPLANPANAMA";
String body2 = "qwertasdfgzxcvb";
String body3 = "1234567890";
SolrCore core = h.getCore();
Map<String,String[]> args = new HashMap<>();
args.put( CommonParams.STREAM_BODY, new String[] {body1} );
// Make sure it got a single stream in and out ok
List<ContentStream> streams = new ArrayList<>();
SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 1, streams.size() );
assertEquals( body1, IOUtils.toString( streams.get(0).getReader() ) );
req.close();
// Now add three and make sure they come out ok
streams = new ArrayList<>();
args.put( CommonParams.STREAM_BODY, new String[] {body1,body2,body3} );
req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
assertEquals( 3, streams.size() );
ArrayList<String> input = new ArrayList<>();
ArrayList<String> output = new ArrayList<>();
input.add( body1 );
input.add( body2 );
input.add( body3 );
output.add( IOUtils.toString( streams.get(0).getReader() ) );
output.add( IOUtils.toString( streams.get(1).getReader() ) );
output.add( IOUtils.toString( streams.get(2).getReader() ) );
// sort them so the output is consistent
Collections.sort( input );
Collections.sort( output );
assertEquals( input.toString(), output.toString() );
req.close();
// set the contentType and make sure tat gets set
String ctype = "text/xxx";
streams = new ArrayList<>();
args.put( CommonParams.STREAM_CONTENTTYPE, new String[] {ctype} );
req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams );
for( ContentStream s : streams ) {
assertEquals( ctype, s.getContentType() );
}
req.close();
}
@Test
public void testStreamURL() throws Exception
{
URL url = getClass().getResource("/README");
assertNotNull("Missing file 'README' in test-resources root folder.", url);
byte[] bytes = IOUtils.toByteArray(url);
SolrCore core = h.getCore();
Map<String,String[]> args = new HashMap<>();
args.put( CommonParams.STREAM_URL, new String[] { url.toExternalForm() } );
// Make sure it got a single stream in and out ok
List<ContentStream> streams = new ArrayList<>();
try (SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams )) {
assertEquals( 1, streams.size() );
try (InputStream in = streams.get(0).getStream()) {
assertArrayEquals( bytes, IOUtils.toByteArray( in ) );
}
}
}
@Test
public void testStreamFile() throws Exception
{
File file = getFile("README");
byte[] bytes = FileUtils.readFileToByteArray(file);
SolrCore core = h.getCore();
Map<String,String[]> args = new HashMap<>();
args.put( CommonParams.STREAM_FILE, new String[] { file.getAbsolutePath() } );
// Make sure it got a single stream in and out ok
List<ContentStream> streams = new ArrayList<>();
try (SolrQueryRequest req = parser.buildRequestFrom( core, new MultiMapSolrParams( args ), streams )) {
assertEquals( 1, streams.size() );
try (InputStream in = streams.get(0).getStream()) {
assertArrayEquals( bytes, IOUtils.toByteArray( in ) );
}
}
}
@Test
public void testUrlParamParsing() throws Exception
{
final String[][] teststr = new String[][] {
{ "this is simple", "this%20is%20simple" },
{ "this is simple", "this+is+simple" },
{ "\u00FC", "%C3%BC" }, // lower-case "u" with diaeresis/umlaut
{ "\u0026", "%26" }, // &
{ "", "" }, // empty
{ "\u20AC", "%E2%82%ac" } // euro, also with lowercase escapes
};
for( String[] tst : teststr ) {
SolrParams params = SolrRequestParsers.parseQueryString( "val="+tst[1] );
assertEquals( tst[0], params.get( "val" ) );
params = SolrRequestParsers.parseQueryString( "val="+tst[1]+"&" );
assertEquals( tst[0], params.get( "val" ) );
params = SolrRequestParsers.parseQueryString( "&&val="+tst[1]+"&" );
assertEquals( tst[0], params.get( "val" ) );
params = SolrRequestParsers.parseQueryString( "&&val="+tst[1]+"&&&val="+tst[1]+"&" );
assertArrayEquals(new String[]{tst[0],tst[0]}, params.getParams("val") );
}
SolrParams params = SolrRequestParsers.parseQueryString("val");
assertEquals("", params.get("val"));
params = SolrRequestParsers.parseQueryString("val&foo=bar=bar&muh&");
assertEquals("", params.get("val"));
assertEquals("bar=bar", params.get("foo"));
assertEquals("", params.get("muh"));
final String[] invalid = {
"q=h%FCllo", // non-UTF-8
"q=h\u00FCllo", // encoded string is not pure US-ASCII
"q=hallo%", // incomplete escape
"q=hallo%1", // incomplete escape
"q=hallo%XX123", // invalid digit 'X' in escape
"=hallo" // missing key
};
for (String s : invalid) {
try {
SolrRequestParsers.parseQueryString(s);
fail("Should throw SolrException");
} catch (SolrException se) {
// pass
}
}
}
@Test
public void testStandardParseParamsAndFillStreams() throws Exception
{
final String getParams = "qt=%C3%BC&dup=foo", postParams = "q=hello&d%75p=bar";
final byte[] postBytes = postParams.getBytes(StandardCharsets.US_ASCII);
// Set up the expected behavior
final String[] ct = new String[] {
"application/x-www-form-urlencoded",
"Application/x-www-form-urlencoded",
"application/x-www-form-urlencoded; charset=utf-8",
"application/x-www-form-urlencoded;"
};
for( String contentType : ct ) {
HttpServletRequest request = getMock("/solr/select", contentType, postBytes.length);
when(request.getMethod()).thenReturn("POST");
when(request.getQueryString()).thenReturn(getParams);
when(request.getInputStream()).thenReturn(new ByteServletInputStream(postBytes));
MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
RawRequestParser raw = new RawRequestParser();
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
assertEquals( "contentType: "+contentType, "hello", p.get("q") );
assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
assertArrayEquals( "contentType: "+contentType, new String[]{"foo","bar"}, p.getParams("dup") );
verify(request).getInputStream();
}
}
static class ByteServletInputStream extends ServletInputStream {
final BufferedInputStream in;
final int len;
int readCount = 0;
public ByteServletInputStream(byte[] data) {
this.len = data.length;
this.in = new BufferedInputStream(new ByteArrayInputStream(data));
}
@Override
public boolean isFinished() {
return readCount == len;
}
@Override
public boolean isReady() {
return true;
}
@Override
public void setReadListener(ReadListener readListener) {
throw new IllegalStateException("Not supported");
}
@Override
public int read() throws IOException {
int read = in.read();
readCount += read;
return read;
}
}
@Test
public void testStandardParseParamsAndFillStreamsISO88591() throws Exception
{
final String getParams = "qt=%FC&dup=foo&ie=iso-8859-1&dup=%FC", postParams = "qt2=%FC&q=hello&d%75p=bar";
final byte[] postBytes = postParams.getBytes(StandardCharsets.US_ASCII);
final String contentType = "application/x-www-form-urlencoded; charset=iso-8859-1";
// Set up the expected behavior
HttpServletRequest request = getMock("/solr/select", contentType, postBytes.length);
when(request.getMethod()).thenReturn("POST");
when(request.getQueryString()).thenReturn(getParams);
when(request.getInputStream()).thenReturn(new ByteServletInputStream(postBytes));
MultipartRequestParser multipart = new MultipartRequestParser( 2048 );
RawRequestParser raw = new RawRequestParser();
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
StandardRequestParser standard = new StandardRequestParser( multipart, raw, formdata );
SolrParams p = standard.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
assertEquals( "contentType: "+contentType, "hello", p.get("q") );
assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt") );
assertEquals( "contentType: "+contentType, "\u00FC", p.get("qt2") );
assertArrayEquals( "contentType: "+contentType, new String[]{"foo","\u00FC","bar"}, p.getParams("dup") );
verify(request).getInputStream();
}
@Test
public void testStandardFormdataUploadLimit() throws Exception
{
final int limitKBytes = 128;
final StringBuilder large = new StringBuilder("q=hello");
// grow exponentially to reach 128 KB limit:
while (large.length() <= limitKBytes * 1024) {
large.append('&').append(large);
}
HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", -1);
when(request.getMethod()).thenReturn("POST");
when(request.getInputStream()).thenReturn(new ByteServletInputStream(large.toString().getBytes(StandardCharsets.US_ASCII)));
FormDataRequestParser formdata = new FormDataRequestParser( limitKBytes );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().contains("upload limit"));
assertEquals(400, solre.code());
}
verify(request).getInputStream();
}
@Test
public void testParameterIncompatibilityException1() throws Exception
{
HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", 100);
// we emulate Jetty that returns empty stream when parameters were parsed before:
when(request.getInputStream()).thenReturn(new ServletInputStream() {
@Override public int read() { return -1; }
@Override
public boolean isFinished() {
return true;
}
@Override
public boolean isReady() {
return true;
}
@Override
public void setReadListener(ReadListener readListener) {
}
});
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
assertEquals(500, solre.code());
}
verify(request).getInputStream();
}
@Test
public void testParameterIncompatibilityException2() throws Exception
{
HttpServletRequest request = getMock("/solr/select", "application/x-www-form-urlencoded", 100);
when(request.getMethod()).thenReturn("POST");
// we emulate Tomcat that throws IllegalStateException when parameters were parsed before:
when(request.getInputStream()).thenThrow(new IllegalStateException());
FormDataRequestParser formdata = new FormDataRequestParser( 2048 );
try {
formdata.parseParamsAndFillStreams(request, new ArrayList<ContentStream>());
fail("should throw SolrException");
} catch (SolrException solre) {
assertTrue(solre.getMessage().startsWith("Solr requires that request parameters"));
assertEquals(500, solre.code());
}
verify(request).getInputStream();
}
@Test
public void testAddHttpRequestToContext() throws Exception {
HttpServletRequest request = getMock("/solr/select", null, -1);
when(request.getMethod()).thenReturn("GET");
when(request.getQueryString()).thenReturn("q=title:solr");
Map<String, String> headers = new HashMap<>();
headers.put("X-Forwarded-For", "10.0.0.1");
when(request.getHeaderNames()).thenReturn(new Vector<>(headers.keySet()).elements());
for(Map.Entry<String,String> entry:headers.entrySet()) {
Vector<String> v = new Vector<>();
v.add(entry.getValue());
when(request.getHeaders(entry.getKey())).thenReturn(v.elements());
}
SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
assertFalse(parsers.isAddRequestHeadersToContext());
SolrQueryRequest solrReq = parsers.parse(h.getCore(), "/select", request);
assertFalse(solrReq.getContext().containsKey("httpRequest"));
parsers.setAddRequestHeadersToContext(true);
solrReq = parsers.parse(h.getCore(), "/select", request);
assertEquals(request, solrReq.getContext().get("httpRequest"));
assertEquals("10.0.0.1", ((HttpServletRequest)solrReq.getContext().get("httpRequest")).getHeaders("X-Forwarded-For").nextElement());
}
public void testPostMissingContentType() throws Exception {
HttpServletRequest request = getMock();
when(request.getMethod()).thenReturn("POST");
SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
try {
parsers.parse(h.getCore(), "/select", request);
} catch (SolrException e) {
log.error("should not throw SolrException", e);
fail("should not throw SolrException");
}
}
@Test
public void testAutoDetect() throws Exception {
String curl = "curl/7.30.0";
for (String method : new String[]{"GET","POST"}) {
doAutoDetect(null, method, "{}=a", null, "{}", "a"); // unknown agent should not auto-detect
doAutoDetect(curl, method, "{}", "application/json", null, null); // curl should auto-detect
doAutoDetect(curl, method, " \t\n\r {} ", "application/json", null, null); // starting with whitespace
doAutoDetect(curl, method, " \t\n\r // how now brown cow\n {} ", "application/json", null, null); // supporting comments
doAutoDetect(curl, method, " \t\n\r #different style comment\n {} ", "application/json", null, null);
doAutoDetect(curl, method, " \t\n\r /* C style comment */\n {} ", "application/json", null, null);
doAutoDetect(curl, method, " \t\n\r <tag>hi</tag> ", "text/xml", null, null);
doAutoDetect(curl, method, " \t\r\n aaa=1&bbb=2&ccc=3", null, "bbb", "2"); // params with whitespace first
doAutoDetect(curl, method, "/x=foo&aaa=1&bbb=2&ccc=3", null, "/x", "foo"); // param name that looks like a path
doAutoDetect(curl, method, " \t\r\n /x=foo&aaa=1&bbb=2&ccc=3", null, "bbb", "2"); // param name that looks like a path
}
}
public void doAutoDetect(String userAgent, String method, final String body, String expectedContentType, String expectedKey, String expectedValue) throws Exception {
String uri = "/solr/select";
String contentType = "application/x-www-form-urlencoded";
int contentLength = -1; // does this mean auto-detect?
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getHeader("User-Agent")).thenReturn(userAgent);
when(request.getRequestURI()).thenReturn(uri);
when(request.getContentType()).thenReturn(contentType);
when(request.getContentLength()).thenReturn(contentLength);
when(request.getMethod()).thenReturn(method);
// we dont pass a content-length to let the security mechanism limit it:
when(request.getQueryString()).thenReturn("foo=1&bar=2");
when(request.getInputStream()).thenReturn(new ByteServletInputStream(body.getBytes(StandardCharsets.US_ASCII)));
SolrRequestParsers parsers = new SolrRequestParsers(h.getCore().getSolrConfig());
SolrQueryRequest req = parsers.parse(h.getCore(), "/select", request);
int num=0;
if (expectedContentType != null) {
for (ContentStream cs : req.getContentStreams()) {
num++;
assertTrue(cs.getContentType().startsWith(expectedContentType));
String returnedBody = IOUtils.toString(cs.getReader());
assertEquals(body, returnedBody);
}
assertEquals(1, num);
}
assertEquals("1", req.getParams().get("foo"));
assertEquals("2", req.getParams().get("bar"));
if (expectedKey != null) {
assertEquals(expectedValue, req.getParams().get(expectedKey));
}
req.close();
verify(request).getInputStream();
}
public HttpServletRequest getMock() {
return getMock("/solr/select", null, -1);
}
public HttpServletRequest getMock(String uri, String contentType, int contentLength) {
HttpServletRequest request = mock(HttpServletRequest.class);
when(request.getRequestURI()).thenReturn(uri);
when(request.getContentType()).thenReturn(contentType);
when(request.getContentLength()).thenReturn(contentLength);
return request;
}
}