/*
* WebUtils.java
*
* Copyright (c) 2007-2011, The University of Sheffield.
*
* This file is part of GATE MÃmir (see http://gate.ac.uk/family/mimir.html),
* and is free software, licenced under the GNU Lesser General Public License,
* Version 3, June 2007 (also included with this distribution as file
* LICENCE-LGPL3.html).
*
* Dominic Rout 5 Apr 2017
* Valentin Tablan, 29 Jan 2010
*
* $Id: WebUtils.java 17423 2014-02-26 10:36:54Z valyt $
*/
package gate.mimir.tool;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CookieStore;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.entity.ByteArrayEntity;
import org.apache.http.entity.SerializableEntity;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import sun.net.www.protocol.http.HttpURLConnection;
import java.io.*;
import java.net.URISyntaxException;
import java.nio.CharBuffer;
import java.util.concurrent.TimeUnit;
/**
* A collection of methods that provide various utility functions for web
* applications.
*/
public class WebUtils {
protected PoolingHttpClientConnectionManager connectionManager;
protected CredentialsProvider credsProvider;
protected CloseableHttpClient client;
protected boolean hasContext;
protected CookieStore cookieJar;
protected UsernamePasswordCredentials creds;
public WebUtils() {
this(null, null, null, 10);
}
public WebUtils(CookieStore cookieJar) {
this(null, null, null, 10);
}
public WebUtils(String userName, String password) {
this(null, userName, password, 10);
}
public WebUtils(CookieStore cookieJar,
String userName, String password) {
this(cookieJar, userName, password, 10);
}
public WebUtils(CookieStore cookieJar,
String userName, String password, int maxConnections) {
connectionManager = new PoolingHttpClientConnectionManager(30, TimeUnit.SECONDS);
// Increase max total connection to 200
connectionManager.setMaxTotal(maxConnections);
// Increase default max connection per route to 20
connectionManager.setDefaultMaxPerRoute(maxConnections);
this.cookieJar = cookieJar;
hasContext = cookieJar != null || userName != null || password != null;
credsProvider = new BasicCredentialsProvider();
if (userName != null && password != null) {
creds = new UsernamePasswordCredentials(userName, password);
} else {
creds = null;
}
RequestConfig globalConfig = RequestConfig.custom()
.setCookieSpec(CookieSpecs.DEFAULT)
.build();
client = HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(globalConfig)
.setDefaultCookieStore(cookieJar)
.build();
}
/**
* Constructs a URL from a base URL segment and a set of query parameters.
* @param urlBase the string that will be the prefix of the returned.
* This should include everything apart from the query part of the URL.
* @param params an array of String values, which should contain alternating
* parameter names and parameter values. It is obvious that the size of this
* array must be an even number.
* @return a URl built according to the provided parameters. If for example
* the following parameter values are provided: <b>urlBase:</b>
* <tt>http://host:8080/appName/service</tt>; <b>params:</b> <tt>foo1, bar1,
* foo2, bar2, foo3, bar3</tt>, then the following URL would be returned:
* <tt>http://host:8080/appName/service?foo1=bar1&foo2=bar2&foo3=bar3</tt>
*/
public static String buildUrl(String urlBase, String... params){
StringBuilder str = new StringBuilder(urlBase);
if(params != null && params.length > 0){
str.append('?');
for(int i = 0 ; i < (params.length/2) - 1; i++){
str.append(params[i * 2]);
str.append('=');
str.append(params[i * 2 + 1]);
str.append('&');
}
//and now, the last parameter
str.append(params[params.length - 2]);
str.append('=');
str.append(params[params.length - 1]);
}
return str.toString();
}
protected HttpContext getContext() {
HttpClientContext context = HttpClientContext.create();
context.setCredentialsProvider(credsProvider);
if (this.cookieJar != null) {
context.setCookieStore(this.cookieJar);
}
return context;
}
public CloseableHttpResponse execute(HttpUriRequest request) throws IOException {
// If we have a context, we have to generate a new one for each request,
// because sharing them between threads seems to break after a few hundred thousan
// requests.
if (hasContext) {
// Fetch a context to use.
HttpContext context = getContext();
if (creds != null) {
// Attach any credentials provided to the given host.
credsProvider.setCredentials(
new AuthScope(request.getURI().getHost(),
AuthScope.ANY_PORT),
creds);
}
// Run the request.
return this.client.execute(request, context);
} else {
// No cookies or auth needed - just run the request as is.
return this.client.execute(request);
}
}
/**
* Calls a web service action (i.e. it connects to a URL). If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will write all content available from the
* input stream of the resulting connection to the provided Appendable.
*
* @param out an {@link Appendable} to which the output is written.
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public void getText(final Appendable out, String baseUrl, String... params)
throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
new RequestExecutor<Void>(this)
.runRequest(request, response -> {
InputStream contentInputStream = response.getEntity().getContent();
try {
Reader r = new InputStreamReader(contentInputStream, "UTF-8");
char[] bufArray = new char[4096];
CharBuffer buf = CharBuffer.wrap(bufArray);
int charsRead = -1;
while ((charsRead = r.read(bufArray)) >= 0) {
buf.position(0);
buf.limit(charsRead);
out.append(buf);
}
} finally {
contentInputStream.close();
}
return null;
});
}
/**
* Calls a web service action (i.e. it connects to a URL), and reads a
* serialised int value from the resulting connection. If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all additional content available from
* either the input and error streams of the resulting connection (which
* should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public int getInt(String baseUrl, String... params)
throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
return new RequestExecutor<Integer>(this)
.runObjectRequest(request, (ObjectInputStream o) -> o.readInt());
}
/**
* Calls a web service action (i.e. it connects to a URL). If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all content available from either the
* input and error streams of the resulting connection (which should permit
* connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public void getVoid(String baseUrl, String... params) throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
new RequestExecutor<Void>(this)
.runRequest(request, response -> null);
}
/**
* Calls a web service action (i.e. it connects to a URL), and reads a
* serialised long value from the resulting connection. If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all additional content available from
* either the input and error streams of the resulting connection (which
* should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public long getLong(String baseUrl, String... params)
throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
return new RequestExecutor<Long>(this)
.runObjectRequest(request, (ObjectInputStream o) -> o.readLong());
}
/**
* Calls a web service action (i.e. it connects to a URL), and reads a
* serialised double value from the resulting connection. If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all additional content available from
* either the input and error streams of the resulting connection (which
* should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public double getDouble(String baseUrl, String... params)
throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
return new RequestExecutor<Double>(this)
.runObjectRequest(request, (ObjectInputStream o) -> o.readDouble());
}
/**
* Calls a web service action (i.e. it connects to a URL), and reads a
* serialised boolean value from the resulting connection. If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all additional content available from
* either the input and error streams of the resulting connection (which
* should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public boolean getBoolean(String baseUrl, String... params)
throws IOException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
return new RequestExecutor<Boolean>(this)
.runObjectRequest(request, ObjectInputStream::readBoolean);
}
/**
* Calls a web service action (i.e. it connects to a URL), and reads a
* serialised Object value from the resulting connection. If the connection
* fails, for whatever reason, or the response code is different from
* {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all additional content available from
* either the input and error streams of the resulting connection (which
* should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
* @throws ClassNotFoundException if the value read from the remote connection
* is of a type unknown to the local JVM.
*/
public Object getObject(String baseUrl, String... params)
throws IOException, ClassNotFoundException {
HttpGet request = new HttpGet(buildUrl(baseUrl, params));
try {
return new RequestExecutor<>(this)
.runObjectRequest(request, ObjectInputStream::readObject);
} catch (RuntimeException e) {
if (e.getCause() instanceof ClassNotFoundException) {
throw (ClassNotFoundException) e.getCause();
} else {
throw e;
}
}
}
/**
* Calls a web service action (i.e. it connects to a URL) using the POST HTTP
* method, sending the given object in Java serialized format as the request
* body. The request is sent using chunked transfer encoding, and the
* request's Content-Type is set to application/octet-stream. If the
* connection fails, for whatever reason, or the response code is different
* from {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all content available from either the
* input and error streams of the resulting connection (which should permit
* connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param object the object to serialize and send in the POST body
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public void postObject(String baseUrl, Serializable object,
String... params) throws IOException {
HttpPost request = new HttpPost(buildUrl(baseUrl, params));
request.setHeader("Content-Type", "application/octet-stream");
// Set up the entity to send to the server.
SerializableEntity entity = new SerializableEntity(object);
entity.setChunked(true);
request.setEntity(entity);
// Now run the request
new RequestExecutor<Void>(this)
.runRequest(request, a -> null);
}
/**
* Calls a web service action (i.e. it connects to a URL) using the POST HTTP
* method, sending the given bytes as the request
* body. The request is sent using chunked transfer encoding, and the
* request's Content-Type is set to application/octet-stream. If the
* connection fails, for whatever reason, or the response code is different
* from {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* This method will drain (and discard) all content available from either the
* input and error streams of the resulting connection (which should permit
* connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param data a {@link ByteArrayOutputStream} containing the data to be
* written. Its {@link ByteArrayOutputStream#writeTo(OutputStream)} method
* will be called causing it to write its data to the output connection.
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @throws IOException if the connection fails.
*/
public void postData(String baseUrl, ByteArrayOutputStream data,
String... params) throws IOException {
HttpPost request = new HttpPost(buildUrl(baseUrl, params));
request.setHeader("Content-Type", "application/octet-stream");
ByteArrayEntity entity = new ByteArrayEntity(data.toByteArray());
entity.setChunked(true);
request.setEntity(entity);
new RequestExecutor<Void>(this)
.runRequest(request, a -> null);
}
/**
* Calls a web service action (i.e. it connects to a URL) using the POST HTTP
* method, sending the given object in Java serialized format as the request
* body. The request is sent using chunked transfer encoding, and the
* request's Content-Type is set to application/octet-stream. If the
* connection fails, for whatever reason, or the response code is different
* from {@link HttpURLConnection#HTTP_OK}, then an IOException is raised.
* The response from the server is read and Java-deserialized, the resulting
* Object being returned.
* <p>
* This method will then drain (and discard) all the remaining content
* available from either the input and error streams of the resulting
* connection (which should permit connection keepalives).
*
* @param baseUrl the constant part of the URL to be accessed.
* @param object the object to serialize and send in the POST body
* @param params an array of String values, that contain an alternation of
* parameter name, and parameter values.
* @return the de-serialized value sent by the remote endpoint.
* @throws IOException if the connection fails.
* @throws ClassNotFoundException if the data sent from the remote endpoint
* cannot be deserialized to a class locally known.
*/
public Object rpcCall(String baseUrl, Serializable object,
String... params) throws IOException, ClassNotFoundException {
HttpPost request = new HttpPost(buildUrl(baseUrl, params));
request.setHeader("Content-Type", "application/octet-stream");
// Set up the entity to send to the server.
SerializableEntity entity = new SerializableEntity(object);
entity.setChunked(true);
request.setEntity(entity);
// Now run the request
return new RequestExecutor<>(this)
.runObjectRequest(request, ObjectInputStream::readObject);
}
protected static class RequestExecutor<T> {
private WebUtils webUtils;
RequestExecutor(WebUtils webUtils) {
this.webUtils = webUtils;
}
public T runRequest(HttpUriRequest request, CheckedRequestConsumer<T> consumer) throws IOException {
CloseableHttpResponse response = webUtils.execute(request);
try {
long code = response.getStatusLine().getStatusCode();
if (code == HttpURLConnection.HTTP_OK) {
// try to get more details
return consumer.run(response);
} else {
// some problem -> try to get more details
String message = response.getStatusLine().getReasonPhrase();
throw new IOException(code
+ (message != null ? " (" + message + ")" : "")
+ " Remote connection failed.");
}
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
} finally {
// make sure the connection is drained, to allow connection keepalive
response.close();
}
}
public T runObjectRequest(HttpUriRequest request, final CheckedObjectInputStreamConsumer<T> consumer) throws IOException {
return runRequest(request, (CloseableHttpResponse response) -> {
InputStream contentInputStream = null;
try {
contentInputStream = response.getEntity().getContent();
return consumer.run(new ObjectInputStream(contentInputStream));
} finally {
contentInputStream.close();
}
});
}
public interface CheckedRequestConsumer<T> {
T run(CloseableHttpResponse response) throws IOException, ClassNotFoundException;
}
public interface CheckedObjectInputStreamConsumer<T> {
T run(ObjectInputStream response) throws IOException, ClassNotFoundException;
}
}
}