HttpRESTUtils.java

/**
 * $Id: HttpRESTUtils.java 105077 2012-02-24 22:54:29Z ottenhoff@longsight.com $
 * $URL: https://source.sakaiproject.org/svn/entitybroker/trunk/utils/src/java/org/sakaiproject/entitybroker/util/http/HttpRESTUtils.java $
 * HttpUtils.java - entity-broker - Jul 20, 2008 11:42:19 AM - azeckoski
 **************************************************************************
 * Copyright (c) 2008, 2009 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.sakaiproject.entitybroker.util.http;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;

import javax.servlet.http.Cookie;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HeaderElement;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.HttpState;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.ByteArrayRequestEntity;
import org.apache.commons.httpclient.methods.DeleteMethod;
import org.apache.commons.httpclient.methods.EntityEnclosingMethod;
import org.apache.commons.httpclient.methods.FileRequestEntity;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.InputStreamRequestEntity;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.methods.PutMethod;
import org.apache.commons.httpclient.methods.RequestEntity;
import org.apache.commons.httpclient.methods.StringRequestEntity;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.httpclient.protocol.Protocol;


/**
 * Utilities for generating and processing http requests,
 * allows the developer to fire off a request and get back information about the response<br/>
 * All encoding is automatically UTF-8
 * 
 * @author Aaron Zeckoski (azeckoski @ gmail.com)
 */
public class HttpRESTUtils {

    public static final String CONTENT_TYPE_UTF8 = "text/xml; charset=UTF-8";
    public static final String ENCODING_UTF8 = "UTF-8";
    public static enum Method {POST, GET, PUT, DELETE, HEAD, OPTIONS, TRACE};
    public static final int MAX_RESPONSE_SIZE_CHARS = 1024*1024; // about a million chars max

    /**
     * Fire off a request to a URL using the specified method,
     * include optional params and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    public static HttpResponse fireRequest(String URL, Method method) {
        return fireRequest(URL, method, null, null, false);
    }

    /**
     * Fire off a request to a URL using the specified method,
     * include optional params and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @param params (optional) params to send along with the request, will be encoded in the query string or in the body depending on the method
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    public static HttpResponse fireRequest(String URL, Method method, Map<String, String> params) {
        return fireRequest(null, URL, method, params, null, null, false);
    }

    /**
     * Fire off a request to a URL using the specified method,
     * include optional params and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @param params (optional) params to send along with the request, will be encoded in the query string or in the body depending on the method
     * @param data (optional) data to send along in the body of the request, this only works for POST and PUT requests, ignored for the other types
     * @param guaranteeSSL if this is true then the request is sent in a mode which will allow self signed certs to work,
     * otherwise https requests will fail if the certs cannot be centrally verified
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    public static HttpResponse fireRequest(String URL, Method method, Map<String, String> params, Object data, boolean guaranteeSSL) {
        return fireRequest(null, URL, method, params, null, data, guaranteeSSL);
    }

    /**
     * Fire off a request to a URL using the specified method,
     * include optional params, headers, and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @param params (optional) params to send along with the request, will be encoded in the query string or in the body depending on the method
     * @param params (optional) headers to send along with the request, will be encoded in the headers
     * @param data (optional) data to send along in the body of the request, this only works for POST and PUT requests, ignored for the other types
     * @param guaranteeSSL if this is true then the request is sent in a mode which will allow self signed certs to work,
     * otherwise https requests will fail if the certs cannot be centrally verified
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    public static HttpResponse fireRequest(String URL, Method method, Map<String, String> params, Map<String, String> headers, Object data, boolean guaranteeSSL) {
        return fireRequest(null, URL, method, params, headers, data, guaranteeSSL);
    }

    /**
     * Fire off a request to a URL using the specified method but reuse the client for efficiency,
     * include optional params and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param httpClientWrapper (optional) allows the http client to be reused for efficiency,
     * if null a new one will be created each time, use {@link #makeReusableHttpClient(boolean, int)} to
     * create a reusable instance
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @param params (optional) params to send along with the request, will be encoded in the query string or in the body depending on the method
     * @param data (optional) data to send along in the body of the request, this only works for POST and PUT requests, ignored for the other types
     * @param guaranteeSSL if this is true then the request is sent in a mode which will allow self signed certs to work,
     * otherwise https requests will fail if the certs cannot be centrally verified
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    public static HttpResponse fireRequest(HttpClientWrapper httpClientWrapper, String URL, Method method, Map<String, String> params, Object data, boolean guaranteeSSL) {
        return fireRequest(httpClientWrapper, URL, method, params, null, data, guaranteeSSL);
    }

    /**
     * Fire off a request to a URL using the specified method but reuse the client for efficiency,
     * include optional params and data in the request,
     * the response data will be returned in the object if the request can be carried out
     * 
     * @param httpClientWrapper (optional) allows the http client to be reused for efficiency,
     * if null a new one will be created each time, use {@link #makeReusableHttpClient(boolean, int)} to
     * create a reusable instance
     * @param URL the url to send the request to (absolute or relative, can include query params)
     * @param method the method to use (e.g. GET, POST, etc.)
     * @param params (optional) params to send along with the request, will be encoded in the query string or in the body depending on the method
     * @param params (optional) headers to send along with the request, will be encoded in the headers
     * @param data (optional) data to send along in the body of the request, this only works for POST and PUT requests, ignored for the other types
     * @param guaranteeSSL if this is true then the request is sent in a mode which will allow self signed certs to work,
     * otherwise https requests will fail if the certs cannot be centrally verified
     * @return an object representing the response, includes data about the response
     * @throws HttpRequestException if the request cannot be processed for some reason (this is unrecoverable)
     */
    @SuppressWarnings("deprecation")
    public static HttpResponse fireRequest(HttpClientWrapper httpClientWrapper, String URL, Method method, Map<String, String> params, Map<String, String> headers, Object data, boolean guaranteeSSL) {
        if (guaranteeSSL) {
            // added this to attempt to force the SSL self signed certs to work
            Protocol myhttps = new Protocol("https", new EasySSLProtocolSocketFactory(), 443);
            Protocol.registerProtocol("https", myhttps);
        }

        if (httpClientWrapper == null || httpClientWrapper.getHttpClient() == null) {
            httpClientWrapper = makeReusableHttpClient(false, 0, null);
        }

        HttpMethod httpMethod = null;
        if (method.equals(Method.GET)) {
            GetMethod gm = new GetMethod(URL);
            // put params into query string
            gm.setQueryString( mergeQueryStringWithParams(gm.getQueryString(), params) );
            // warn about data being set
            if (data != null) {
                System.out.println("WARN: data cannot be passed in GET requests, data will be ignored (org.sakaiproject.entitybroker.util.http.HttpUtils#fireRequest)");
            }
            gm.setFollowRedirects(true);
            httpMethod = gm;
        } else if (method.equals(Method.POST)) {
            PostMethod pm = new PostMethod(URL);
            // special handling for post params
            if (params != null) {
                for (Entry<String, String> entry : params.entrySet()) {
                    if (entry.getKey() == null
                            || entry.getValue() == null) {
                        System.out.println("WARN: null value supplied for param name ("+entry.getKey()+") or value ("
                                +entry.getValue()+") (org.sakaiproject.entitybroker.util.http.HttpUtils#fireRequest)");
                    }
                    pm.addParameter(entry.getKey(), entry.getValue());
                }
            }
            // handle data
            handleRequestData(pm, data);
            httpMethod = pm;
        } else if (method.equals(Method.PUT)) {
            PutMethod pm = new PutMethod(URL);
            // put params into query string
            pm.setQueryString( mergeQueryStringWithParams(pm.getQueryString(), params) );
            // handle data
            handleRequestData(pm, data);
            httpMethod = pm;
        } else if (method.equals(Method.DELETE)) {
            DeleteMethod dm = new DeleteMethod(URL);
            // put params into query string
            dm.setQueryString( mergeQueryStringWithParams(dm.getQueryString(), params) );
            // warn about data being set
            if (data != null) {
                System.out.println("WARN: data cannot be passed in DELETE requests, data will be ignored (org.sakaiproject.entitybroker.util.http.HttpUtils#fireRequest)");
            }
            httpMethod = dm;
        } else {
            throw new IllegalArgumentException("Cannot handle method: " + method);
        }
        // set the headers for the request
        if (headers != null) {
            for (Entry<String, String> entry : headers.entrySet()) {
                httpMethod.addRequestHeader(entry.getKey(), entry.getValue());
            }
        }

        HttpResponse response = null;
        try {
            int responseCode = httpClientWrapper.getHttpClient().executeMethod(httpMethod);
            response = new HttpResponse(responseCode);
            
            // Avoid DOS because of large responses using up all memory in the system - https://jira.sakaiproject.org/browse/SAK-20405
			InputStream is = httpMethod.getResponseBodyAsStream();
			StringBuffer out = new StringBuffer();
		    byte[] b = new byte[4096];
		    for (int n; (n = is.read(b)) != -1;) {
		        out.append(new String(b, 0, n));
		        if (out.length() > MAX_RESPONSE_SIZE_CHARS) {
		        	// die if the response exceeds the maximum chars allowed
		        	throw new HttpRequestException("Response size ("+out.length()+" chars) from url ("+URL+") exceeded the maximum allowed batch response size ("+MAX_RESPONSE_SIZE_CHARS+" chars) while processing the response");
		        }
		    }
			String body = out.toString();

            //String body = httpMethod.getResponseBodyAsString();
            //         byte[] responseBody = httpMethod.getResponseBody();
            //         if (responseBody != null) {
            //            body = new String(responseBody, "UTF-8");
            //         }
            response.setResponseBody(body);
            response.setResponseMessage(httpMethod.getStatusText());
            // now get the headers
            HashMap<String, String[]> responseHeaders = new HashMap<String, String[]>();
            Header[] respHeaders = httpMethod.getResponseHeaders();
            for (int i = 0; i < respHeaders.length; i++) {
                Header header = respHeaders[i];
                // now we convert the headers from these odd pairs into something more like servlets expect
                HeaderElement[] elements = header.getElements();
                if (elements == null || elements.length == 0) {
                    continue;
                } else if (elements.length >= 1) {
                    String[] values = new String[elements.length];
                    StringBuilder sb = new StringBuilder();
                    for (int j = 0; j < elements.length; j++) {
                        sb.setLength(0); // clear the StringBuilder
                        sb.append(elements[j].getName());
                        if (elements[j].getValue() != null) {
                            sb.append("=");
                            sb.append(elements[j].getValue());
                        }
                        values[j] = sb.toString();
                    }
                    responseHeaders.put(header.getName(), values);
                }
            }
            response.setResponseHeaders(responseHeaders);
        }
        catch (HttpException he) {
            // error contained in he.getMessage()
            throw new HttpRequestException("Fatal HTTP Request Error: " +
                    "Could not sucessfully fire request to url ("+URL+") using method ("+method+")  :: " + he.getMessage(), he);
        }
        catch (IOException ioe) {
            // other exception
            throw new HttpIOException("IOException (transport/connection) Error: " +
                    "Could not sucessfully fire request to url ("+URL+") using method ("+method+")  :: " + ioe.getMessage(), ioe);
        } finally {
            httpMethod.releaseConnection();
        }
        return response;
    }

    /**
     * Merges an existing queryString with a set of params to create one queryString,
     * this basically just adds the params to the end of the existing query string and will not insert a "?"
     * but will take care of the "&"s
     * @param queryString the query string in URL encoded form, without a leading '?'
     * @param params a set of key->value strings to use as params for the request
     * @return the merged queryString with the params included
     */
    public static String mergeQueryStringWithParams(String queryString, Map<String, String> params) {
        if (queryString == null) {
            queryString = "";
        }
        if (params != null) {
            StringBuilder sb = new StringBuilder(queryString.trim());
            for (Entry<String, String> entry : params.entrySet()) {
                if (sb.length() > 0) {
                    sb.append("&");
                }
                sb.append( urlEncodeUTF8(entry.getKey()) );
                sb.append("=");
                sb.append( urlEncodeUTF8(entry.getValue()) );
            }
            queryString = sb.toString();
        }
        return queryString;
    }

    /**
     * Get the query parameters out of a query string and return them as a map,
     * this can be 
     * @param urlString a complete URL with query string at the end or a partial URL
     * or just a query string only (e.g. /thing/stuff?blah=1&apple=fruit
     * @return the map of all query string parameters (e.g. blah => 1, apple => fruit)
     */
    public static Map<String, String> parseURLintoParams(String urlString) {
        if (urlString == null) {
            throw new IllegalArgumentException("URLstring must not be null");
        }
        Map<String, String> map = new HashMap<String, String>();
        // first reduce it to everything after the ?
        int questionLoc = urlString.indexOf('?');
        if (questionLoc >= 0) {
            if (questionLoc+1 < urlString.length()) {
                urlString = urlString.substring(questionLoc+1);
            }
        }
        // next make sure we have at least one param
        if (urlString.indexOf('=') > 0) {
            String[] params = urlString.split("&");
            for (String param : params) {
                int eqLoc = param.indexOf('=');
                if (eqLoc > 0 
                        && eqLoc < param.length()) {
                    String name = param.substring(0, eqLoc);
                    String value = param.substring(eqLoc+1);
                    map.put(name, value);
                }
            }
        }
        return map;
    }

    protected static String urlEncodeUTF8(String toEncode) {
        String encoded = null;
        if (toEncode != null) {
            try {
                encoded = URLEncoder.encode(toEncode, ENCODING_UTF8);
            } catch (UnsupportedEncodingException e) {
                throw new HttpIOException("Encoding to URL using UTF8 failed for: " + toEncode + " :: " + e.getMessage(), e);
            }
        }
        return encoded;
    }

    protected static void handleRequestData(EntityEnclosingMethod method, Object data) {
        if (method == null) {
            throw new IllegalArgumentException("Invalid method, cannot be null");
        }
        if (data != null) {
            RequestEntity re = null;
            if (data.getClass().isAssignableFrom(InputStream.class)) {
                re = new InputStreamRequestEntity((InputStream) data, CONTENT_TYPE_UTF8);
            } else if (data.getClass().isAssignableFrom(byte[].class)) {
                re = new ByteArrayRequestEntity((byte[]) data, CONTENT_TYPE_UTF8);
            } else if (data.getClass().isAssignableFrom(File.class)) {
                re = new FileRequestEntity((File) data, CONTENT_TYPE_UTF8);
            } else {
                // handle as a string
                try {
                    re = new StringRequestEntity(data.toString(), "text/xml", ENCODING_UTF8);
                } catch (UnsupportedEncodingException e) {
                    throw new HttpIOException("Encoding data using UTF8 failed :: " + e.getMessage(), e);
                }
            }
            method.setRequestEntity(re);
        }
    }

    /**
     * Parses a url string into component pieces,
     * unlike the java URL class, this will work with partial urls
     * @param urlString any URL string
     * @return the URL data object
     */
    public static URLData parseURL(String urlString) {
        return new URLData(urlString);
    }

    /**
     * Turns a method string ("get") into a {@link Method} enum object
     * @param method a method string (case is not important) e.g. GET, Post, put, DeLeTe
     * @return the corresponding {@link Method} enum
     */
    public static Method makeMethodFromString(String method) {
        Method m;
        if (method == null || "".equals(method)) {
            throw new IllegalArgumentException("method cannot be null");
        }
        if (method.equalsIgnoreCase("GET")) {
            m = Method.GET;
        } else if (method.equalsIgnoreCase("POST")) {
            m = Method.POST;
        } else if (method.equalsIgnoreCase("PUT")) {
            m = Method.PUT;
        } else if (method.equalsIgnoreCase("DELETE")) {
            m = Method.DELETE;
        } else if (method.equalsIgnoreCase("HEAD")) {
            m = Method.HEAD;
        } else if (method.equalsIgnoreCase("OPTIONS")) {
            m = Method.OPTIONS;
        } else if (method.equalsIgnoreCase("TRACE")) {
            m = Method.TRACE;
        } else {
            throw new IllegalArgumentException("Unknown http method type ("+method+"): should be GET, POST, PUT, DELETE, HEAD, OPTIONS, or TRACE");
        }
        return m;
    }

    /**
     * Generates a reusable http client wrapper which can be given to {@link #fireRequest(HttpClientWrapper, String, Method, Map, Object, boolean)}
     * as an efficiency mechanism
     * 
     * @param multiThreaded true if you want to allow the client to run in multiple threads
     * @param idleConnectionTimeout if this is 0 then it will use the defaults, otherwise connections will be timed out after this long (ms)
     * @param cookies to send along with every request from this client
     * @return the reusable http client wrapper
     */
    public static HttpClientWrapper makeReusableHttpClient(boolean multiThreaded, int idleConnectionTimeout, Cookie[] cookies) {
        HttpClientWrapper wrapper;
        HttpClient client;
        MultiThreadedHttpConnectionManager connectionManager = null;
        if (multiThreaded) {
            connectionManager = new MultiThreadedHttpConnectionManager();
            client = new HttpClient(connectionManager);
        } else {
            client = new HttpClient();
        }
        if (idleConnectionTimeout <= 0) {
            idleConnectionTimeout = 5000;
        }
        client.getHttpConnectionManager().closeIdleConnections(idleConnectionTimeout);
        client.getHttpConnectionManager().getParams().setConnectionTimeout(idleConnectionTimeout);
        // create the initial state
        HttpState initialState = new HttpState();
        if (cookies != null && cookies.length > 0) {
            for (int i = 0; i < cookies.length; i++) {
                Cookie c = cookies[i];
                org.apache.commons.httpclient.Cookie mycookie = 
                    new org.apache.commons.httpclient.Cookie(c.getDomain(), c.getName(), c.getValue(), c.getPath(), c.getMaxAge(), c.getSecure());
                initialState.addCookie(mycookie);
            }
            client.setState(initialState);
        }
        // set some defaults
        client.getParams().setParameter(HttpMethodParams.USER_AGENT, "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1");
        client.getParams().setCookiePolicy(CookiePolicy.BROWSER_COMPATIBILITY);
        client.getParams().setBooleanParameter(HttpMethodParams.SINGLE_COOKIE_HEADER, true);
        wrapper = new HttpClientWrapper(client, connectionManager, initialState);
        return wrapper;
    }

    /**
     * Encode a date into an http date string
     * @param date the date
     * @return the date string or an empty string if the date is null
     */
    public static String encodeDateHttp(Date date) {
        String str = "";
        if (date != null) {
            SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss Z"); // RFC 2822
            str = sdf.format(date);
        }
        return str;
    }

    /**
     * Indicates a general failure
     */
    public static class HttpRequestException extends RuntimeException {
		private static final long serialVersionUID = -5911335085507422663L;
		public HttpRequestException(String message, Throwable cause) {
            super(message, cause);
        }
		public HttpRequestException(String message) {
            super(message);
        }
    }

    /**
     * Indicates an IO failure
     */
    public static class HttpIOException extends RuntimeException {
		private static final long serialVersionUID = 9040247081054206662L;
		public HttpIOException(String message, Throwable cause) {
            super(message, cause);
        }
    }

}