/*
GNU GENERAL PUBLIC LICENSE
Copyright (C) 2006 The Lobo Project
This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public
License as published by the Free Software Foundation; either
verion 2 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
You should have received a copy of the GNU General Public
License along with this library; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Contact info: lobochief@users.sourceforge.net
*/
package org.lobobrowser.request;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.CookieHandler;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.ProtocolException;
import java.net.Proxy;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.security.AccessControlContext;
import java.security.AccessController;
import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.StringTokenizer;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import javax.net.ssl.HttpsURLConnection;
import org.eclipse.jdt.annotation.NonNull;
import org.lobobrowser.clientlet.CancelClientletException;
import org.lobobrowser.clientlet.ClientletException;
import org.lobobrowser.clientlet.ClientletRequest;
import org.lobobrowser.clientlet.ClientletResponse;
import org.lobobrowser.clientlet.Header;
import org.lobobrowser.main.ExtensionManager;
import org.lobobrowser.main.PlatformInit;
import org.lobobrowser.settings.BooleanSettings;
import org.lobobrowser.settings.CacheSettings;
import org.lobobrowser.settings.ConnectionSettings;
import org.lobobrowser.store.CacheManager;
import org.lobobrowser.ua.Parameter;
import org.lobobrowser.ua.ParameterInfo;
import org.lobobrowser.ua.ProgressType;
import org.lobobrowser.ua.RequestType;
import org.lobobrowser.ua.UserAgent;
import org.lobobrowser.ua.UserAgentContext;
import org.lobobrowser.ua.UserAgentContext.Request;
import org.lobobrowser.ua.UserAgentContext.RequestKind;
import org.lobobrowser.util.BoxedObject;
import org.lobobrowser.util.ID;
import org.lobobrowser.util.NameValuePair;
import org.lobobrowser.util.SimpleThreadPool;
import org.lobobrowser.util.SimpleThreadPoolTask;
import org.lobobrowser.util.Strings;
import org.lobobrowser.util.Urls;
import org.lobobrowser.util.io.Files;
import org.lobobrowser.util.io.IORoutines;
public final class RequestEngine {
private static final int MAX_REDIRECT_COUNT = 30;
private static final Logger logger = Logger.getLogger(RequestEngine.class.getName());
private static final boolean loggerInfo = logger.isLoggable(Level.INFO);
private final SimpleThreadPool threadPool;
private final Collection<RequestInfo> processingRequests = new HashSet<>();
private final CookieStore cookieStore = CookieStore.getInstance();
private final CacheSettings cacheSettings;
private final BooleanSettings booleanSettings;
private final ConnectionSettings connectionSettings;
private RequestEngine() {
// Use few threads to avoid excessive parallelism. Note that
// downloads are not handled by this thread pool.
this.threadPool = new SimpleThreadPool("RequestEngineThreadPool", 3, 5, 60 * 1000);
// Security: Private fields that require privileged access to get
// initialized.
this.cacheSettings = CacheSettings.getInstance();
this.connectionSettings = ConnectionSettings.getInstance();
this.booleanSettings = BooleanSettings.getInstance();
}
private static final RequestEngine instance = new RequestEngine();
public static RequestEngine getInstance() {
return instance;
}
public String getCookie(final java.net.URL url) {
final Collection<Cookie> cookies = this.cookieStore.getCookies(url.getProtocol(), url.getHost(), url.getPath());
final StringBuffer cookieText = new StringBuffer();
cookies.forEach(cookie -> {
cookieText.append(cookie.getName());
cookieText.append('=');
cookieText.append(cookie.getValue());
cookieText.append(';');
});
// Note: Return blank if there are no cookies, not null.
return cookieText.toString();
}
public void setCookie(final URL url, final String cookieSpec) {
try {
this.cookieStore.saveCookie(url.toURI(), cookieSpec);
} catch (final URISyntaxException e) {
throw new RuntimeException(e);
}
}
public void cancelAllRequests() {
this.threadPool.cancelAll();
}
public void cancelRequest(final RequestHandler rhToDelete) {
this.threadPool.cancel(new RequestHandlerTask(rhToDelete));
this.cancelRequestIfRunning(rhToDelete);
}
public void cancelRequestIfRunning(final RequestHandler rhToDelete) {
rhToDelete.cancel();
final List<RequestInfo> handlersToCancel = new ArrayList<>();
synchronized (this.processingRequests) {
final Iterator<RequestInfo> ri = this.processingRequests.iterator();
while (ri.hasNext()) {
final RequestInfo rinfo = ri.next();
if (rinfo.getRequestHandler() == rhToDelete) {
handlersToCancel.add(rinfo);
}
}
}
final Iterator<RequestInfo> ri2 = handlersToCancel.iterator();
while (ri2.hasNext()) {
final RequestInfo rinfo = ri2.next();
rinfo.abort();
}
}
public void scheduleRequest(final RequestHandler handler) {
// Note: Important to create task with current access context if there's
// a security manager.
final SecurityManager sm = System.getSecurityManager();
final AccessControlContext context = sm == null ? null : AccessController.getContext();
this.threadPool.schedule(new RequestHandlerTask(handler, context));
}
private static final String NORMAL_FORM_ENCODING = "application/x-www-form-urlencoded";
private void postData(final URLConnection connection, final ParameterInfo pinfo, final String altPostData) throws IOException {
final BooleanSettings boolSettings = this.booleanSettings;
final String encoding = pinfo != null ? pinfo.getEncoding() : NORMAL_FORM_ENCODING;
if ((encoding == null) || NORMAL_FORM_ENCODING.equalsIgnoreCase(encoding)) {
final ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
if (pinfo != null) {
final Parameter[] parameters = pinfo.getParameters();
boolean firstParam = true;
for (final Parameter parameter : parameters) {
final String name = parameter.getName();
final String encName = URLEncoder.encode(name, "UTF-8");
if (parameter.isText()) {
if (firstParam) {
firstParam = false;
} else {
bufOut.write((byte) '&');
}
final String valueStr = parameter.getTextValue();
final String encValue = URLEncoder.encode(valueStr, "UTF-8");
bufOut.write(encName.getBytes("UTF-8"));
bufOut.write((byte) '=');
bufOut.write(encValue.getBytes("UTF-8"));
} else {
logger.warning("postData(): Ignoring non-textual parameter " + name + " for POST with encoding " + encoding + ".");
}
}
} else {
// No pinfo provided - check alternative POST data
if (altPostData != null) {
bufOut.write(altPostData.getBytes("UTF-8"));
}
}
// Do not add a line break to post content. Some servers
// can be picky about that (namely, java.net).
final byte[] postContent = bufOut.toByteArray();
logInfo("postData(): Will post: " + new String(postContent));
if (connection instanceof HttpURLConnection) {
if (boolSettings.isHttpUseChunkedEncodingPOST()) {
((HttpURLConnection) connection).setChunkedStreamingMode(8192);
} else {
((HttpURLConnection) connection).setFixedLengthStreamingMode(postContent.length);
}
}
connection.setRequestProperty("Content-Type", NORMAL_FORM_ENCODING);
// connection.setRequestProperty("Content-Length",
// String.valueOf(postContent.length));
final OutputStream postOut = connection.getOutputStream();
postOut.write(postContent);
postOut.flush();
} else if ("multipart/form-data".equalsIgnoreCase(encoding)) {
final long id = ID.generateLong();
final String boundary = "----------------" + id;
final boolean chunked = boolSettings.isHttpUseChunkedEncodingPOST();
OutputStream mfstream;
if (chunked) {
mfstream = connection.getOutputStream();
} else {
mfstream = new ByteArrayOutputStream();
}
final MultipartFormDataWriter writer = new MultipartFormDataWriter(mfstream, boundary);
try {
if (pinfo != null) {
final Parameter[] parameters = pinfo.getParameters();
for (final Parameter parameter : parameters) {
final String name = parameter.getName();
if (parameter.isText()) {
writer.writeText(name, parameter.getTextValue(), "UTF-8");
} else if (parameter.isFile()) {
final File file = parameter.getFileValue();
try (
final FileInputStream in = new FileInputStream(parameter.getFileValue())) {
final BufferedInputStream bin = new BufferedInputStream(in, 8192);
writer.writeFileData(name, file.getName(), Files.getContentType(file), bin);
}
} else {
logger.warning("postData(): Skipping parameter " + name + " of unknown type for POST with encoding " + encoding + ".");
}
}
}
} finally {
writer.send();
}
connection.addRequestProperty("Content-Type", encoding + "; boundary=" + boundary);
if (chunked) {
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).setChunkedStreamingMode(8192);
}
} else {
final byte[] content = ((ByteArrayOutputStream) mfstream).toByteArray();
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).setFixedLengthStreamingMode(content.length);
}
final OutputStream out = connection.getOutputStream();
out.write(content);
}
} else {
throw new IllegalArgumentException("Unknown encoding: " + encoding);
}
}
private static String completeGetUrl(final String baseURL, final ParameterInfo pinfo, final String ref) throws Exception {
String newNoRefURL;
final Parameter[] parameters = pinfo.getParameters();
if ((parameters != null) && (parameters.length > 0)) {
final StringBuffer sb = new StringBuffer(baseURL);
final int qmIdx = baseURL.indexOf('?');
char separator = qmIdx == -1 ? '?' : '&';
for (final Parameter parameter : parameters) {
if (parameter.isText()) {
sb.append(separator);
sb.append(parameter.getName());
sb.append('=');
final String paramText = parameter.getTextValue();
sb.append(URLEncoder.encode(paramText, "UTF-8"));
separator = '&';
} else {
logger.warning("completeGetUrl(): Skipping non-textual parameter " + parameter.getName() + " in GET request.");
}
}
newNoRefURL = sb.toString();
} else {
newNoRefURL = baseURL;
}
if ((ref != null) && (ref.length() != 0)) {
return newNoRefURL + "#" + ref;
} else {
return newNoRefURL;
}
}
private static void addRequestProperties(final URLConnection connection, final ClientletRequest request, final CacheInfo cacheInfo,
final String requestMethod,
final URL lastRequestURL, final RequestHandler rhandler) throws ProtocolException {
final UserAgent userAgent = request.getUserAgent();
connection.addRequestProperty("User-Agent", userAgent.toString());
connection.addRequestProperty("Accept-Encoding", "gzip, deflate");
// The following two headers are for GH #174
connection.addRequestProperty("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
connection.addRequestProperty("Accept-Language", "en-US,en;q=0.5");
// TODO: Harshad: Commenting out X-Java-Version. Check if required.
// connection.addRequestProperty("X-Java-Version", userAgent.getJavaVersion());
// TODO: Commenting out X-Session-ID. Needs to be privately generated
// or available with the right permissions only. Extensions should not
// have access to the private field. This is not doable if extensions
// should have a permission to open up access to any member.
//
// connection.addRequestProperty("X-Session-ID",
// userAgent.getSessionID(connection.getURL()));
final String referrer = request.getReferrer();
if (referrer != null) {
if (rhandler.getContext().isRequestPermitted(new Request(request.getRequestURL(), RequestKind.Referrer))) {
connection.addRequestProperty("Referer", referrer);
}
}
if (cacheInfo != null) {
final String date = cacheInfo.getDateAsText();
if (date != null) {
connection.addRequestProperty("If-Modified-Since", date);
}
}
if (connection instanceof HttpURLConnection) {
final HttpURLConnection hconnection = (HttpURLConnection) connection;
hconnection.setRequestMethod(requestMethod);
}
final Header[] headers = request.getExtraHeaders();
if (headers != null) {
for (final Header header : headers) {
final String headerName = header.getName();
if (headerName.startsWith("X-")) {
connection.addRequestProperty(headerName, header.getValue());
} else {
logger.warning("run(): Ignoring request header: " + headerName);
}
}
}
}
private static CacheInfo getCacheInfo(final RequestHandler rhandler, final URL url, final boolean isGet) throws Exception {
final RequestType requestType = rhandler.getRequestType();
if (isGet && isOKToRetrieveFromCache(requestType)) {
return AccessController.doPrivileged(new PrivilegedAction<CacheInfo>() {
// Reason: Caller in context may not have privilege to access
// the local file system, yet it's necessary to be able to load
// a cache file.
public CacheInfo run() {
byte[] persistentContent = null;
final CacheManager cm = CacheManager.getInstance();
final MemoryCacheEntry entry = (MemoryCacheEntry) cm.getTransient(url);
if (entry == null) {
if (!"file".equalsIgnoreCase(url.getProtocol()) || !Strings.isBlank(url.getHost())) {
try {
persistentContent = CacheManager.getPersistent(url, false);
} catch (final java.io.IOException ioe) {
logger.log(Level.WARNING, "getCacheInfo(): Unable to load cache file.", ioe);
}
}
}
if ((persistentContent == null) && (entry == null)) {
return null;
}
final CacheInfo cinfo = new CacheInfo(entry, persistentContent, url);
return cinfo;
}
});
} else {
return null;
}
}
private static void cache(final RequestHandler rhandler, final java.net.URL url, final URLConnection connection, final byte[] content,
final java.io.Serializable altPersistentObject, final Object altObject, final int approxAltObjectSize) {
AccessController.doPrivileged(new PrivilegedAction<Object>() {
// Reason: Caller might not have permission to access the
// file system. Yet, caching should be allowed.
public Object run() {
try {
final long currentTime = System.currentTimeMillis();
logInfo("cache(): url=" + url + ",content.length=" + content.length + ",currentTime=" + currentTime);
final Long expiration = Urls.getExpiration(connection, currentTime);
if ((expiration != null) && (expiration > 0)) {
storeCacheEntry(url, connection, content, altPersistentObject, altObject, approxAltObjectSize, currentTime, expiration);
}
} catch (final Exception err) {
logger.log(Level.WARNING, "cache()", err);
}
return null;
}
});
}
private static void storeCacheEntry(final java.net.URL url, final URLConnection connection, final byte[] content,
final java.io.Serializable altPersistentObject, final Object altObject, final int approxAltObjectSize, final long currentTime,
final Long expiration) throws UnsupportedEncodingException, IOException {
int actualApproxObjectSize = 0;
if (altObject != null) {
if (approxAltObjectSize < content.length) {
actualApproxObjectSize = content.length;
} else {
actualApproxObjectSize = approxAltObjectSize;
}
}
final List<NameValuePair> headers = Urls.getHeaders(connection);
final MemoryCacheEntry memEntry = new MemoryCacheEntry(content, headers, expiration, altObject, actualApproxObjectSize);
final int approxMemEntrySize = content.length + (altObject == null ? 0 : approxAltObjectSize);
final CacheManager cm = CacheManager.getInstance();
cm.putTransient(url, memEntry, approxMemEntrySize);
try (
final ByteArrayOutputStream out = new ByteArrayOutputStream()) {
boolean hadDate = false;
boolean hadContentLength = false;
for (int counter = 0; true; counter++) {
final String headerKey = connection.getHeaderFieldKey(counter);
if (headerKey != null) {
if (!hadDate && "date".equalsIgnoreCase(headerKey)) {
hadDate = true;
}
if (!hadContentLength && "content-length".equalsIgnoreCase(headerKey)) {
hadContentLength = true;
}
}
final String headerValue = connection.getHeaderField(counter);
if (headerValue == null) {
break;
}
if (CacheInfo.HEADER_REQUEST_TIME.equalsIgnoreCase(headerKey)) {
continue;
}
// Fix #142: When stored in cache, decoding of input stream has been already done. Hence, don't store the content-encoding header
// TODO: Evaluate the trade-offs of storing the original response with compression. Will save disk-space but increase read-back time.
if ("content-encoding".equalsIgnoreCase(headerKey)) {
continue;
}
final String headerPrefix = (headerKey == null) || (headerKey.length() == 0) ? "" : headerKey + ": ";
final byte[] headerBytes = (headerPrefix + headerValue + "\r\n").getBytes("ISO-8859-1");
out.write(headerBytes);
}
if (!hadDate) {
final String currentDate = Urls.PATTERN_RFC1123.format(new java.util.Date());
final byte[] headerBytes = ("Date: " + currentDate + "\r\n").getBytes("ISO-8859-1");
out.write(headerBytes);
}
if (!hadContentLength) {
final byte[] headerBytes = ("Content-Length: " + content.length + "\r\n").getBytes("ISO-8859-1");
out.write(headerBytes);
}
final byte[] rtHeaderBytes = (CacheInfo.HEADER_REQUEST_TIME + ": " + currentTime + "\r\n").getBytes("ISO-8859-1");
out.write(rtHeaderBytes);
out.write(IORoutines.LINE_BREAK_BYTES);
out.write(content);
try {
CacheManager.putPersistent(url, out.toByteArray(), false);
} catch (final IOException err) {
logger.log(Level.WARNING, "cache(): Unable to cache response content.", err);
}
}
if (altPersistentObject != null) {
try {
final ByteArrayOutputStream fileOut = new ByteArrayOutputStream();
// No need to buffer - Java API already does.
final ObjectOutputStream objOut = new ObjectOutputStream(fileOut);
objOut.writeObject(altPersistentObject);
objOut.flush();
final byte[] byteArray = fileOut.toByteArray();
if (byteArray.length == 0) {
logger
.log(Level.WARNING, "cache(): Serialized content has zero bytes for persistent object " + altPersistentObject + ".");
}
CacheManager.putPersistent(url, byteArray, true);
} catch (final Exception err) {
logger.log(Level.WARNING, "cache(): Unable to write persistent cached object.", err);
}
}
}
private static boolean mayBeCached(final HttpURLConnection connection) {
final String cacheControl = connection.getHeaderField("Cache-Control");
if (cacheControl != null) {
final StringTokenizer tok = new StringTokenizer(cacheControl, ",");
while (tok.hasMoreTokens()) {
final String token = tok.nextToken().trim();
if ("no-cache".equalsIgnoreCase(token)) {
return false;
}
}
}
return true;
}
private static void printRequestHeaders(final URLConnection connection) {
final Map<String, List<String>> headers = connection.getRequestProperties();
final StringBuffer buffer = new StringBuffer();
for (final Map.Entry<String, List<String>> entry : headers.entrySet()) {
buffer.append(entry.getKey() + ": " + entry.getValue());
buffer.append(System.getProperty("line.separator"));
}
logger.info("printRequestHeaders(): Request headers for URI=[" + connection.getURL() + "]\r\n" + buffer.toString());
}
public void inlineRequest(final RequestHandler rhandler) {
// Security checked by low-level APIs in this case.
this.processHandler(rhandler, 0, false);
}
public byte[] loadBytes(final String urlOrPath, final UserAgentContext uaContext) throws Exception {
return this.loadBytes(DomainValidation.guessURL(urlOrPath), uaContext);
}
private byte[] loadBytes(final @NonNull URL url, final UserAgentContext uaContext) throws Exception {
final BoxedObject boxed = new BoxedObject();
this.inlineRequest(new SimpleRequestHandler(url, RequestType.ELEMENT, uaContext) {
@Override
public boolean handleException(final ClientletResponse response, final Throwable exception, final RequestType requestType)
throws ClientletException {
if (exception instanceof ClientletException) {
throw (ClientletException) exception;
} else {
throw new ClientletException(exception);
}
}
public void processResponse(final ClientletResponse response) throws ClientletException, IOException {
final byte[] bytes = org.lobobrowser.util.io.IORoutines.load(response.getInputStream(), 4096);
boxed.setObject(bytes);
}
});
return (byte[]) boxed.getObject();
}
/*
public AsyncResult<byte[]> loadBytesAsync(final String urlOrPath, final UserAgentContext uaContext) throws java.net.MalformedURLException {
return this.loadBytesAsync(Urls.guessURL(urlOrPath), uaContext);
}
public AsyncResult<byte[]> loadBytesAsync(final URL url, final UserAgentContext uaContext) {
final AsyncResultImpl<byte[]> asyncResult = new AsyncResultImpl<>();
this.scheduleRequest(new SimpleRequestHandler(url, RequestType.ELEMENT, uaContext) {
@Override
public boolean handleException(final ClientletResponse response, final Throwable exception, final RequestType requestType) throws ClientletException {
asyncResult.setException(exception);
return true;
}
public void processResponse(final ClientletResponse response) throws ClientletException, IOException {
final byte[] bytes = org.lobobrowser.util.io.IORoutines.load(response.getInputStream(), 4096);
asyncResult.setResult(bytes);
}
});
return asyncResult;
} */
/**
* Whether possibly cached request should always be revalidated, i.e. any
* expiration information is ignored.
*/
private static boolean shouldRevalidateAlways(final URL connectionUrl, final RequestType requestType) {
return requestType == RequestType.ADDRESS_BAR;
}
/**
* Whether the request type should always be obtained from cache if it is
* there.
*/
private static boolean doesNotExpire(final RequestType requestType) {
return requestType == RequestType.HISTORY;
}
private static ExtensionManager getSafeExtensionManager() {
return AccessController.doPrivileged(new PrivilegedAction<ExtensionManager>() {
public ExtensionManager run() {
return ExtensionManager.getInstance();
}
});
}
private URLConnection getURLConnection(final URL connectionUrl, final ClientletRequest request, final String protocol,
final String method,
final RequestHandler rhandler, final CacheInfo cacheInfo) throws IOException {
URLConnection connection;
if (cacheInfo != null) {
final RequestType requestType = rhandler.getRequestType();
if (doesNotExpire(requestType)) {
if (loggerInfo) {
if (cacheInfo.hasTransientEntry()) {
logger.info("getURLConnection(): FROM-RAM: " + connectionUrl + ".");
} else {
logger.info("getURLConnection(): FROM-FILE: " + connectionUrl + ".");
}
}
return cacheInfo.getURLConnection();
} else if (!shouldRevalidateAlways(connectionUrl, requestType)) {
Long expires = cacheInfo.getExpires();
if (expires == null) {
final int defaultOffset = this.cacheSettings.getDefaultCacheExpirationOffset();
expires = cacheInfo.getExpiresGivenOffset(defaultOffset);
if (loggerInfo) {
final java.util.Date expiresDate = expires == null ? null : new Date(expires);
logger.info("getURLConnection(): Used default offset for " + connectionUrl + ": expires=" + expiresDate);
}
}
if (expires != null) {
if (expires.longValue() > System.currentTimeMillis()) {
if (loggerInfo) {
final long secondsToExpiration = (expires.longValue() - System.currentTimeMillis()) / 1000;
if (cacheInfo.hasTransientEntry()) {
logger.info("getURLConnection(): FROM-RAM: " + connectionUrl + ". Expires in " + secondsToExpiration + " seconds.");
} else {
logger.info("getURLConnection(): FROM-FILE: " + connectionUrl + ". Expires in " + secondsToExpiration + " seconds.");
}
}
return cacheInfo.getURLConnection();
} else {
if (loggerInfo) {
logger.info("getURLConnection(): EXPIRED: " + connectionUrl + ". Expired on " + new Date(expires) + ".");
}
}
}
// If the document has expired, the cache may still
// be used, but only after validation.
}
}
final boolean isPost = "POST".equalsIgnoreCase(method);
final String host = connectionUrl.getHost();
final boolean isResURL = "res".equalsIgnoreCase(protocol);
if (isResURL || (host == null) || (host.length() == 0)) {
connection = connectionUrl.openConnection();
} else {
final Proxy proxy = this.connectionSettings.getProxy(host);
if (proxy == Proxy.NO_PROXY) {
// Workaround for JRE 1.5.0.
connection = connectionUrl.openConnection();
} else {
connection = connectionUrl.openConnection(proxy);
}
}
if (connection instanceof HttpsURLConnection) {
((HttpsURLConnection) connection).setHostnameVerifier(rhandler.getHostnameVerifier());
}
if (isPost) {
connection.setDoOutput(true);
}
connection.setUseCaches(false);
if (connection instanceof HttpURLConnection) {
final HttpURLConnection hconnection = (HttpURLConnection) connection;
hconnection.setConnectTimeout(60000);
hconnection.setReadTimeout(90000);
}
addRequestProperties(connection, request, cacheInfo, method, connectionUrl, rhandler);
addRequestedHeadersToRequest(connection, rhandler);
// Moved add cookies here since connection is initiated in this method for POST requests.
// And we can't add headers after the connection is made.
addCookiesToRequest(connection, rhandler);
// dumpRequestInfo(connection);
// Allow extensions to modify the connection object.
// Doing it after addRequestProperties() to allow such
// functionality as altering the Accept header.
connection = getSafeExtensionManager().dispatchPreConnection(connection);
// Print request headers
if (logger.isLoggable(Level.FINE)) {
printRequestHeaders(connection);
}
// POST data if we need to.
if (isPost) {
final ParameterInfo pinfo = rhandler instanceof RedirectRequestHandler ? null : request.getParameterInfo();
final String altPostData = rhandler instanceof RedirectRequestHandler ? null : request.getAltPostData();
if ((pinfo == null) && (altPostData == null)) {
logger.info("POST has no parameter information");
} else {
this.postData(connection, pinfo, altPostData);
}
}
return connection;
}
private static boolean isOKToRetrieveFromCache(final RequestType requestType) {
return (requestType != RequestType.SOFT_RELOAD) && (requestType !=
RequestType.HARD_RELOAD) && (requestType != RequestType.DOWNLOAD);
}
private static void logInfo(final String msg) {
if (loggerInfo) {
logger.info(msg);
}
}
private static void logInfo(final String msg, final Throwable cce) {
if (loggerInfo) {
logger.log(Level.INFO, msg, cce);
}
}
@SuppressWarnings("unused")
private static void dumpRequestInfo(final URLConnection connection) {
if (PlatformInit.getInstance().debugOn) {
System.out.println("URL: " + connection.getURL());
System.out.println(" Request Headers: ");
connection.getRequestProperties().forEach((key, value) -> System.out.println(" " + key + " : " + value));
}
}
private static void dumpResponseInfo(final URLConnection connection) {
if (PlatformInit.getInstance().debugOn) {
System.out.println("URL: " + connection.getURL());
System.out.println(" Response Headers: ");
connection.getHeaderFields().forEach((key, value) -> System.out.println(" " + key + " : " + value));
}
}
private void processHandler(final RequestHandler rhandler, final int recursionLevel, final boolean trackRequestInfo) {
// Method must be private.
final URL baseURL = rhandler.getLatestRequestURL();
RequestInfo rinfo = null;
ClientletResponseImpl response = null;
final String method = rhandler.getLatestRequestMethod().toUpperCase();
try {
final ClientletRequest request = rhandler.getRequest();
// TODO: Hack: instanceof below
final ParameterInfo pinfo = rhandler instanceof RedirectRequestHandler ? null : request.getParameterInfo();
final boolean isGet = "GET".equals(method);
final URL url = makeCompleteURL(baseURL, pinfo, isGet);
final String protocol = url.getProtocol();
final URL connectionUrl = makeConnectionURL(url, protocol);
final CacheInfo cacheInfo = getCacheInfo(rhandler, connectionUrl, isGet);
try {
URLConnection connection = this.getURLConnection(connectionUrl, request, protocol, method, rhandler, cacheInfo);
// This causes exceptions sometimes (when the connection is already open)
// dumpRequestInfo(connection);
rinfo = new RequestInfo(connection, rhandler);
// InputStream responseIn = null;
if (trackRequestInfo) {
synchronized (this.processingRequests) {
this.processingRequests.add(rinfo);
}
}
try {
if (rhandler.isCancelled()) {
throw new CancelClientletException("cancelled");
}
rhandler.handleProgress(ProgressType.CONNECTING, url, method, 0, -1);
// Handle response
boolean isContentCached = (cacheInfo != null) && cacheInfo.isCacheConnection(connection);
boolean isCacheable = false;
if ((connection instanceof HttpURLConnection) && !isContentCached) {
final HttpURLConnection hconnection = (HttpURLConnection) connection;
hconnection.setInstanceFollowRedirects(false);
final int responseCode = hconnection.getResponseCode();
logInfo("run(): ResponseCode=" + responseCode + " for url=" + connectionUrl);
// dumpResponseInfo(connection);
handleCookies(connectionUrl, hconnection, rhandler);
if (responseCode == HttpURLConnection.HTTP_OK) {
logInfo("run(): FROM-HTTP: " + connectionUrl);
if (mayBeCached(hconnection)) {
isCacheable = true;
} else {
logInfo("run(): NOT CACHEABLE: " + connectionUrl);
if (cacheInfo != null) {
cacheInfo.delete();
}
}
// responseIn = connection.getInputStream();
// rinfo.setConnection(connection, responseIn);
rinfo.setConnection(connection);
} else if (responseCode == HttpURLConnection.HTTP_NOT_MODIFIED) {
if (cacheInfo == null) {
throw new IllegalStateException("Cache info missing but it is necessary to process response code " + responseCode + ".");
}
logInfo("run(): FROM-VALIDATION: " + connectionUrl);
// Disconnect the HTTP connection.
hconnection.disconnect();
isContentCached = true;
// Even though the response is actually from the cache,
// we need to cache it again, if only to update the
// request time (used to calculate default expiration).
// TODO: Can this special case be optimized?
isCacheable = true;
connection = cacheInfo.getURLConnection();
// responseIn = connection.getInputStream();
// rinfo.setConnection(connection, responseIn);
rinfo.setConnection(connection);
} else if ((responseCode == HttpURLConnection.HTTP_MOVED_PERM) || (responseCode == HttpURLConnection.HTTP_MOVED_TEMP)
|| (responseCode == HttpURLConnection.HTTP_SEE_OTHER)) {
logInfo("run(): REDIRECTING: ResponseCode=" + responseCode + " for url=" + url);
final RequestHandler newHandler = new RedirectRequestHandler(rhandler, hconnection);
Thread.yield();
if (recursionLevel > MAX_REDIRECT_COUNT) {
throw new ClientletException("Exceeded redirect recursion limit.");
}
this.processHandler(newHandler, recursionLevel + 1, trackRequestInfo);
return;
}
} else {
// Force it to throw exception if file does not exist
// responseIn = connection.getInputStream();
// rinfo.setConnection(connection, responseIn);
rinfo.setConnection(connection);
}
if (rinfo.isAborted()) {
throw new CancelClientletException("Stopped");
}
// Give a chance to extensions to post-process the connection.
final URLConnection newConnection = getSafeExtensionManager().dispatchPostConnection(connection);
if (newConnection != connection) {
// responseIn = newConnection.getInputStream();
connection = newConnection;
}
// Create clientlet response.
response = new ClientletResponseImpl(rhandler, connection, url, isContentCached, cacheInfo, isCacheable,
rhandler.getRequestType());
rhandler.processResponse(response);
updateCache(rhandler, response, connectionUrl, cacheInfo, connection, isCacheable);
} finally {
if (trackRequestInfo) {
synchronized (this.processingRequests) {
this.processingRequests.remove(rinfo);
}
}
/*
if (responseIn != null) {
try {
responseIn.close();
} catch (final java.io.IOException ioe) {
// ignore
}
}*/
// TODO: Possible optimization. By not disconnecting, we might be able to get a faster response for next request
/*
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).disconnect();
}*/
}
} finally {
if (cacheInfo != null) {
// This is necessary so that the file stream doesn't stay open potentially.
cacheInfo.dispose();
}
}
} catch (final CancelClientletException cce) {
logInfo("run(): Clientlet cancelled: " + baseURL, cce);
} catch (final Exception exception) {
if ((rinfo != null) && rinfo.isAborted()) {
logInfo("run(): Exception ignored because request aborted.", exception);
} else {
try {
if (!rhandler.handleException(response, exception, rhandler.getRequestType())) {
logger.log(Level.WARNING, "Was unable to handle exception.", exception);
}
} catch (final Exception err) {
System.out.println("Exception while handling exception:" + exception);
exception.printStackTrace();
logger.log(Level.WARNING, "Exception handler threw an exception.", err);
}
}
} finally {
rhandler.handleProgress(ProgressType.DONE, baseURL, method, 0, 0);
}
}
final private CookieHandler cookieHandler = new CookieHandlerImpl();
private static void addRequestedHeadersToRequest(final URLConnection connection, final RequestHandler rhandler) {
final Optional<Map<String, String>> requestedHeadersOpt = rhandler.getRequestedHeaders();
if (requestedHeadersOpt.isPresent()) {
final Map<String, String> requestedHeaders = requestedHeadersOpt.get();
requestedHeaders.forEach((key, value) -> connection.addRequestProperty(key, value));
}
}
private void addCookiesToRequest(final URLConnection connection, final RequestHandler rhandler) {
try {
final String protocol = connection.getURL().getProtocol();
if ("http".equalsIgnoreCase(protocol) || "https".equalsIgnoreCase(protocol)) {
final URL url = connection.getURL();
final URI uri = url.toURI();
// TODO: optimization #1: list is not required if we directly call our CookieHandler implementation
// TODO: optimization #2: even if we use the CookieHandler interface, we can avoid the joining of List entries, since our impl always returns a single element list
final Map<String, List<String>> cookieHeaders = cookieHandler.get(uri, null);
if (!cookieHeaders.isEmpty()) {
if (rhandler.getContext().isRequestPermitted(new Request(url, RequestKind.Cookie))) {
addCookieHeaderToRequest(connection, cookieHeaders, "Cookie");
}
}
}
} catch (IOException | URISyntaxException e) {
logger.warning("Couldn't add cookies for : " + connection.getURL());
logger.warning(" .. reason: " + e.getMessage());
// TODO: These exceptions should be either not captured, or failure should be
// propagated to caller by return value.
}
}
private static void addCookieHeaderToRequest(final URLConnection connection, final Map<String, List<String>> cookieHeaders,
final String cookieKey) {
final List<String> cookieValue = cookieHeaders.get(cookieKey);
if (cookieValue != null) {
final String cookieValueStr = cookieValue.stream().collect(Collectors.joining(";"));
connection.addRequestProperty(cookieKey, cookieValueStr);
}
}
private void handleCookies(final URL url, final HttpURLConnection hconnection, final RequestHandler rhandler) throws URISyntaxException,
IOException {
final Map<String, List<String>> headerFields = hconnection.getHeaderFields();
final boolean cookieSetterExists = headerFields.keySet().stream().anyMatch(key ->
"Set-Cookie".equalsIgnoreCase(key) // || "Set-Cookie2".equalsIgnoreCase(key)
);
if (cookieSetterExists) {
if (rhandler.getContext().isRequestPermitted(new Request(url, RequestKind.Cookie))) {
cookieHandler.put(url.toURI(), headerFields);
}
}
}
private static void updateCache(final RequestHandler rhandler, final ClientletResponseImpl response, final URL connectionUrl,
final CacheInfo cacheInfo,
final URLConnection connection, final boolean isCacheable) throws IOException {
if (isCacheable) {
// Make sure stream reaches EOF so we don't get null stored content.
response.ensureReachedEOF();
final byte[] content = response.getStoredContent();
if (content != null) {
final Serializable persObject = response.getNewPersistentCachedObject();
final Object altObject = response.getNewTransientCachedObject();
final int altObjectSize = response.getNewTransientObjectSize();
cache(rhandler, connectionUrl, connection, content, persObject, altObject, altObjectSize);
} else {
logger.warning("processHandler(): Cacheable response not available: " + connectionUrl);
}
} else if ((cacheInfo != null) && !cacheInfo.hasTransientEntry()) {
// Content that came from cache cannot be cached again, but a RAM entry was missing.
final byte[] persContent = cacheInfo.getPersistentContent();
final Object altObject = response.getNewTransientCachedObject();
final int altObjectSize = response.getNewTransientObjectSize();
final MemoryCacheEntry newMemEntry = new MemoryCacheEntry(persContent, cacheInfo.getExpires(), cacheInfo.getRequestTime(), altObject,
altObjectSize);
final int actualApproxObjectSize = altObject == null ? 0 : Math.max(altObjectSize, persContent.length);
// Reason: Privileges needed to access CacheManager.
AccessController.doPrivileged((PrivilegedAction<Object>) () -> {
CacheManager.getInstance().putTransient(connectionUrl, newMemEntry, actualApproxObjectSize + persContent.length);
return null;
});
}
}
private static URL makeConnectionURL(final URL url, final String protocol) throws MalformedURLException {
if ((url.getQuery() != null) && "file".equalsIgnoreCase(protocol)) {
// Remove query (replace file with path) if "file" protocol.
final String ref = url.getRef();
final String refText = (ref == null) || (ref.length() == 0) ? "" : "#" + ref;
return new URL(protocol, url.getHost(), url.getPort(), url.getPath() + refText);
} else {
return url;
}
}
private static URL makeCompleteURL(final URL baseURL, final ParameterInfo pinfo, final boolean isGet) throws Exception,
MalformedURLException {
if (isGet && (pinfo != null)) {
final String ref = baseURL.getRef();
final String noRefForm = Urls.getNoRefForm(baseURL);
final String newURLText = completeGetUrl(noRefForm, pinfo, ref);
return new URL(newURLText);
} else {
return baseURL;
}
}
private static class RequestInfo {
private final RequestHandler requestHandler;
private volatile boolean isAborted = false;
// private volatile InputStream inputStream;
private volatile URLConnection connection;
RequestInfo(final URLConnection connection, final RequestHandler rhandler) {
this.connection = connection;
this.requestHandler = rhandler;
}
boolean isAborted() {
return this.isAborted;
}
void abort() {
try {
this.isAborted = true;
if (this.connection instanceof HttpURLConnection) {
((HttpURLConnection) this.connection).disconnect();
}
// final InputStream in = this.inputStream;
// if (in != null) {
// in.close();
// }
} catch (final Exception err) {
logger.log(Level.SEVERE, "abort()", err);
}
}
RequestHandler getRequestHandler() {
return this.requestHandler;
}
// void setConnection(final URLConnection connection, final InputStream in) {
void setConnection(final URLConnection connection) {
this.connection = connection;
// this.inputStream = in;
}
}
private class RequestHandlerTask implements SimpleThreadPoolTask {
private final RequestHandler handler;
private final AccessControlContext accessContext;
private RequestHandlerTask(final RequestHandler handler, final AccessControlContext accessContext) {
this.handler = handler;
this.accessContext = accessContext;
}
private RequestHandlerTask(final RequestHandler handler) {
this.handler = handler;
this.accessContext = null;
}
public void run() {
final SecurityManager sm = System.getSecurityManager();
if ((sm != null) && (this.accessContext != null)) {
final PrivilegedAction<Object> action = () -> {
processHandler(handler, 0, true);
return null;
};
// This way we ensure scheduled requests have the same
// protection as inline requests, particularly in relation
// to file and host name checks.
AccessController.doPrivileged(action, this.accessContext);
} else {
processHandler(this.handler, 0, true);
}
}
public void cancel() {
cancelRequestIfRunning(this.handler);
}
@Override
public int hashCode() {
return this.handler.hashCode();
}
@Override
public boolean equals(final Object other) {
return (other instanceof RequestHandlerTask) && ((RequestHandlerTask) other).handler.equals(this.handler);
}
@Override
public String toString() {
return "RequestHandlerTask[host=" + this.handler.getLatestRequestURL().getHost() + "]";
}
}
}