/* ********************************************************************** **
** Copyright notice **
** **
** (c) 2005-2009 RSSOwl Development Team **
** http://www.rssowl.org/ **
** **
** All rights reserved **
** **
** This program and the accompanying materials are made available under **
** the terms of the Eclipse Public License v1.0 which accompanies this **
** distribution, and is available at: **
** http://www.rssowl.org/legal/epl-v10.html **
** **
** A copy is found in the file epl-v10.html and important notices to the **
** license from the team is found in the textfile LICENSE.txt distributed **
** in this package. **
** **
** This copyright notice MUST APPEAR in all copies of the file! **
** **
** Contributors: **
** RSSOwl Development Team - initial API and implementation **
** **
** ********************************************************************** */
package org.rssowl.core.internal.connection;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.NTCredentials;
import org.apache.commons.httpclient.SimpleHttpConnectionManager;
import org.apache.commons.httpclient.URIException;
import org.apache.commons.httpclient.UsernamePasswordCredentials;
import org.apache.commons.httpclient.auth.AuthScope;
import org.apache.commons.httpclient.auth.AuthState;
import org.apache.commons.httpclient.cookie.CookiePolicy;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.protocol.DefaultProtocolSocketFactory;
import org.apache.commons.httpclient.protocol.Protocol;
import org.apache.commons.httpclient.protocol.ProtocolSocketFactory;
import org.eclipse.core.runtime.Assert;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.osgi.util.NLS;
import org.osgi.service.url.URLStreamHandlerService;
import org.rssowl.core.Owl;
import org.rssowl.core.connection.AuthenticationRequiredException;
import org.rssowl.core.connection.ConnectionException;
import org.rssowl.core.connection.CredentialsException;
import org.rssowl.core.connection.HttpConnectionInputStream;
import org.rssowl.core.connection.IAbortable;
import org.rssowl.core.connection.IConditionalGetCompatible;
import org.rssowl.core.connection.IConnectionPropertyConstants;
import org.rssowl.core.connection.ICredentials;
import org.rssowl.core.connection.ICredentialsProvider;
import org.rssowl.core.connection.IProtocolHandler;
import org.rssowl.core.connection.IProxyCredentials;
import org.rssowl.core.connection.MonitorCanceledException;
import org.rssowl.core.connection.NotModifiedException;
import org.rssowl.core.connection.ProxyAuthenticationRequiredException;
import org.rssowl.core.connection.SyncConnectionException;
import org.rssowl.core.internal.Activator;
import org.rssowl.core.interpreter.EncodingException;
import org.rssowl.core.persist.IConditionalGet;
import org.rssowl.core.persist.IFeed;
import org.rssowl.core.persist.IModelFactory;
import org.rssowl.core.util.CoreUtils;
import org.rssowl.core.util.StringUtils;
import org.rssowl.core.util.SyncUtils;
import org.rssowl.core.util.Triple;
import org.rssowl.core.util.URIUtils;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
/**
* The <code>DefaultFeedHandler</code> is an implementation of
* <code>IProtocolHandler</code> that works on HTTP, HTTPS and the FILE
* Protocol. After loading the Inputstream of the given URL, the stream is
* passed to the Interpreter-Component to interpret it as one of the supported
* XML-Formats for Newsfeeds.
*
* @author bpasero
*/
public class DefaultProtocolHandler implements IProtocolHandler {
/* Http Status Codes */
private static final int HTTP_ERRORS = 400;
private static final int HTTP_STATUS_NOT_MODIFIED = 304;
private static final int HTTP_ERROR_AUTH_REQUIRED = 401;
private static final int HTTP_ERROR_FORBIDDEN = 403;
private static final int HTTP_ERROR_PROXY_AUTH_REQUIRED = 407;
/* Header Constants */
private static final String HEADER_REQUEST_COOKIE = "Cookie"; //$NON-NLS-1$
private static final String HEADER_REQUEST_ACCEPT_LANGUAGE = "Accept-Language"; //$NON-NLS-1$
private static final String HEADER_REQUEST_USER_AGENT = "User-Agent"; //$NON-NLS-1$
private static final String HEADER_REQUEST_ACCEPT_ENCODING = "Accept-Encoding"; //$NON-NLS-1$
private static final String HEADER_RESPONSE_IF_NONE_MATCH = "If-None-Match"; //$NON-NLS-1$
private static final String HEADER_RESPONSE_IF_MODIFIED_SINCE = "If-Modified-Since"; //$NON-NLS-1$
private static final String HEADER_RESPONSE_CONTENT_ENCODING = "Content-Encoding"; //$NON-NLS-1$
/* Google Error Response Codes */
private static final String HEADER_RESPONSE_ERROR = "Error"; //$NON-NLS-1$
private static final String HEADER_RESPONSE_URL = "Url"; //$NON-NLS-1$
private static final String ERROR_BAD_AUTH = "BadAuthentication"; //$NON-NLS-1$
private static final String ERROR_NOT_VERIFIED = "NotVerified"; //$NON-NLS-1$
private static final String ERROR_NO_TERMS = "TermsNotAgreed"; //$NON-NLS-1$
private static final String ERROR_CAPTCHA_REQUIRED = "CaptchaRequired"; //$NON-NLS-1$
private static final String ERROR_UNKNOWN = "Unknown"; //$NON-NLS-1$
private static final String ERROR_ACCOUNT_DELETED = "AccountDeleted"; //$NON-NLS-1$
private static final String ERROR_ACCOUNT_DISABLED = "AccountDisabled"; //$NON-NLS-1$
private static final String ERROR_SERVICE_DISABLED = "ServiceDisabled"; //$NON-NLS-1$
private static final String ERROR_SERVICE_UNAVAILABLE = "ServiceUnavailable"; //$NON-NLS-1$
/** Property to tell the XML parser to use platform encoding */
public static final String USE_PLATFORM_ENCODING = "org.rssowl.core.internal.connection.DefaultProtocolHandler.UsePlatformEncoding"; //$NON-NLS-1$
/* The Default Connection Timeout */
private static final int DEFAULT_CON_TIMEOUT = 30000;
/* Timeout for loading a Feed or Label for a Feed */
private static final int FEED_LABEL_CON_TIMEOUT = 10000;
/* Timeout for loading a Favicon */
private static final int FAVICON_CON_TIMEOUT = 5000;
/* Set a limit for titles that are looked up from feeds */
private static final int MAX_DETECTED_TITLE_LENGTH = 1024;
private static final String USER_AGENT = CoreUtils.getUserAgent();
private static boolean fgSSLInitialized;
private static boolean fgFeedProtocolInitialized;
/*
* @see org.rssowl.core.connection.IProtocolHandler#reload(java.net.URI,
* org.eclipse.core.runtime.IProgressMonitor, java.util.Map)
*/
public Triple<IFeed, IConditionalGet, URI> reload(URI link, IProgressMonitor monitor, Map<Object, Object> properties) throws CoreException {
IModelFactory typesFactory = Owl.getModelFactory();
/* Create a new empty feed from the existing one */
IFeed feed = typesFactory.createFeed(null, link);
/* Add Monitor to support early cancelation */
if (properties == null)
properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
/* Retrieve the InputStream out of the Feed's Link */
InputStream inS = openStream(link, properties);
/* Retrieve Conditional Get if present */
IConditionalGet conditionalGet = getConditionalGet(link, inS);
/* Return on Cancelation or Shutdown */
if (monitor.isCanceled()) {
closeStream(inS, true);
return null;
}
/* Pass the Stream to the Interpreter */
try {
Owl.getInterpreter().interpret(inS, feed, null);
} catch (EncodingException e) {
/* Return on Cancelation or Shutdown */
if (monitor.isCanceled()) {
closeStream(inS, true);
return null;
}
/* Re-retrieve InputStream from the Feed's Link */
inS = openStream(link, properties);
/* Re-retrieve Conditional Get if present */
conditionalGet = getConditionalGet(link, inS);
/* Return on Cancelation or Shutdown */
if (monitor.isCanceled()) {
closeStream(inS, true);
return null;
}
/* Second try: Use platform encoding */
Owl.getInterpreter().interpret(inS, feed, Collections.singletonMap((Object) USE_PLATFORM_ENCODING, (Object) Boolean.TRUE));
}
/* Return actual URI that was connected to (supporting redirects) */
if (inS instanceof HttpConnectionInputStream)
return Triple.create(feed, conditionalGet, ((HttpConnectionInputStream) inS).getLink());
/* Otherwise just use input URI */
return Triple.create(feed, conditionalGet, link);
}
protected IConditionalGet getConditionalGet(URI link, InputStream inS) {
IModelFactory typesFactory = Owl.getModelFactory();
if (inS instanceof IConditionalGetCompatible) {
String ifModifiedSince = ((IConditionalGetCompatible) inS).getIfModifiedSince();
String ifNoneMatch = ((IConditionalGetCompatible) inS).getIfNoneMatch();
if (ifModifiedSince != null || ifNoneMatch != null)
return typesFactory.createConditionalGet(ifModifiedSince, link, ifNoneMatch);
}
return null;
}
protected void closeStream(InputStream inS, boolean abort) {
try {
if (abort && inS instanceof IAbortable)
((IAbortable) inS).abort();
else if (inS != null)
inS.close();
} catch (IOException ex) {
/* Ignore */
}
}
/*
* @see org.rssowl.core.connection.IProtocolHandler#getFeedIcon(java.net.URI,
* org.eclipse.core.runtime.IProgressMonitor)
*/
public byte[] getFeedIcon(URI link, IProgressMonitor monitor) {
/* Try to load the Favicon directly from the supplied Link */
byte[] favicon = loadFavicon(link, false, false, monitor);
/* Fallback: Scan the Homepage of the Link for a Favicon entry */
if (favicon == null || favicon.length == 0) {
try {
URI topLevelUri = URIUtils.toTopLevel(link);
if (topLevelUri != null) {
URI faviconUri = getFavicon(topLevelUri, monitor);
if (faviconUri != null && faviconUri.isAbsolute())
return loadFavicon(faviconUri, true, false, monitor);
}
} catch (ConnectionException e) {
} catch (URISyntaxException e) {
} catch (Throwable t) {
Activator.getDefault().logError(t.getMessage(), t);
}
}
return favicon;
}
/* Load a possible Favicon from the given Feed */
byte[] loadFavicon(URI link, boolean isFavicon, boolean rewriteHost, IProgressMonitor monitor) {
InputStream inS = null;
boolean isError = false;
try {
/* Define Properties for Connection */
Map<Object, Object> properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.CON_TIMEOUT, FAVICON_CON_TIMEOUT);
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
/* Load Favicon */
URI faviconLink = isFavicon ? link : URIUtils.toFaviconUrl(link, rewriteHost);
if (faviconLink == null)
return null;
inS = openStream(faviconLink, properties);
ByteArrayOutputStream fos = new ByteArrayOutputStream();
byte buffer[] = new byte[0xffff];
int nbytes;
while ((nbytes = inS.read(buffer)) != -1)
fos.write(buffer, 0, nbytes);
return fos.toByteArray();
} catch (URISyntaxException e) {
/* Ignore */
} catch (ConnectionException e) {
isError = true;
/* Try rewriting the Host to obtain the Favicon */
if (!rewriteHost && !isFavicon) {
String exceptionName = e.getClass().getName();
/* Only retry in case this is a generic ConnectionException */
if (ConnectionException.class.getName().equals(exceptionName))
return loadFavicon(link, false, true, monitor);
}
} catch (IOException e) {
/* Ignore */
} finally {
closeStream(inS, isError);
}
return null;
}
private URI getFavicon(URI link, IProgressMonitor monitor) throws ConnectionException {
/* Define Properties for Connection */
Map<Object, Object> properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
properties.put(IConnectionPropertyConstants.CON_TIMEOUT, FAVICON_CON_TIMEOUT);
/* Open Stream */
InputStream inS = openStream(link, properties);
BufferedInputStream bufIns = new BufferedInputStream(inS);
BufferedReader reader = new BufferedReader(new InputStreamReader(bufIns));
try {
/* Use real Base if possible */
if (inS instanceof HttpConnectionInputStream)
return CoreUtils.findFavicon(reader, ((HttpConnectionInputStream) inS).getLink(), monitor);
/* Otherwise use request URI */
return CoreUtils.findFavicon(reader, link, monitor);
}
/* Finally close the Stream */
finally {
closeStream(inS, true); //Abort the stream to avoid downloading the full content
}
}
/**
* Do not override default URLStreamHandler of HTTP/HTTPS and therefor return
* NULL.
*
* @see org.rssowl.core.connection.IProtocolHandler#getURLStreamHandler()
*/
public URLStreamHandlerService getURLStreamHandler() {
return null;
}
/*
* @see org.rssowl.core.connection.IProtocolHandler#openStream(java.net.URI,
* org.eclipse.core.runtime.IProgressMonitor, java.util.Map)
*/
public InputStream openStream(URI link, IProgressMonitor monitor, Map<Object, Object> properties) throws ConnectionException {
/* Add Monitor to support early cancelation */
if (monitor != null) {
if (properties == null)
properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
}
return openStream(link, properties);
}
/**
* Load the Contents of the given URL by connecting to it. The additional
* properties may be used in conjunction with the
* <code>IConnectionPropertyConstants</code> to define connection related
* properties..
*
* @param link The URL to load.
* @param properties Connection related properties as defined in
* <code>IConnectionPropertyConstants</code> for example, or <code>NULL</code>
* if none.
* @return The Content of the URL as InputStream.
* @throws ConnectionException Checked Exception to be used in case of any
* Exception.
* @see AuthenticationRequiredException
* @see NotModifiedException
*/
protected InputStream openStream(URI link, Map<Object, Object> properties) throws ConnectionException {
/* Retrieve the InputStream out of the Link */
try {
return internalOpenStream(link, link, null, properties);
}
/* Handle Authentication Required */
catch (AuthenticationRequiredException e) {
/* Realm required from here on */
if (e.getRealm() == null)
throw e;
/* Try to load credentials using Host / Port / Realm */
URI normalizedUri = URIUtils.normalizeUri(link, true);
ICredentials authCredentials = Owl.getConnectionService().getAuthCredentials(normalizedUri, e.getRealm());
/* Credentials based on Host / Port / Realm provided */
if (authCredentials != null) {
/* Store for plain URI too */
ICredentialsProvider credProvider = Owl.getConnectionService().getCredentialsProvider(link);
if (credProvider.getPersistedAuthCredentials(normalizedUri, e.getRealm()) != null)
credProvider.setAuthCredentials(authCredentials, link, null);
else
credProvider.setInMemoryAuthCredentials(authCredentials, link, null);
/* Reopen Stream */
try {
return internalOpenStream(link, normalizedUri, e.getRealm(), properties);
} catch (AuthenticationRequiredException ex) {
Owl.getConnectionService().getCredentialsProvider(normalizedUri).deleteAuthCredentials(normalizedUri, e.getRealm());
throw ex;
}
}
/* Otherwise throw exception to callee */
throw e;
}
}
private InputStream internalOpenStream(URI link, URI authLink, String authRealm, Map<Object, Object> properties) throws ConnectionException {
/* Handle File Protocol at first */
if (URIUtils.FILE_SCHEME.equals(link.getScheme()))
return loadFileProtocol(link);
/* SSL Support */
if (URIUtils.HTTPS_SCHEME.equals(link.getScheme()))
initSSLProtocol();
/* Feed Support */
if (URIUtils.FEED_SCHEME.equals(link.getScheme()))
initFeedProtocol();
/* Init Client */
HttpClient client = initClient(properties);
/* Init the connection */
HttpMethodBase method = null;
InputStream inS = null;
try {
/* Create Method (GET or POST) */
method = initConnection(link, properties);
/* Proxy if required */
setupProxy(link, client);
/* Authentication if required */
setupAuthentication(authLink, authRealm, client, method);
/* Open the connection */
inS = openConnection(client, method);
/* Try to pipe the resulting stream into a GZipInputStream */
if (inS != null)
inS = pipeStream(inS, method);
} catch (IOException e) {
abortAndRelease(method);
throw new ConnectionException(Activator.getDefault().createErrorStatus(e.getMessage(), e));
}
/* In case authentication required */
int statusCode = method.getStatusCode();
if (statusCode == HTTP_ERROR_AUTH_REQUIRED) {
AuthState hostAuthState = method.getHostAuthState();
String realm = hostAuthState != null ? hostAuthState.getRealm() : null;
abortAndRelease(method);
throw new AuthenticationRequiredException(realm, Activator.getDefault().createErrorStatus(Messages.DefaultProtocolHandler_ERROR_AUTHENTICATION_REQUIRED, null));
}
/* In case sync authentication failed (Forbidden) */
else if (isSyncAuthenticationIssue(method, authLink)) {
abortAndRelease(method);
throw new AuthenticationRequiredException(null, Activator.getDefault().createErrorStatus(Messages.DefaultProtocolHandler_GR_ERROR_BAD_AUTH, null));
}
/* In case of Forbidden Status with Error Code (Google Reader) */
else if (statusCode == HTTP_ERROR_FORBIDDEN && method.getResponseHeader(HEADER_RESPONSE_ERROR) != null)
handleForbidden(method);
/* In case proxy-authentication required / failed */
else if (statusCode == HTTP_ERROR_PROXY_AUTH_REQUIRED) {
abortAndRelease(method);
throw new ProxyAuthenticationRequiredException(Activator.getDefault().createErrorStatus(Messages.DefaultProtocolHandler_ERROR_PROXY_AUTHENTICATION_REQUIRED, null));
}
/* If status code is 4xx, throw an IOException with the status code included */
else if (statusCode >= HTTP_ERRORS) {
String error = getError(statusCode);
abortAndRelease(method);
if (error != null)
throw new ConnectionException(Activator.getDefault().createErrorStatus(NLS.bind(Messages.DefaultProtocolHandler_ERROR_HTTP_STATUS_MSG, String.valueOf(statusCode), error), null));
throw new ConnectionException(Activator.getDefault().createErrorStatus(NLS.bind(Messages.DefaultProtocolHandler_ERROR_HTTP_STATUS, String.valueOf(statusCode)), null));
}
/* In case the Feed has not been modified since */
else if (statusCode == HTTP_STATUS_NOT_MODIFIED) {
abortAndRelease(method);
throw new NotModifiedException(Activator.getDefault().createInfoStatus(Messages.DefaultProtocolHandler_INFO_NOT_MODIFIED_SINCE, null));
}
/* In case response body is not available */
if (inS == null) {
abortAndRelease(method);
throw new ConnectionException(Activator.getDefault().createErrorStatus(Messages.DefaultProtocolHandler_ERROR_STREAM_UNAVAILABLE, null));
}
/* Check wether a Progress Monitor is provided to support early cancelation */
IProgressMonitor monitor = null;
if (properties != null && properties.containsKey(IConnectionPropertyConstants.PROGRESS_MONITOR))
monitor = (IProgressMonitor) properties.get(IConnectionPropertyConstants.PROGRESS_MONITOR);
/* Return a Stream that releases the connection once closed */
return new HttpConnectionInputStream(link, method, monitor, inS);
}
private void abortAndRelease(HttpMethodBase method) {
if (method != null) {
method.abort();
method.releaseConnection();
}
}
private boolean isSyncAuthenticationIssue(HttpMethodBase method, URI link) {
/* Handle Google Error Response "Forbidden" for synced connections */
if (method.getStatusCode() == HTTP_ERROR_FORBIDDEN && SyncUtils.fromGoogle(link.toString())) {
Header errorHeader = method.getResponseHeader(HEADER_RESPONSE_ERROR);
if (errorHeader == null || ERROR_BAD_AUTH.equals(errorHeader.getValue()))
return true;
}
return false;
}
protected void handleForbidden(HttpMethodBase method) throws ConnectionException {
String errorMsg = null;
String errorUrl = null;
/* Lookup Google Error if present */
Header errorHeader = method.getResponseHeader(HEADER_RESPONSE_ERROR);
if (errorHeader != null && StringUtils.isSet(errorHeader.getValue())) {
String errorCode = errorHeader.getValue();
if (ERROR_BAD_AUTH.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_BAD_AUTH;
else if (ERROR_NOT_VERIFIED.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_NOT_VERIFIED;
else if (ERROR_NO_TERMS.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_NO_TERMS;
else if (ERROR_UNKNOWN.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_UNKNOWN;
else if (ERROR_ACCOUNT_DELETED.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_ACCOUNT_DELETED;
else if (ERROR_ACCOUNT_DISABLED.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_ACCOUNT_DISABLED;
else if (ERROR_SERVICE_DISABLED.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_SERVICE_DISABLED;
else if (ERROR_SERVICE_UNAVAILABLE.equals(errorCode))
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_SERVICE_UNAVAILABLE;
else if (ERROR_CAPTCHA_REQUIRED.equals(errorCode)) {
errorMsg = Messages.DefaultProtocolHandler_GR_ERROR_CAPTCHA_REQUIRED;
errorUrl = SyncUtils.CAPTCHA_UNLOCK_URL;
}
/* Also look up specified Error URL as necessary */
if (errorUrl == null) {
Header urlHeader = method.getResponseHeader(HEADER_RESPONSE_URL);
if (urlHeader != null && StringUtils.isSet(urlHeader.getValue()))
errorUrl = urlHeader.getValue();
}
}
/* Otherwise throw generic Forbidden Exception */
if (errorMsg == null)
errorMsg = Messages.DefaultProtocolHandler_ERROR_FORBIDDEN;
abortAndRelease(method);
if (errorUrl != null)
throw new SyncConnectionException(errorUrl, Activator.getDefault().createErrorStatus(errorMsg, null));
throw new ConnectionException(Activator.getDefault().createErrorStatus(errorMsg, null));
}
/* Some HTTP Error Messages */
private String getError(int errorCode) {
switch (errorCode) {
case 400:
return "Bad Request"; //$NON-NLS-1$
case 403:
return "Forbidden"; //$NON-NLS-1$
case 404:
return "Not Found"; //$NON-NLS-1$
case 408:
return "Request Timeout"; //$NON-NLS-1$
case 500:
return "Internal Server Error"; //$NON-NLS-1$
case 502:
return "Bad Gateway"; //$NON-NLS-1$
case 503:
return "Service Unavailable"; //$NON-NLS-1$
}
return null;
}
private InputStream loadFileProtocol(URI link) throws ConnectionException {
try {
File file = new File(link);
return new BufferedInputStream(new FileInputStream(file));
} catch (FileNotFoundException e) {
throw new ConnectionException(Activator.getDefault().createErrorStatus(e.getMessage(), e));
}
}
private void setupAuthentication(URI link, String realm, HttpClient client, HttpMethodBase method) throws URIException, CredentialsException {
ICredentials authCredentials = Owl.getConnectionService().getAuthCredentials(link, realm);
if (authCredentials != null) {
client.getParams().setAuthenticationPreemptive(true);
/* Require Host */
String host = method.getURI().getHost();
/* Create the UsernamePasswordCredentials */
NTCredentials userPwCreds = new NTCredentials(authCredentials.getUsername(), authCredentials.getPassword(), host, (authCredentials.getDomain() != null) ? authCredentials.getDomain() : ""); //$NON-NLS-1$
/* Authenticate to the Server */
client.getState().setCredentials(AuthScope.ANY, userPwCreds);
method.setDoAuthentication(true);
}
}
private void setupProxy(URI link, HttpClient client) throws CredentialsException {
IProxyCredentials creds = Owl.getConnectionService().getProxyCredentials(link);
if (creds != null) {
/* Apply Proxy Config to HTTPClient */
client.getParams().setAuthenticationPreemptive(true);
client.getHostConfiguration().setProxy(creds.getHost(), creds.getPort());
/* Authenticate if required */
if (creds.getUsername() != null || creds.getPassword() != null) {
String user = StringUtils.isSet(creds.getUsername()) ? creds.getUsername() : ""; //$NON-NLS-1$
String pw = StringUtils.isSet(creds.getPassword()) ? creds.getPassword() : ""; //$NON-NLS-1$
AuthScope proxyAuthScope = new AuthScope(creds.getHost(), creds.getPort());
/* Use NTLM Credentials if Domain is set */
if (creds.getDomain() != null)
client.getState().setProxyCredentials(proxyAuthScope, new NTCredentials(user, pw, creds.getHost(), creds.getDomain()));
/* Use normal Credentials if Domain is not set */
else
client.getState().setProxyCredentials(proxyAuthScope, new UsernamePasswordCredentials(user, pw));
}
}
}
private synchronized void initSSLProtocol() {
if (fgSSLInitialized)
return;
/* Register Easy Protocol Socket Factory with HTTPS */
Protocol easyHttpsProtocol = new Protocol(URIUtils.HTTPS_SCHEME, (ProtocolSocketFactory) Owl.getConnectionService().getSecureProtocolSocketFactory(), 443);
Protocol.registerProtocol(URIUtils.HTTPS_SCHEME, easyHttpsProtocol);
fgSSLInitialized = true;
}
private synchronized void initFeedProtocol() {
if (fgFeedProtocolInitialized)
return;
Protocol feed = new Protocol(URIUtils.FEED_SCHEME, new DefaultProtocolSocketFactory(), 80);
Protocol.registerProtocol(URIUtils.FEED_SCHEME, feed);
fgFeedProtocolInitialized = true;
}
private void setHeaders(Map<Object, Object> properties, HttpMethodBase method) {
method.setRequestHeader(HEADER_REQUEST_ACCEPT_ENCODING, "gzip"); //$NON-NLS-1$
method.setRequestHeader(HEADER_REQUEST_USER_AGENT, USER_AGENT);
/* Add Conditional GET Headers if present */
if (properties != null) {
String ifModifiedSince = (String) properties.get(IConnectionPropertyConstants.IF_MODIFIED_SINCE);
String ifNoneMatch = (String) properties.get(IConnectionPropertyConstants.IF_NONE_MATCH);
if (ifModifiedSince != null)
method.setRequestHeader(HEADER_RESPONSE_IF_MODIFIED_SINCE, ifModifiedSince);
if (ifNoneMatch != null)
method.setRequestHeader(HEADER_RESPONSE_IF_NONE_MATCH, ifNoneMatch);
}
/* Add Accept-Language Header if present */
if (properties != null && properties.containsKey(IConnectionPropertyConstants.ACCEPT_LANGUAGE))
method.setRequestHeader(HEADER_REQUEST_ACCEPT_LANGUAGE, (String) properties.get(IConnectionPropertyConstants.ACCEPT_LANGUAGE));
/* Add Cookie Header if present */
if (properties != null && properties.containsKey(IConnectionPropertyConstants.COOKIE))
method.setRequestHeader(HEADER_REQUEST_COOKIE, (String) properties.get(IConnectionPropertyConstants.COOKIE));
/* Add more Headers */
if (properties != null && properties.containsKey(IConnectionPropertyConstants.HEADERS)) {
Map<?, ?> headers = (Map<?, ?>) properties.get(IConnectionPropertyConstants.HEADERS);
Set<?> entries = headers.entrySet();
for (Object obj : entries) {
Entry<?, ?> entry = (Entry<?, ?>) obj;
method.setRequestHeader((String) entry.getKey(), (String) entry.getValue());
}
}
}
private HttpClient initClient(Map<Object, Object> properties) {
/* Retrieve Connection Timeout from Properties if set */
int conTimeout = DEFAULT_CON_TIMEOUT;
if (properties != null && properties.containsKey(IConnectionPropertyConstants.CON_TIMEOUT))
conTimeout = (Integer) properties.get(IConnectionPropertyConstants.CON_TIMEOUT);
/* Create a new HttpClient */
HttpClient client = new HttpClient(new SimpleHttpConnectionManager(true));
/* Socket Timeout - Max. time to wait for an answer */
client.getHttpConnectionManager().getParams().setSoTimeout(conTimeout);
/* Connection Timeout - Max. time to wait for a connection */
client.getHttpConnectionManager().getParams().setConnectionTimeout(conTimeout);
return client;
}
private HttpMethodBase initConnection(URI link, Map<Object, Object> properties) throws IOException {
/* Create the Method. Wrap any RuntimeException into an IOException */
HttpMethodBase method = null;
try {
if (properties != null && properties.containsKey(IConnectionPropertyConstants.POST))
method = new PostMethod(link.toString());
else
method = new GetMethod(link.toString());
} catch (RuntimeException e) {
throw new IOException(e.getMessage());
}
/* Ignore Cookies */
if (method instanceof GetMethod)
method.getParams().setCookiePolicy(CookiePolicy.IGNORE_COOKIES);
/* Set Headers */
setHeaders(properties, method);
/* Set Parameters (POST only) */
if (method instanceof PostMethod && properties != null && properties.containsKey(IConnectionPropertyConstants.PARAMETERS)) {
Map<?, ?> parameters = (Map<?, ?>) properties.get(IConnectionPropertyConstants.PARAMETERS);
Set<?> entries = parameters.entrySet();
for (Object obj : entries) {
Entry<?, ?> entry = (Entry<?, ?>) obj;
String key = (String) entry.getKey();
if (entry.getValue() instanceof String)
((PostMethod) method).addParameter(key, (String) entry.getValue());
else if (entry.getValue() instanceof String[]) {
String[] parameterValues = (String[]) entry.getValue();
for (String value : parameterValues) {
((PostMethod) method).addParameter(key, value);
}
}
}
}
/* Follow Redirects */
if (method instanceof GetMethod)
method.setFollowRedirects(true);
return method;
}
private InputStream openConnection(HttpClient client, HttpMethodBase method) throws HttpException, IOException {
/* Execute the Method */
client.executeMethod(method);
/* Finally retrieve the InputStream from the respond body */
return method.getResponseBodyAsStream();
}
private InputStream pipeStream(InputStream inputStream, HttpMethodBase method) throws IOException {
Assert.isNotNull(inputStream);
/* Retrieve the Content Encoding */
String contentEncoding = method.getResponseHeader(HEADER_RESPONSE_CONTENT_ENCODING) != null ? method.getResponseHeader(HEADER_RESPONSE_CONTENT_ENCODING).getValue() : null;
boolean isGzipStream = false;
/*
* Return in case the Content Encoding is not given and the InputStream does
* not support mark() and reset()
*/
if ((contentEncoding == null || !contentEncoding.equals("gzip")) && !inputStream.markSupported()) //$NON-NLS-1$
return inputStream;
/* Content Encoding is set to gzip, so use the GZipInputStream */
if (contentEncoding != null && contentEncoding.equals("gzip")) { //$NON-NLS-1$
isGzipStream = true;
}
/* Detect if the Stream is gzip encoded */
else if (inputStream.markSupported()) {
inputStream.mark(2);
int id1 = inputStream.read();
int id2 = inputStream.read();
inputStream.reset();
/* Check for GZip Magic Numbers (See RFC 1952) */
if (id1 == 0x1F && id2 == 0x8B)
isGzipStream = true;
}
/* Create the GZipInputStream then */
if (isGzipStream) {
try {
return new GZIPInputStream(inputStream);
} catch (IOException e) {
return inputStream;
}
}
return inputStream;
}
/*
* @see org.rssowl.core.connection.IProtocolHandler#getLabel(java.net.URI,
* org.eclipse.core.runtime.IProgressMonitor)
*/
public String getLabel(URI link, IProgressMonitor monitor) throws ConnectionException {
String title = ""; //$NON-NLS-1$
/* Define Properties for Connection */
Map<Object, Object> properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
properties.put(IConnectionPropertyConstants.CON_TIMEOUT, FEED_LABEL_CON_TIMEOUT);
/* Open Stream */
InputStream inS = openStream(link, properties);
try {
/* Return on Cancelation or Shutdown */
if (monitor.isCanceled())
return null;
/* Buffered Stream to support mark and reset */
BufferedInputStream bufIns = new BufferedInputStream(inS);
bufIns.mark(8192);
/* Try to read Encoding out of XML Document */
String encoding = getEncodingFromXML(new InputStreamReader(bufIns), monitor);
/* Avoid lowercase UTF-8 notation */
if ("utf-8".equalsIgnoreCase(encoding)) //$NON-NLS-1$
encoding = "UTF-8"; //$NON-NLS-1$
/* Reset the Stream to its beginning */
bufIns.reset();
/* Grab Title using supplied Encoding */
if (StringUtils.isSet(encoding) && Charset.isSupported(encoding))
title = getTitleFromFeed(new BufferedReader(new InputStreamReader(bufIns, encoding)), monitor);
/* Grab Title using Default Encoding */
else
title = getTitleFromFeed(new BufferedReader(new InputStreamReader(bufIns)), monitor);
/* Remove the title tags (also delete attributes in title tag) */
title = title.replaceAll("<title[^>]*>", ""); //$NON-NLS-1$ //$NON-NLS-2$
title = title.replaceAll("</title>", ""); //$NON-NLS-1$ //$NON-NLS-2$
/* Remove potential CDATA Tags */
title = title.replaceAll(Pattern.quote("<![CDATA["), ""); //$NON-NLS-1$ //$NON-NLS-2$
title = title.replaceAll(Pattern.quote("]]>"), ""); //$NON-NLS-1$ //$NON-NLS-2$
} catch (IOException e) {
if (!(e instanceof MonitorCanceledException))
Activator.safeLogError(e.getMessage(), e);
}
/* Finally close the Stream */
finally {
closeStream(inS, true); //Abort the stream to avoid downloading the full content
}
// Have an upper maximum of title length to protect against issues
String result = StringUtils.stripTags(title.trim(), true);
if (result.length() > MAX_DETECTED_TITLE_LENGTH)
result = result.substring(0, MAX_DETECTED_TITLE_LENGTH);
return result;
}
/* Tries to read the encoding information from the given InputReader */
private String getEncodingFromXML(InputStreamReader inputReader, IProgressMonitor monitor) throws IOException {
String encoding = null;
/* Read the first line or until the Tag is closed */
StringBuilder strBuf = new StringBuilder();
int c;
while (!monitor.isCanceled() && (c = inputReader.read()) != -1) {
char character = (char) c;
/* Append all Characters, except for closing Tag or CR */
if (character != '>' && character != '\n' && character != '\r')
strBuf.append(character);
/* Closing Tag is the last one to append */
else if (character == '>') {
strBuf.append(character);
break;
}
/* End of Line or Tag reached */
else
break;
}
/* Save the first Line */
String firstLine = strBuf.toString();
/* Look if Encoding is supplied */
if (firstLine.indexOf("encoding") >= 0) { //$NON-NLS-1$
/* Extract the Encoding Value */
String regEx = "<\\?.*encoding=[\"']([^\\s]*)[\"'].*\\?>"; //$NON-NLS-1$
Pattern pattern = Pattern.compile(regEx);
Matcher match = pattern.matcher(firstLine);
/* Get first matching String */
if (match.find())
return match.group(1);
}
return encoding;
}
/* Tries to find the title information from the given Reader */
private String getTitleFromFeed(BufferedReader inputReader, IProgressMonitor monitor) throws IOException {
String title = ""; //$NON-NLS-1$
String firstLine;
boolean titleFound = false;
/* Read the file until the Title is found or EOF is reached */
while (true) {
/* Return on Cancelation or Shutdown */
if (monitor.isCanceled())
return null;
/* Will throw an IOException on EOF reached */
firstLine = inputReader.readLine();
/* EOF reached */
if (firstLine == null)
break;
/* If the line contains the title, break loop */
if (firstLine.indexOf("<title") >= 0 && firstLine.indexOf("</title>") >= 0) { //$NON-NLS-1$ //$NON-NLS-2$
title = firstLine.trim();
titleFound = true;
break;
}
}
/* Return if no title was found */
if (!titleFound)
return title;
/* Extract the title String */
String regEx = "<title[^>]*>[^<]*</title>"; //$NON-NLS-1$
Pattern pattern = Pattern.compile(regEx);
Matcher match = pattern.matcher(title);
/* Get first matching String */
if (match.find())
title = match.group();
return title;
}
/*
* @see org.rssowl.core.connection.IProtocolHandler#getFeed(java.net.URI,
* org.eclipse.core.runtime.IProgressMonitor)
*/
public URI getFeed(final URI website, IProgressMonitor monitor) throws ConnectionException {
/* Define Properties for Connection */
Map<Object, Object> properties = new HashMap<Object, Object>();
properties.put(IConnectionPropertyConstants.PROGRESS_MONITOR, monitor);
properties.put(IConnectionPropertyConstants.CON_TIMEOUT, FEED_LABEL_CON_TIMEOUT);
/* Open Stream */
InputStream inS = openStream(website, properties);
BufferedInputStream bufIns = new BufferedInputStream(inS);
BufferedReader reader = new BufferedReader(new InputStreamReader(bufIns));
try {
/* Our HttpConnectionInputStream */
if (inS instanceof HttpConnectionInputStream) {
/* Check the content type and return early if already a feed */
String contentType = ((HttpConnectionInputStream) inS).getContentType();
if (contentType != null) {
for (String feedContentType : CoreUtils.FEED_MIME_TYPES) {
if (contentType.toLowerCase().contains(feedContentType))
return website;
}
}
/* Use real Base if possible */
return CoreUtils.findFeed(reader, ((HttpConnectionInputStream) inS).getLink(), monitor);
}
/* Normal Stream (use request URI) */
return CoreUtils.findFeed(reader, website, monitor);
}
/* Finally close the Stream */
finally {
closeStream(inS, true); //Abort the stream to avoid downloading the full content
}
}
}