/*
* Copyright 2014-15 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.stream.module.twitter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.net.URI;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.springframework.beans.DirectFieldAccessor;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.client.ClientHttpRequest;
import org.springframework.http.client.ClientHttpRequestFactory;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.client.SimpleClientHttpRequestFactory;
import org.springframework.integration.endpoint.MessageProducerSupport;
import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
import org.springframework.social.twitter.api.impl.TwitterTemplate;
import org.springframework.util.StringUtils;
import org.springframework.web.client.DefaultResponseErrorHandler;
import org.springframework.web.client.HttpStatusCodeException;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;
/**
* Abstract class for the twitter inbound channel adapter.
*
* @author David Turanski
* @author Gary Russell
*/
//todo: Move this class to a common twitter support subproject
public abstract class AbstractTwitterInboundChannelAdapter extends MessageProducerSupport {
private final static AtomicInteger instance = new AtomicInteger();
private final TwitterTemplate twitter;
private final ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
private final Object monitor = new Object();
private final AtomicBoolean running = new AtomicBoolean(false);
// Backoff values, as per https://dev.twitter.com/docs/streaming-apis/connecting#Reconnecting
private final AtomicInteger linearBackOff = new AtomicInteger(250);
private final AtomicInteger httpErrorBackOff = new AtomicInteger(5000);
private final AtomicInteger rateLimitBackOff = new AtomicInteger(60000);
protected AbstractTwitterInboundChannelAdapter(TwitterTemplate twitter) {
this.twitter = twitter;
// Fix to get round TwitterErrorHandler not handling 401s etc.
this.twitter.getRestTemplate().setErrorHandler(new DefaultResponseErrorHandler());
this.setPhase(Integer.MAX_VALUE);
}
/**
* The read timeout for the underlying URLConnection to the twitter stream.
*/
public void setReadTimeout(int millis) {
// Hack to get round Spring's dynamic loading of http client stuff
ClientHttpRequestFactory f = getRequestFactory();
if (f instanceof SimpleClientHttpRequestFactory) {
((SimpleClientHttpRequestFactory) f).setReadTimeout(millis);
}
else {
((HttpComponentsClientHttpRequestFactory) f).setReadTimeout(millis);
}
}
/**
* The connection timeout for making a connection to Twitter.
*/
public void setConnectTimeout(int millis) {
ClientHttpRequestFactory f = getRequestFactory();
if (f instanceof SimpleClientHttpRequestFactory) {
((SimpleClientHttpRequestFactory) f).setConnectTimeout(millis);
}
else {
((HttpComponentsClientHttpRequestFactory) f).setConnectTimeout(millis);
}
}
@Override
protected void onInit() {
this.taskExecutor.setThreadNamePrefix("twitterSource-" + instance.incrementAndGet() + "-");
this.taskExecutor.initialize();
}
@Override
protected void doStart() {
synchronized (this.monitor) {
if (this.running.get()) {
// already running
return;
}
this.running.set(true);
this.taskExecutor.execute(new StreamReadingTask());
}
}
@Override
protected void doStop() {
this.running.set(false);
this.taskExecutor.getThreadPoolExecutor().shutdownNow();
try {
if (!this.taskExecutor.getThreadPoolExecutor().awaitTermination(10, TimeUnit.SECONDS)) {
logger.error("Reader task failed to stop");
}
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
}
protected abstract URI buildUri();
protected abstract void doSendLine(String line);
protected class StreamReadingTask implements Runnable {
@Override
public void run() {
while (running.get()) {
try {
readStream(twitter.getRestTemplate());
}
catch (HttpStatusCodeException sce) {
if (sce.getStatusCode() == HttpStatus.UNAUTHORIZED) {
logger.error("Twitter authentication failed: " + sce.getMessage());
running.set(false);
}
else if (sce.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS) {
waitRateLimitBackoff();
}
else {
waitHttpErrorBackoff();
}
}
catch (Exception e) {
logger.warn("Exception while reading stream.", e);
waitLinearBackoff();
}
}
}
private void readStream(RestTemplate restTemplate) {
restTemplate.execute(buildUri(), HttpMethod.GET, new RequestCallback() {
@Override
public void doWithRequest(ClientHttpRequest request) throws IOException {
}
},
new ResponseExtractor<String>() {
@Override
public String extractData(ClientHttpResponse response) throws IOException {
InputStream inputStream = response.getBody();
LineNumberReader reader = null;
try {
reader = new LineNumberReader(new InputStreamReader(inputStream));
resetBackOffs();
while (running.get()) {
String line = reader.readLine();
if (!StringUtils.hasText(line)) {
break;
}
doSendLine(line);
}
}
finally {
if (reader != null) {
reader.close();
}
}
return null;
}
}
);
}
}
private ClientHttpRequestFactory getRequestFactory() {
// InterceptingClientHttpRequestFactory doesn't let us access the underlying object
DirectFieldAccessor f = new DirectFieldAccessor(twitter.getRestTemplate().getRequestFactory());
Object requestFactory = f.getPropertyValue("requestFactory");
return (ClientHttpRequestFactory) requestFactory;
}
private void resetBackOffs() {
linearBackOff.set(250);
rateLimitBackOff.set(60000);
httpErrorBackOff.set(5000);
}
private void waitLinearBackoff() {
int millis = linearBackOff.get();
logger.warn("Exception while reading stream, waiting for " + millis + " ms before restarting");
wait(millis);
if (millis < 16000) // 16 seconds max
linearBackOff.set(millis + 250);
}
private void waitRateLimitBackoff() {
int millis = rateLimitBackOff.get();
logger.warn("Rate limit error, waiting for " + millis / 1000 + " seconds before restarting");
wait(millis);
rateLimitBackOff.set(millis * 2);
}
private void waitHttpErrorBackoff() {
int millis = httpErrorBackOff.get();
logger.warn("Http error, waiting for " + millis / 1000 + " seconds before restarting");
wait(millis);
if (millis < 320000) // 320 seconds max
httpErrorBackOff.set(millis * 2);
}
protected void wait(int millis) {
try {
Thread.sleep(millis);
}
catch (InterruptedException e) {
Thread.currentThread().interrupt();
if (!this.running.get()) {
// no longer running
return;
}
throw new IllegalStateException(e);
}
}
}