/*
* Copyright (C) 2012 Michael Koppen
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package de.fhb.twitalyse.spout;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import redis.clients.jedis.Jedis;
import twitter4j.Status;
import twitter4j.StatusDeletionNotice;
import twitter4j.StatusListener;
import twitter4j.TwitterException;
import twitter4j.TwitterStream;
import twitter4j.TwitterStreamFactory;
import twitter4j.auth.AccessToken;
import twitter4j.conf.ConfigurationBuilder;
import twitter4j.json.DataObjectFactory;
import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;
import backtype.storm.utils.InprocMessaging;
import backtype.storm.utils.Utils;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
* This Spout connects to the Twitter API and opens up a Stream. The Spout
* listens for new Twitter Stati posted on the public Twitter Channel and push
* it in the System.
*
* @author Michael Koppen <koppen@fh-brandenburg.de>
*/
public class TwitterStreamSpout implements IRichSpout, StatusListener {
private final static Logger LOGGER = Logger.getLogger(TwitterStreamSpout.class.getName());
// Keys die die App identifizieren
private final String CONSUMER_KEY;
private final String CONSUMER_KEY_SECURE;
// Keys die den Account des Users identifizieren
private final String TOKEN;
private final String TOKEN_SECRET;
private int id;
private transient SpoutOutputCollector collector;
private transient TwitterStream twitterStream;
private String host;
private int port;
public TwitterStreamSpout(String consumerKey, String consumerKeySecure,
String token, String tokenSecret, String host, int port) {
this.CONSUMER_KEY = consumerKey;
this.CONSUMER_KEY_SECURE = consumerKeySecure;
this.TOKEN = token;
this.TOKEN_SECRET = tokenSecret;
this.host = host;
this.port = port;
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("id", "json"));
}
@Override
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) {
this.collector = collector;
// enable JSONStore
ConfigurationBuilder cb = new ConfigurationBuilder();
cb.setJSONStoreEnabled(true);
TwitterStreamFactory twitterStreamFactory = new TwitterStreamFactory(
cb.build());
twitterStream = twitterStreamFactory.getInstance();
AccessToken givenAccessToken = new AccessToken(TOKEN, TOKEN_SECRET);
twitterStream.setOAuthConsumer(CONSUMER_KEY, CONSUMER_KEY_SECURE);
twitterStream.setOAuthAccessToken(givenAccessToken);
twitterStream.addListener(this);
twitterStream.sample();
}
@Override
public void onStatus(Status status) {
String json = DataObjectFactory.getRawJSON(status);
id = InprocMessaging.acquireNewPort();
InprocMessaging.sendMessage(id, new Values(status.getId(), json));
}
@Override
public void nextTuple() {
List<Object> value = (List<Object>) InprocMessaging.pollMessage(id);
if (value == null) {
Utils.sleep(50);
} else {
Jedis jedis = new Jedis(host, port);
jedis.getClient().setTimeout(9999);
Date today = new Date();
SimpleDateFormat sdf = new SimpleDateFormat("dd_MM_yyyy");
// Saves # of all stati
jedis.incr("#stati");
// Saves # of stati today
jedis.incr("#stati_" + sdf.format(today));
// Status ID + Status-JSON
collector.emit(new Values(value.get(0), value.get(1)), id);
jedis.disconnect();
}
}
@Override
public void close() {
twitterStream.shutdown();
}
@Override
public void ack(Object msgId) {
}
@Override
public void fail(Object msgId) {
LOGGER.log(Level.SEVERE, "FAIL {0}", msgId.toString());
}
@Override
public void onException(Exception ex) {
LOGGER.log(Level.SEVERE, null, ex);
TwitterException tex = (TwitterException) ex;
if (400 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,
"Rate limit texceeded. Clients may not make more than {0} requests per hour. \nThe ntext reset is {1}",
new Object[] { tex.getRateLimitStatus().getHourlyLimit(),
tex.getRateLimitStatus().getResetTime() });
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (401 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,"Authentication credentials were missing or incorrect.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (403 == tex.getStatusCode()) {
LOGGER.log(Level.SEVERE,"Duplicated status.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (404 == tex.getStatusCode()) {
LOGGER.log(Level.SEVERE,"The URI requested is invalid or the resource requested, such as a user, does not exists.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (406 == tex.getStatusCode()) {
LOGGER.log(Level.SEVERE,"Request returned - invalid format is specified in the request.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (420 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,"Too many logins with your account in a short time.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (500 == tex.getStatusCode()) {
LOGGER.log(Level.SEVERE,"Something is broken. Please post to the group so the Twitter team can investigate.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (502 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,"Twitter is down or being upgraded.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (503 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,"The Twitter servers are up, but overloaded with requests. Try again later.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else if (-1 == tex.getStatusCode()) {
close();
LOGGER.log(Level.SEVERE,"Can not connect to the internet or the host is down.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
} else {
close();
LOGGER.log(Level.SEVERE,"Unknown twitter-error occured.");
LOGGER.log(Level.SEVERE,"Exception: {0},\nMessage: {1},\nCause: {2}",
new Object[] { tex, tex.getMessage(), tex.getCause() });
}
}
@Override
public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
}
@Override
public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
}
@Override
public void onScrubGeo(long userId, long upToStatusId) {
}
@Override
public void activate() {
}
@Override
public void deactivate() {
}
@Override
public Map<String, Object> getComponentConfiguration() {
return new HashMap<String, Object>();
}
}