TwitterStream.java

package net.fortytwo.twitlogic.larkc;

import eu.larkc.core.data.CloseableIterator;
import eu.larkc.core.data.SetOfStatements;
import net.fortytwo.twitlogic.TwitLogic;
import net.fortytwo.twitlogic.flow.Handler;
import net.fortytwo.twitlogic.flow.NullHandler;
import net.fortytwo.twitlogic.larkc.sail.QueueingSail;
import net.fortytwo.twitlogic.logging.TweetReceivedLogger;
import net.fortytwo.twitlogic.model.Tweet;
import net.fortytwo.twitlogic.model.User;
import net.fortytwo.twitlogic.persistence.TweetPersister;
import net.fortytwo.twitlogic.persistence.TweetStore;
import net.fortytwo.twitlogic.persistence.TweetStoreException;
import net.fortytwo.twitlogic.services.twitter.HandlerException;
import net.fortytwo.twitlogic.services.twitter.CustomTwitterClient;
import net.fortytwo.twitlogic.services.twitter.TwitterClientException;
import net.fortytwo.twitlogic.syntax.Matcher;
import net.fortytwo.twitlogic.syntax.MultiMatcher;
import net.fortytwo.twitlogic.syntax.TopicSniffer;
import net.fortytwo.twitlogic.syntax.TweetAnnotator;
import net.fortytwo.twitlogic.syntax.afterthought.DemoAfterthoughtMatcher;
import net.fortytwo.twitlogic.util.Factory;
import net.fortytwo.twitlogic.util.properties.PropertyException;
import org.openrdf.model.Statement;
import org.openrdf.sail.NotifyingSail;
import org.openrdf.sail.Sail;
import org.openrdf.sail.SailConnection;
import org.openrdf.sail.SailConnectionListener;
import org.openrdf.sail.SailException;
import org.openrdf.sail.memory.MemoryStore;

import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.logging.Logger;

/**
 * A streaming SetOfStatements implementation which draws statements from a stream of RDFized tweets.
 * <p/>
 *
 * @author Joshua Shinavier (http://fortytwo.net)
 */
public class TwitterStream extends StreamingSetOfStatements {
    private static final Logger LOGGER = TwitLogic.getLogger(TwitterStream.class);

    private boolean closed = false;

    public TwitterStream(final StreamingPlugin.OverflowPolicy overflowPolicy) {
        super(overflowPolicy);
    }

    public CloseableIterator<Statement> getStatements() {
        final StatementQueuingListener l;

        int capacity;
        try {
            capacity = TwitLogic.getConfiguration().getInt(TwitLogicPlugin.QUEUE_CAPACITY, TwitLogicPlugin.DEFAULT_QUEUE_CAPACITY);
        } catch (PropertyException e) {
            LOGGER.severe(e.toString());
            throw new IllegalStateException(e);
        }

        final ArrayBlockingQueue<Statement> queue = new ArrayBlockingQueue<Statement>(capacity);

        final Factory<SailConnectionListener> factory = new Factory<SailConnectionListener>() {
            public SailConnectionListener create() {
                try {
                    return new StatementQueuingListener(queue, overflowPolicy);
                } catch (PropertyException e) {
                    LOGGER.severe(e.toString());
                    throw new IllegalStateException(e);
                }
            }
        };

        SimpleCallback onClose = new SimpleCallback() {
            public void execute() {
                closed = true;
            }
        };

        new Thread(new Runnable() {
            public void run() {
                try {
                    start(factory);
                } catch (Exception e) {
                    LOGGER.severe(e.toString());
                    throw new IllegalStateException(e);
                }
            }
        }).start();

        return new StreamingQueueIterator<Statement>(queue, onClose);
    }

    public SetOfStatements toRDF(final SetOfStatements setOfStatements) {
        // TODO: is this right?
        return this;
    }

    private void start(final Factory<SailConnectionListener> factory) throws Exception {
        NotifyingSail baseSail = new MemoryStore();
        baseSail.initialize();
        Sail sail = new QueueingSail(baseSail, factory.create());

        try {
            // Create a persistent store.
            TweetStore store = new TweetStore(sail);
            store.setSailConnectionListenerFactory(factory);
            store.doNotRefreshCoreMetadata();
            store.initialize();

            try {
                // Create a client for communication with Twitter.
                CustomTwitterClient client = new CustomTwitterClient();

                Set<User> users = TwitLogic.findFollowList(client);
                Set<String> terms = TwitLogic.findTrackTerms();

                final Handler<Tweet> annotator
                        = createAnnotator(store, client);

                final SailConnection c = sail.getConnection();
                //c.addConnectionListener(listener);

                try {
                    Handler<Tweet> adder = new Handler<Tweet>() {
                        public boolean isOpen() {
                            return !closed && annotator.isOpen();
                        }

                        public void handle(final Tweet tweet) throws HandlerException {
                            try {
                                c.clear();
                                c.commit();
                                c.begin();
                            } catch (SailException e) {
                                throw new HandlerException(e);
                            }

                            annotator.handle(tweet);
                        }
                    };

                    // Can't use a deleter here.
                    NullHandler<Tweet> d = new NullHandler<Tweet>();

                    // TODO: optionally gather historical tweets

                    TweetReceivedLogger rLogger = new TweetReceivedLogger(client.getStatistics(), adder);

                    if (0 < users.size() || 0 < terms.size()) {
                        client.processFilterStream(users, terms, null, rLogger, d, 0);
                    } else {
                        client.processSampleStream(rLogger, d);
                    }
                } finally {
                    c.close();
                }
            } finally {
                store.shutDown();
            }
        } finally {
            sail.shutDown();
        }
    }

    private static Handler<Tweet> createAnnotator(final TweetStore store,
                                                  final CustomTwitterClient client) throws TweetStoreException, TwitterClientException {
        // Create the tweet persister.
        TweetPersister persister = new TweetPersister(store, client);

        // Add a "topic sniffer".
        TopicSniffer topicSniffer = new TopicSniffer(persister);

        // Add a tweet annotator.
        Matcher matcher = new MultiMatcher(
                new DemoAfterthoughtMatcher());

        return new TweetAnnotator(matcher, topicSniffer);
    }
}