EarthquakeTweets.java

package edu.rpi.cs.datascience;

import info.aduna.iteration.CloseableIteration;
import net.fortytwo.twitlogic.TwitLogic;
import net.fortytwo.twitlogic.TwitLogicAgent;
import net.fortytwo.twitlogic.flow.Handler;
import net.fortytwo.twitlogic.model.Tweet;
import net.fortytwo.twitlogic.model.User;
import net.fortytwo.twitlogic.persistence.TweetDeleter;
import net.fortytwo.twitlogic.persistence.TweetPersister;
import net.fortytwo.twitlogic.persistence.TweetStore;
import net.fortytwo.twitlogic.persistence.TweetStoreConnection;
import net.fortytwo.twitlogic.persistence.TweetStoreException;
import net.fortytwo.twitlogic.persistence.UserRegistry;
import net.fortytwo.twitlogic.services.twitter.CommandListener;
import net.fortytwo.twitlogic.services.twitter.CustomTwitterClient;
import org.openrdf.model.Literal;
import org.openrdf.query.BindingSet;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.QueryEvaluationException;
import org.openrdf.query.impl.MapBindingSet;
import org.openrdf.query.parser.ParsedQuery;
import org.openrdf.query.parser.sparql.SPARQLParser;
import org.openrdf.sail.SailConnection;
import org.openrdf.sail.SailException;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;

/**
 * @author Joshua Shinavier (http://fortytwo.net).
 */
public class EarthquakeTweets {

    private static final String
            TIMESTAMP = "timestamp",
            LOCATION = "location",
            TEXT = "text";

    private static final String
            SELECT_DUMP_FIELDS = "" +
            "PREFIX sioc: <http://rdfs.org/sioc/ns#>\n" +
            "PREFIX dcterms: <http://purl.org/dc/terms/>\n" +
            "PREFIX foaf: <http://xmlns.com/foaf/0.1/>\n" +
            "PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>\n" +
            "SELECT ?" + TIMESTAMP + " ?" + LOCATION + " ?" + TEXT + "\n" +
            "WHERE { ?item sioc:has_creator ?user .\n" +
            "    ?user sioc:account_of ?person.\n" +
            "    ?item dcterms:created ?" + TIMESTAMP + ".\n" +
            "    ?person foaf:based_near ?place.\n" +
            "    ?place rdfs:comment ?" + LOCATION + ".\n" +
            "    ?item sioc:content ?" + TEXT + ".\n" +
            "}";

    public static void main(final String[] args) throws Exception {
        System.out.println(SELECT_DUMP_FIELDS);
        System.exit(0);
        try {
            doit();
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }

    private static void doit() throws Exception {
        try {
            Properties conf = new Properties();
            conf.load(EarthquakeTweets.class.getResourceAsStream("datascience.properties"));
            TwitLogic.setConfiguration(conf);

            // Create a persistent store.
            TweetStore store = new TweetStore();
            store.initialize();

            try {
                //store.dump(System.out);
                dumpTabSeparatedFile(store, new File("/tmp/earthquaketweets.txt"));

                CustomTwitterClient client = new CustomTwitterClient();
                UserRegistry userRegistry = new UserRegistry(client);

                TweetPersister baseStatusHandler = new TweetPersister(store, client);
                TweetDeleter d = new TweetDeleter(store);

                // Create an agent to listen for commands.
                // Also take the opportunity to memoize users we're following.
                TwitLogicAgent agent = new TwitLogicAgent(client);
                Handler<Tweet> statusHandler
                        = userRegistry.createUserRegistryFilter(
                        new CommandListener(agent, baseStatusHandler));

                Set<User> users = new HashSet<User>();
                Set<String> terms = new HashSet<String>();
                terms.add("earthquake");
                client.processFilterStream(users, terms, null, statusHandler, d, 0);

                System.out.println("Done.");
            } finally {
                store.shutDown();
            }
        } catch (Throwable t) {
            t.printStackTrace();
        }
    }

    private static void dumpTabSeparatedFile(final TweetStore store,
                                             final File outputFile) throws TweetStoreException, MalformedQueryException, SailException, QueryEvaluationException, IOException {
        OutputStream out = new FileOutputStream(outputFile);
        PrintStream ps = new PrintStream(out);

        try {
            TweetStoreConnection c = store.createConnection();
            try {
                ParsedQuery q = parseQuery(SELECT_DUMP_FIELDS);
                BindingSet bs = new MapBindingSet();
                SailConnection sc = c.getSailConnection();
                try {
                    sc.begin();
                    CloseableIteration<? extends BindingSet, QueryEvaluationException> results
                            = sc.evaluate(q.getTupleExpr(), q.getDataset(), bs, false);
                    try {
                        while (results.hasNext()) {
                            BindingSet r = results.next();
                            String timestamp = ((Literal) r.getBinding(TIMESTAMP).getValue()).getLabel().replaceAll("\t", " ");
                            String location = ((Literal) r.getBinding(LOCATION).getValue()).getLabel().replaceAll("\t", " ");
                            String text = ((Literal) r.getBinding(TEXT).getValue()).getLabel()
                                    .replaceAll("\t", " ")
                                    .replaceAll("\n", " ")
                                    .replaceAll("\r", " ");

                            ps.println(timestamp + "\t" + location + "\t" + text);
                        }
                    } finally {
                        results.close();
                    }
                } finally {
                    sc.rollback();
                    sc.close();
                }
            } finally {
                c.close();
            }
        } finally {
            out.close();
        }
    }

    private static ParsedQuery parseQuery(final String query) throws MalformedQueryException {
        SPARQLParser parser = new SPARQLParser();
        String baseURI = "http://example.org/bogusBaseURI/";
        return parser.parseQuery(query, baseURI);
    }
}