/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.pathirage.freshet.helpers;
import au.com.bytecode.opencsv.CSVWriter;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.schwering.irc.lib.IRCConnection;
import org.schwering.irc.lib.IRCEventListener;
import org.schwering.irc.lib.IRCModeParser;
import org.schwering.irc.lib.IRCUser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileWriter;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* Wikipedia activity feed IRC channel lister. This code is extracted from Samza examples code for the purpose
* of generating test data for Freshet.
*/
public class WikipediaActivityFeed {
private static final Logger log = LoggerFactory.getLogger(WikipediaActivityFeed.class);
private static final Random random = new Random();
private static final ObjectMapper jsonMapper = new ObjectMapper();
private final Map<String, Set<WikipediaFeedListener>> channelListeners;
private final String host;
private final int port;
private final IRCConnection conn;
private final String nick;
public WikipediaActivityFeed(String host, int port) {
this.channelListeners = new HashMap<String, Set<WikipediaFeedListener>>();
this.host = host;
this.port = port;
this.nick = "freshet-bot-" + Math.abs(random.nextInt());
this.conn = new IRCConnection(host, new int[] { port }, "", nick, nick, nick);
this.conn.addIRCEventListener(new WikipediaActivityFeedIrcListener());
this.conn.setEncoding("UTF-8");
this.conn.setPong(true);
this.conn.setColors(false);
}
public void start() {
try {
this.conn.connect();
} catch (IOException e) {
throw new RuntimeException("Unable to connect to " + host + ":" + port + ".", e);
}
}
public void stop() {
this.conn.interrupt();
try {
this.conn.join();
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while trying to shutdown IRC connection for " + host + ":" + port, e);
}
if (this.conn.isAlive()) {
throw new RuntimeException("Unable to shutdown IRC connection for " + host + ":" + port);
}
}
public void listen(String channel, WikipediaFeedListener listener) {
Set<WikipediaFeedListener> listeners = channelListeners.get(channel);
if (listeners == null) {
listeners = new HashSet<WikipediaFeedListener>();
channelListeners.put(channel, listeners);
join(channel);
}
listeners.add(listener);
}
public void unlisten(String channel, WikipediaFeedListener listener) {
Set<WikipediaFeedListener> listeners = channelListeners.get(channel);
if (listeners == null) {
throw new RuntimeException("Trying to unlisten to a channel that has no listeners in it.");
} else if (!listeners.contains(listener)) {
throw new RuntimeException("Trying to unlisten to a channel that listener is not listening to.");
}
listeners.remove(listener);
if (listeners.size() == 0) {
leave(channel);
}
}
public void join(String channel) {
conn.send("JOIN " + channel);
}
public void leave(String channel) {
conn.send("PART " + channel);
}
public class WikipediaActivityFeedIrcListener implements IRCEventListener {
public void onRegistered() {
log.info("Connected");
}
public void onDisconnected() {
log.info("Disconnected");
}
public void onError(String msg) {
log.info("Error: " + msg);
}
public void onError(int num, String msg) {
log.info("Error #" + num + ": " + msg);
}
public void onInvite(String chan, IRCUser u, String nickPass) {
log.info(chan + "> " + u.getNick() + " invites " + nickPass);
}
public void onJoin(String chan, IRCUser u) {
log.info(chan + "> " + u.getNick() + " joins");
}
public void onKick(String chan, IRCUser u, String nickPass, String msg) {
log.info(chan + "> " + u.getNick() + " kicks " + nickPass);
}
public void onMode(IRCUser u, String nickPass, String mode) {
log.info("Mode: " + u.getNick() + " sets modes " + mode + " " + nickPass);
}
public void onMode(String chan, IRCUser u, IRCModeParser mp) {
log.info(chan + "> " + u.getNick() + " sets mode: " + mp.getLine());
}
public void onNick(IRCUser u, String nickNew) {
log.info("Nick: " + u.getNick() + " is now known as " + nickNew);
}
public void onNotice(String target, IRCUser u, String msg) {
log.info(target + "> " + u.getNick() + " (notice): " + msg);
}
public void onPart(String chan, IRCUser u, String msg) {
log.info(chan + "> " + u.getNick() + " parts");
}
public void onPrivmsg(String chan, IRCUser u, String msg) {
Set<WikipediaFeedListener> listeners = channelListeners.get(chan);
if (listeners != null) {
WikipediaFeedEvent event = new WikipediaFeedEvent(System.currentTimeMillis(), chan, u.getNick(), msg);
for (WikipediaFeedListener listener : listeners) {
listener.onEvent(event);
}
}
log.debug(chan + "> " + u.getNick() + ": " + msg);
}
public void onQuit(IRCUser u, String msg) {
log.info("Quit: " + u.getNick());
}
public void onReply(int num, String value, String msg) {
log.info("Reply #" + num + ": " + value + " " + msg);
}
public void onTopic(String chan, IRCUser u, String topic) {
log.info(chan + "> " + u.getNick() + " changes topic into: " + topic);
}
public void onPing(String p) {
}
public void unknown(String a, String b, String c, String d) {
log.warn("UNKNOWN: " + a + " " + b + " " + c + " " + d);
}
}
public static interface WikipediaFeedListener {
void onEvent(WikipediaFeedEvent event);
}
public static final class WikipediaFeedEvent {
private final long time;
private final String channel;
private final String source;
private final String rawEvent;
public WikipediaFeedEvent(long time, String channel, String source, String rawEvent) {
this.time = time;
this.channel = channel;
this.source = source;
this.rawEvent = rawEvent;
}
public WikipediaFeedEvent(Map<String, Object> jsonObject) {
this((Long) jsonObject.get("time"), (String) jsonObject.get("channel"), (String) jsonObject.get("source"), (String) jsonObject.get("raw"));
}
public long getTime() {
return time;
}
public String getChannel() {
return channel;
}
public String getSource() {
return source;
}
public String getRawEvent() {
return rawEvent;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((channel == null) ? 0 : channel.hashCode());
result = prime * result + ((rawEvent == null) ? 0 : rawEvent.hashCode());
result = prime * result + ((source == null) ? 0 : source.hashCode());
result = prime * result + (int) (time ^ (time >>> 32));
return result;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
WikipediaFeedEvent other = (WikipediaFeedEvent) obj;
if (channel == null) {
if (other.channel != null)
return false;
} else if (!channel.equals(other.channel))
return false;
if (rawEvent == null) {
if (other.rawEvent != null)
return false;
} else if (!rawEvent.equals(other.rawEvent))
return false;
if (source == null) {
if (other.source != null)
return false;
} else if (!source.equals(other.source))
return false;
if (time != other.time)
return false;
return true;
}
@Override
public String toString() {
return "WikipediaFeedEvent [time=" + time + ", channel=" + channel + ", source=" + source + ", rawEvent=" + rawEvent + "]";
}
public String toJson() {
return toJson(this);
}
public static Map<String, Object> toMap(WikipediaFeedEvent event) {
Map<String, Object> jsonObject = new HashMap<String, Object>();
jsonObject.put("time", event.getTime());
jsonObject.put("channel", event.getChannel());
jsonObject.put("source", event.getSource());
jsonObject.put("raw", event.getRawEvent());
return jsonObject;
}
public static String toJson(WikipediaFeedEvent event) {
Map<String, Object> jsonObject = toMap(event);
try {
return jsonMapper.writeValueAsString(jsonObject);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
@SuppressWarnings("unchecked")
public static WikipediaFeedEvent fromJson(String json) {
try {
return new WikipediaFeedEvent((Map<String, Object>) jsonMapper.readValue(json, Map.class));
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
public static class WikipediaActivitiesToCSV implements WikipediaFeedListener {
private CSVWriter writer;
public WikipediaActivitiesToCSV(){
try {
this.writer = new CSVWriter(new FileWriter("wikipedia-actvities-" + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.S").format(new Date()) + ".csv"), ',');
} catch (IOException e) {
throw new RuntimeException("Cannot create file writer.");
}
}
@Override
public void onEvent(WikipediaFeedEvent event) {
Map<String,Object> parsedEvent = ParseWikipediaActivity.parse(event.getRawEvent());
List<String> row = new LinkedList<String>();
if(parsedEvent == null){
return;
}
row.add(event.channel);
row.add(event.source);
row.add(Long.toString(event.time));
row.add((String)parsedEvent.get("title"));
row.add((String)parsedEvent.get("user"));
row.add(Integer.toString((Integer)parsedEvent.get("diff-bytes")));
row.add((String)parsedEvent.get("diff-url"));
row.add((String)parsedEvent.get("summary"));
Map<String,Boolean> flags = (Map<String,Boolean>)parsedEvent.get("flags");
row.add(Boolean.toString(flags.get("is-minor")));
row.add(Boolean.toString(flags.get("is-talk")));
row.add(Boolean.toString(flags.get("is-bot-edit")));
row.add(Boolean.toString(flags.get("is-new")));
row.add(Boolean.toString(flags.get("is-unpatrolled")));
row.add(Boolean.toString(flags.get("is-special")));
row.add((String)parsedEvent.get("unparsed-flags"));
writer.writeNext(row.toArray(new String[row.size()]));
try {
writer.flush();
} catch (IOException e) {
throw new RuntimeException("Cannot flush the content to CSV file.");
}
}
}
public static void main(String[] args) throws InterruptedException {
WikipediaActivityFeed wikipediaActivityFeed = new WikipediaActivityFeed("irc.wikimedia.org", 6667);
wikipediaActivityFeed.start();
wikipediaActivityFeed.listen("#en.wikipedia", new WikipediaActivitiesToCSV());
Thread.sleep(60000);
wikipediaActivityFeed.stop();
}
}