/* * This file is part of the Heritrix web crawler (crawler.archive.org). * * Licensed to the Internet Archive (IA) by one or more individual * contributors. * * The IA licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.archive.modules.deciderules; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.logging.Level; import java.util.logging.Logger; import org.archive.modules.AMQPProducer; import org.archive.modules.CrawlURI; import org.archive.modules.net.CrawlHost; import org.archive.modules.net.ServerCache; import org.archive.util.ArchiveUtils; import org.json.JSONObject; import org.springframework.beans.factory.annotation.Autowired; import com.rabbitmq.client.AMQP; import com.rabbitmq.client.AMQP.BasicProperties; public class DecideRuleSequenceWithAMQPFeed extends DecideRuleSequence { private static final long serialVersionUID = 1L; private static final Logger logger = Logger.getLogger(DecideRuleSequenceWithAMQPFeed.class.getName()); protected String amqpUri = "amqp://guest:guest@localhost:5672/%2f"; public String getAmqpUri() { return this.amqpUri; } public void setAmqpUri(String uri) { this.amqpUri = uri; } protected String exchange = "heritrix.realTimeFeed"; public String getExchange() { return exchange; } public void setExchange(String exchange) { this.exchange = exchange; } protected String routingKey = "scopeLog"; public String getRoutingKey() { return routingKey; } public void setRoutingKey(String routingKey) { this.routingKey = routingKey; } protected ServerCache serverCache; public ServerCache getServerCache() { return this.serverCache; } @Autowired public void setServerCache(ServerCache serverCache) { this.serverCache = serverCache; } transient protected AMQPProducer amqpProducer; protected AMQPProducer amqpProducer() { if (amqpProducer == null) { amqpProducer = new AMQPProducer(getAmqpUri(), getExchange(), getRoutingKey()); } return amqpProducer; } @Override synchronized public void stop() { if (!isRunning) { return; } super.stop(); if (amqpProducer != null) { amqpProducer.stop(); } } @Override protected void decisionMade(CrawlURI curi, DecideRule decisiveRule, int decisiveRuleNumber, DecideResult result) { super.decisionMade(curi, decisiveRule, decisiveRuleNumber, result); JSONObject jo = buildJson(curi, decisiveRuleNumber, decisiveRule, result); byte[] message; try { message = jo.toString().getBytes("UTF-8"); } catch (UnsupportedEncodingException e) { throw new RuntimeException(e); } try { amqpProducer().publishMessage(message, props); if (logger.isLoggable(Level.FINEST)) { logger.log(Level.FINEST, "sent message to amqp: " + jo); } } catch (IOException e) { logger.log(Level.WARNING, "failed to send message to amqp: " + jo, e); } } protected BasicProperties props = new AMQP.BasicProperties.Builder(). contentType("application/json").build(); protected JSONObject buildJson(CrawlURI curi, int decisiveRuleNumber, DecideRule decisiveRule, DecideResult result) { JSONObject jo = new JSONObject(); jo.put("timestamp", ArchiveUtils.getLog17Date(System.currentTimeMillis())); jo.put("decisiveRuleNo", decisiveRuleNumber); jo.put("decisiveRule", decisiveRule.getClass().getSimpleName()); jo.put("result", result.toString()); jo.put("url", curi.toString()); CrawlHost host = getServerCache().getHostFor(curi.getUURI()); if (host != null) { jo.put("host", host.fixUpName()); } else { jo.put("host", JSONObject.NULL); } jo.put("sourceSeed", curi.getSourceTag()); jo.put("via", curi.flattenVia()); return jo; } }