/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache license, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the license for the specific language governing permissions and
* limitations under the license.
*/
package org.apache.logging.log4j.flume.appender;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.flume.Event;
import org.apache.flume.EventDeliveryException;
import org.apache.flume.agent.embedded.EmbeddedAgent;
import org.apache.logging.log4j.LoggingException;
import org.apache.logging.log4j.core.appender.ManagerFactory;
import org.apache.logging.log4j.core.config.ConfigurationException;
import org.apache.logging.log4j.core.config.Property;
import org.apache.logging.log4j.core.util.NameUtil;
import org.apache.logging.log4j.util.PropertiesUtil;
import org.apache.logging.log4j.util.Strings;
/**
*
*/
public class FlumeEmbeddedManager extends AbstractFlumeManager {
private static final String FILE_SEP = PropertiesUtil.getProperties().getStringProperty("file.separator");
private static final String IN_MEMORY = "InMemory";
private static FlumeManagerFactory factory = new FlumeManagerFactory();
private final EmbeddedAgent agent;
private final String shortName;
/**
* Constructor
* @param name The unique name of this manager.
* @param shortName The short version of the agent name.
* @param agent The embedded agent.
*/
protected FlumeEmbeddedManager(final String name, final String shortName, final EmbeddedAgent agent) {
super(name);
this.agent = agent;
this.shortName = shortName;
}
/**
* Returns a FlumeEmbeddedManager.
* @param name The name of the manager.
* @param agents The agents to use.
* @param properties Properties for the embedded manager.
* @param batchSize The number of events to include in a batch.
* @param dataDir The directory where the Flume FileChannel should write to.
* @return A FlumeAvroManager.
*/
public static FlumeEmbeddedManager getManager(final String name, final Agent[] agents, final Property[] properties,
int batchSize, final String dataDir) {
if (batchSize <= 0) {
batchSize = 1;
}
if ((agents == null || agents.length == 0) && (properties == null || properties.length == 0)) {
throw new IllegalArgumentException("Either an Agent or properties are required");
} else if (agents != null && agents.length > 0 && properties != null && properties.length > 0) {
throw new IllegalArgumentException("Cannot configure both Agents and Properties.");
}
final StringBuilder sb = new StringBuilder();
boolean first = true;
if (agents != null && agents.length > 0) {
sb.append(name).append('[');
for (final Agent agent : agents) {
if (!first) {
sb.append('_');
}
sb.append(agent.getHost()).append('-').append(agent.getPort());
first = false;
}
sb.append(']');
} else {
String sep = Strings.EMPTY;
sb.append(name).append('-');
final StringBuilder props = new StringBuilder();
for (final Property prop : properties) {
props.append(sep);
props.append(prop.getName()).append('=').append(prop.getValue());
sep = "_";
}
sb.append(NameUtil.md5(props.toString()));
}
return getManager(sb.toString(), factory,
new FactoryData(name, agents, properties, batchSize, dataDir));
}
@Override
public void send(final Event event) {
try {
agent.put(event);
} catch (final EventDeliveryException ex) {
throw new LoggingException("Unable to deliver event to Flume Appender " + shortName, ex);
}
}
@Override
protected boolean releaseSub(final long timeout, final TimeUnit timeUnit) {
agent.stop();
return true;
}
/**
* Factory data.
*/
private static class FactoryData {
private final Agent[] agents;
private final Property[] properties;
private final int batchSize;
private final String dataDir;
private final String name;
/**
* Constructor.
* @param name The name of the Appender.
* @param agents The agents.
* @param properties The Flume configuration properties.
* @param batchSize The number of events to include in a batch.
* @param dataDir The directory where Flume should write to.
*/
public FactoryData(final String name, final Agent[] agents, final Property[] properties, final int batchSize,
final String dataDir) {
this.name = name;
this.agents = agents;
this.batchSize = batchSize;
this.properties = properties;
this.dataDir = dataDir;
}
}
/**
* Avro Manager Factory.
*/
private static class FlumeManagerFactory implements ManagerFactory<FlumeEmbeddedManager, FactoryData> {
/**
* Create the FlumeAvroManager.
* @param name The name of the entity to manage.
* @param data The data required to create the entity.
* @return The FlumeAvroManager.
*/
@Override
public FlumeEmbeddedManager createManager(final String name, final FactoryData data) {
try {
final Map<String, String> props = createProperties(data.name, data.agents, data.properties,
data.batchSize, data.dataDir);
final EmbeddedAgent agent = new EmbeddedAgent(name);
agent.configure(props);
agent.start();
LOGGER.debug("Created Agent " + name);
return new FlumeEmbeddedManager(name, data.name, agent);
} catch (final Exception ex) {
LOGGER.error("Could not create FlumeEmbeddedManager", ex);
}
return null;
}
private Map<String, String> createProperties(final String name, final Agent[] agents,
final Property[] properties, final int batchSize, String dataDir) {
final Map<String, String> props = new HashMap<>();
if ((agents == null || agents.length == 0) && (properties == null || properties.length == 0)) {
LOGGER.error("No Flume configuration provided");
throw new ConfigurationException("No Flume configuration provided");
}
if (agents != null && agents.length > 0 && properties != null && properties.length > 0) {
LOGGER.error("Agents and Flume configuration cannot both be specified");
throw new ConfigurationException("Agents and Flume configuration cannot both be specified");
}
if (agents != null && agents.length > 0) {
if (Strings.isNotEmpty(dataDir)) {
if (dataDir.equals(IN_MEMORY)) {
props.put("channel.type", "memory");
} else {
props.put("channel.type", "file");
if (!dataDir.endsWith(FILE_SEP)) {
dataDir = dataDir + FILE_SEP;
}
props.put("channel.checkpointDir", dataDir + "checkpoint");
props.put("channel.dataDirs", dataDir + "data");
}
} else {
props.put("channel.type", "file");
}
final StringBuilder sb = new StringBuilder();
String leading = Strings.EMPTY;
final int priority = agents.length;
for (int i = 0; i < priority; ++i) {
sb.append(leading).append("agent").append(i);
leading = " ";
final String prefix = "agent" + i;
props.put(prefix + ".type", "avro");
props.put(prefix + ".hostname", agents[i].getHost());
props.put(prefix + ".port", Integer.toString(agents[i].getPort()));
props.put(prefix + ".batch-size", Integer.toString(batchSize));
props.put("processor.priority." + prefix, Integer.toString(agents.length - i));
}
props.put("sinks", sb.toString());
props.put("processor.type", "failover");
} else {
String[] sinks = null;
for (final Property property : properties) {
final String key = property.getName();
if (Strings.isEmpty(key)) {
final String msg = "A property name must be provided";
LOGGER.error(msg);
throw new ConfigurationException(msg);
}
final String upperKey = key.toUpperCase(Locale.ENGLISH);
if (upperKey.startsWith(name.toUpperCase(Locale.ENGLISH))) {
final String msg =
"Specification of the agent name is not allowed in Flume Appender configuration: " + key;
LOGGER.error(msg);
throw new ConfigurationException(msg);
}
final String value = property.getValue();
if (Strings.isEmpty(value)) {
final String msg = "A value for property " + key + " must be provided";
LOGGER.error(msg);
throw new ConfigurationException(msg);
}
if (upperKey.equals("SINKS")) {
sinks = value.trim().split(" ");
}
props.put(key, value);
}
if (sinks == null || sinks.length == 0) {
final String msg = "At least one Sink must be specified";
LOGGER.error(msg);
throw new ConfigurationException(msg);
}
}
return props;
}
}
}