/*
* NOTE: This copyright does *not* cover user programs that use HQ
* program services by normal system calls through the application
* program interfaces provided as part of the Hyperic Plug-in Development
* Kit or the Hyperic Client Development Kit - this is merely considered
* normal use of the program, and does *not* fall under the heading of
* "derived work".
*
* Copyright (C) [2004-2012], VMWare, Inc.
* This file is part of HQ.
*
* HQ is free software; you can redistribute it and/or modify
* it under the terms version 2 of the GNU General Public License as
* published by the Free Software Foundation. This program is distributed
* in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
* even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*/
package org.hyperic.hq.plugin.multilogtrack;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hyperic.hq.product.LogFileTailPlugin;
import org.hyperic.hq.product.LogMessageFolder;
import org.hyperic.hq.product.LogTrackPlugin;
import org.hyperic.hq.product.PluginException;
import org.hyperic.hq.product.TrackEvent;
import org.hyperic.hq.product.TypeInfo;
import org.hyperic.sigar.FileInfo;
import org.hyperic.sigar.FileTail;
import org.hyperic.sigar.Sigar;
import org.hyperic.sigar.SigarException;
import org.hyperic.util.config.BooleanConfigOption;
import org.hyperic.util.config.ConfigOption;
import org.hyperic.util.config.ConfigResponse;
import org.hyperic.util.config.ConfigSchema;
public class MultiLogTrackPlugin extends LogFileTailPlugin {
private static final Log log = LogFactory.getLog(MultiLogTrackPlugin.class);
private Pattern includePattern;
private Pattern excludePattern;
private Pattern includePattern_2;
private Pattern excludePattern_2;
private AtomicReference<FileTail> watcher = new AtomicReference<FileTail>();
private Sigar sigar;
private final Long matchSleepTime;
protected static final String FILE_SCAN_INTERVAL = "300000";
private static final HashSet<String> firstTimeConfig = new HashSet<String>();
/** picks up new files as they appear */
private static AtomicReference<Collection<MultiDirWatcher>> fileWatcherThreads =
new AtomicReference<Collection<MultiDirWatcher>>();
static final String INCLUDE_PATTERN = "includepattern";
static final String EXCLUDE_PATTERN = "excludepattern";
static final String SECONDARY_PATTEN_SUFFIX = "_2";
static final String INCLUDE_PATTERN_2 = "includepattern" + SECONDARY_PATTEN_SUFFIX;
static final String EXCLUDE_PATTERN_2 = "excludepattern" + SECONDARY_PATTEN_SUFFIX;
public MultiLogTrackPlugin() {
this.matchSleepTime = getSleepTime();
}
public TrackEvent processLine(FileInfo info, String line, String basedir, String logfilepattern, int offset) {
final TrackEvent event = processLine(
INCLUDE_PATTERN, includePattern, includePattern, excludePattern, info, line, basedir, logfilepattern, offset);
final TrackEvent event2 = processLine(
INCLUDE_PATTERN_2, includePattern, includePattern_2, excludePattern_2, info, line, basedir, logfilepattern, offset);
// don't want to return a null event if one was evaluated. Need to make sure we return the correct one
if (event != null) {
return event;
} else if (event2 != null) {
return event2;
} else {
return null;
}
}
private TrackEvent processLine(String property, Pattern primaryIncludePattern, Pattern includePattern,
Pattern excludePattern, FileInfo info, String line, String basedir,
String logfilepattern, int offset) {
if (null == includePattern || !includePattern.matcher(line).find()) {
return null;
}
if (null != excludePattern && excludePattern.matcher(line).find()) {
return null;
}
MultiLogTrackMeasurementPlugin.incrementNumLines(property, basedir, logfilepattern,
primaryIncludePattern.toString(), offset);
return newTrackEvent(System.currentTimeMillis(), LogTrackPlugin.LOGLEVEL_ANY, info.getName(), line);
}
public void configure(ConfigResponse config) throws PluginException {
this.config = config;
setFileWatcherThread(config);
final Reference<Pattern> includePatternRef = new Reference<Pattern>();
configure(config, INCLUDE_PATTERN, includePatternRef);
includePattern = includePatternRef.get();
final Reference<Pattern> excludePatternRef = new Reference<Pattern>();
configure(config, EXCLUDE_PATTERN, excludePatternRef);
excludePattern = excludePatternRef.get();
final Reference<Pattern> includePatternRef_2 = new Reference<Pattern>();
configure(config, INCLUDE_PATTERN_2, includePatternRef_2);
includePattern_2 = includePatternRef_2.get();
final Reference<Pattern> excludePatternRef_2 = new Reference<Pattern>();
configure(config, EXCLUDE_PATTERN_2, excludePatternRef_2);
excludePattern_2 = excludePatternRef_2.get();
}
public void configure(ConfigResponse config, String configProp, Reference<Pattern> patternRef) throws PluginException {
String pattern = config.getValue(configProp, "");
pattern = pattern.trim().startsWith("*") ? "." + pattern.trim() : pattern.trim();
if (pattern != null && pattern.trim().length() != 0) {
try {
patternRef.set(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE));
} catch (Exception e) {
throw new PluginException("error compiling pattern " + configProp + "=" + pattern + ": " + e, e);
}
} else {
patternRef.set(null);
}
}
private void setFileWatcherThread(final ConfigResponse c) {
final String basedir = MultiLogTrackServerDetector.getBasedir(c);
final String logfilepattern = MultiLogTrackServerDetector.getLogfilepattern(c);
final Thread thread = new MultiDirWatcher(c);
Collection<MultiDirWatcher> threads = fileWatcherThreads.get();
if (threads == null) {
threads = new ArrayList<MultiDirWatcher>();
fileWatcherThreads.set(threads);
}
boolean startThread = true;
for (MultiDirWatcher t : threads) {
if (t.basedir.equals(basedir) && t.logfilepattern.equals(logfilepattern)) {
startThread = false;
break;
}
}
if (startThread) {
thread.setDaemon(true);
thread.start();
}
}
private class MultiDirWatcher extends Thread {
private final ConfigResponse cfg;
private String basedir;
private String logfilepattern;
private MultiDirWatcher(ConfigResponse c) {
this.cfg = c;
this.basedir = MultiLogTrackServerDetector.getBasedir(c);
this.logfilepattern = MultiLogTrackServerDetector.getLogfilepattern(c);
setName("MultiDirWatcher-" + basedir + "-" + logfilepattern);
}
public void run() {
if (log.isDebugEnabled()) {
log.debug("starting multi log track watcher thread for basedir=" + basedir +
", logfilepattern=" + logfilepattern);
}
while (true) {
try {
setFiles(cfg);
long sleepTime = Long.parseLong(FILE_SCAN_INTERVAL);
try {
sleepTime = Long.parseLong(cfg.getValue("fileScanInterval", FILE_SCAN_INTERVAL));
if (sleepTime < 60000) {
log.warn("file scan interval set to " + sleepTime +
", will not scan more than once a minute. Setting interval to one minute");
sleepTime = 60000;
}
} catch (NumberFormatException e) {
log.debug(e,e);
}
Thread.sleep(sleepTime);
} catch (Throwable t) {
log.error(t,t);
}
}
}
}
private void setFiles(ConfigResponse cfg) throws PluginException {
try {
final Set<String> oldFiles = new HashSet<String>();
final String basedir = MultiLogTrackServerDetector.getBasedir(cfg);
final String includePatternBuf = cfg.getValue(INCLUDE_PATTERN);
final String logfilepattern = MultiLogTrackServerDetector.getLogfilepattern(cfg);
final String key = logfilepattern + "|" + basedir + "|" + includePatternBuf;
if (firstTimeConfig.contains(key)) {
MultiLogTrackServerDetector.getBasedirAndSetFilesFromCache(cfg, oldFiles, false);
} else {
firstTimeConfig.add(key);
}
final List<String> files =
MultiLogTrackServerDetector.getFilesCached(logfilepattern, basedir, includePatternBuf, true);
for (final String logfile : files) {
boolean exists = oldFiles.remove(logfile);
if (!exists) {
final String fs = MultiLogTrackServerDetector.getFileSeparator(basedir);
configureLogfile(basedir + fs + logfile, cfg);
}
}
final FileTail fileWatcher = getFileWatcher(cfg);
for (final String nonExistentLogFile : oldFiles) {
fileWatcher.remove(nonExistentLogFile);
}
} catch (SigarException e) {
throw new PluginException(e.getMessage(), e);
}
}
private void configureLogfile(String logfile, ConfigResponse cfg) throws SigarException {
FileTail fileWatcher = getFileWatcher(cfg);
if (!new File(logfile).exists()) {
log.warn("logfile=" + logfile + " does not exist");
if (log.isDebugEnabled()) log.debug("logfile=" + logfile + " does not exist", new Throwable());
return;
}
fileWatcher.add(logfile);
if (log.isDebugEnabled()) {
log.debug("Adding Multi Log Track file watchers for file=" + logfile);
}
try {
Field field = LogTrackPlugin.class.getDeclaredField("folder");
field.setAccessible(true);
final LogMessageFolder folder = new LogMessageFolder(this);
folder.setRepeatMax(Long.MAX_VALUE);
field.set(this, folder);
} catch (Exception e) {
log.error(e,e);
}
}
private FileTail getFileWatcher(final ConfigResponse cfg) {
if (this.watcher.get() == null) {
if (sigar == null) {
sigar = new Sigar();
}
this.watcher.set(new MultiFileTail(sigar, cfg));
getManager().addFileWatcher(this.watcher.get());
if (log.isDebugEnabled()) {
log.debug("init file tail basedir=" + MultiLogTrackServerDetector.getBasedir(cfg) +
", pattern=" + MultiLogTrackServerDetector.getLogfilepattern(cfg) +
", watcher=" + watcher +
", this=" + this +
", cfg=" + cfg +
", sigar=" + sigar);
}
}
return this.watcher.get();
}
private class MultiFileTail extends FileTail {
private final ConfigResponse cfg;
private MultiFileTail(Sigar sigar, ConfigResponse cfg) {
super(sigar);
this.cfg = cfg;
}
public void tail(FileInfo info, Reader reader) {
FileInfo previous = info.getPreviousInfo();
if (info.getInode() == 0 || previous.getInode() == 0) {
long prevSize = previous.getSize();
if (log.isDebugEnabled()) {
final String msg = "Inode=0, will attempt to seek to previous Inode size=" + prevSize;
log.debug((info.getInode() != 0 ? "previous " : "") + msg);
}
if (info.getSize() == prevSize) {
log.debug("log watcher event occured but there are no lines to read since info.getSize() == previous.getSize()");
return;
}
try {
reader.skip(prevSize);
} catch (IOException e) {
log.error("Could not skip to the previous offset on Inode=0: " + e, e);
return;
}
}
BufferedReader buffer = new BufferedReader(reader);
String basedir = MultiLogTrackServerDetector.getBasedir(cfg);
String logfilepattern = MultiLogTrackServerDetector.getLogfilepattern(cfg);
try {
String line;
final boolean debug = log.isDebugEnabled();
boolean first = true;
int i=0;
while ((line = buffer.readLine()) != null) {
if (!first && matchSleepTime != null && matchSleepTime > 0) {
sleep(matchSleepTime);
}
first = false;
if (debug) {
log.debug("processing file=" + info.getName() + ", line=" + line +
", basedir=" + MultiLogTrackServerDetector.getBasedir(cfg) +
", pattern=" + MultiLogTrackServerDetector.getLogfilepattern(cfg) +
", includepattern=" + includePattern +
", this=" + toString());
}
TrackEvent event = processLine(info, line, basedir, logfilepattern, i++);
if (event != null && reportEvents()) {
getManager().reportEvent(event);
}
}
} catch (IOException e) {
log.error(info.getName() + ": " + e, e);
}
}
private void sleep(Long sleepTime) {
try {
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
log.debug(e,e);
}
}
private boolean reportEvents() {
String value = cfg.getValue(MultiLogTrackServerDetector.ENABLE_ONLY_METRICS);
if (value == null || value.equalsIgnoreCase("false")) {
return true;
}
return false;
}
public int hashCode() {
return cfg.hashCode();
}
public boolean equals(Object o) {
if (this == o) {
return true;
} else if (o instanceof MultiFileTail) {
MultiFileTail t = (MultiFileTail) o;
return cfg.equals(t.cfg);
}
return false;
}
}
private Long getSleepTime() {
// TODO should handle this via the AgentConfig, but for now doing it via System.getProperty()
String sleepTimeBuf = System.getProperty("multilogtrack.sleep");
if (sleepTimeBuf == null) {
return null;
}
try {
return Long.parseLong(sleepTimeBuf);
} catch (NumberFormatException e) {
log.warn("multilogtrack.sleep not a number: " + sleepTimeBuf + ", plugin will not sleep btwn log matches");
}
return null;
}
public ConfigSchema getConfigSchema(TypeInfo info, ConfigResponse config) {
ConfigSchema schema = new ConfigSchema();
ConfigOption enableOption = getEnableOption(info, config);
if (enableOption != null) {
schema.addOption(enableOption);
enableOption.setDefault("true");
enableOption.setOptional(false);
}
final String enableProp = MultiLogTrackServerDetector.ENABLE_LOG_SERVICES_PROP;
ConfigOption option = new BooleanConfigOption(enableProp, "Enable Service Creation", false);
option.setOptional(true);
schema.addOption(option);
final String metricsProp = MultiLogTrackServerDetector.ENABLE_ONLY_METRICS;
option = new BooleanConfigOption(metricsProp, "Only Enable Log Metrics, Do Not Send Logs To The HQ Server", false);
option.setOptional(true);
schema.addOption(option);
final String overrideChecks = MultiLogTrackServerDetector.OVERRIDE_CHECKS;
option = new BooleanConfigOption(overrideChecks, "Override file checks. Validate if files do not exist", false);
option.setOptional(true);
schema.addOption(option);
return schema;
}
public class Reference<T> {
private T value;
public Reference() {}
public Reference(T initialValue) {
this.value = initialValue;
}
public T get() {
return value;
}
public void set(T value) {
this.value = value;
}
public int hashCode() {
return value.hashCode();
}
public boolean equals(Object rhs) {
if (this == rhs) {
return true;
}
return rhs.equals(value);
}
public String toString() {
return value.toString();
}
}
}