package com.linkedin.thirdeye.dashboard.resources; import com.linkedin.thirdeye.anomaly.detection.AnomalyDetectionInputContext; import com.linkedin.thirdeye.anomaly.detection.AnomalyDetectionInputContextBuilder; import com.linkedin.thirdeye.api.DimensionMap; import com.linkedin.thirdeye.api.MetricTimeSeries; import com.linkedin.thirdeye.client.DAORegistry; import com.linkedin.thirdeye.constant.MetricAggFunction; import com.linkedin.thirdeye.datalayer.dto.AnomalyFunctionDTO; import com.linkedin.thirdeye.datalayer.dto.MergedAnomalyResultDTO; import com.linkedin.thirdeye.datalayer.dto.RawAnomalyResultDTO; import com.linkedin.thirdeye.detector.function.AnomalyFunction; import com.linkedin.thirdeye.detector.function.AnomalyFunctionFactory; import com.linkedin.thirdeye.detector.function.BaseAnomalyFunction; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import javax.ws.rs.Consumes; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.commons.io.IOUtils; import org.joda.time.DateTime; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Path("thirdeye/function") @Produces(MediaType.APPLICATION_JSON) public class AnomalyFunctionResource { private static final Logger LOG = LoggerFactory.getLogger(AnomalyFunctionResource.class); private static final DAORegistry DAO_REGISTRY = DAORegistry.getInstance(); private final Map<String, Object> anomalyFunctionMetadata = new HashMap<>(); private final AnomalyFunctionFactory anomalyFunctionFactory; public AnomalyFunctionResource(String functionConfigPath) { buildFunctionMetadata(functionConfigPath); this.anomalyFunctionFactory = new AnomalyFunctionFactory(functionConfigPath); } private void buildFunctionMetadata(String functionConfigPath) { Properties props = new Properties(); InputStream input = null; try { input = new FileInputStream(functionConfigPath); props.load(input); } catch (IOException e) { LOG.error("Function config not found at {}", functionConfigPath); } finally { IOUtils.closeQuietly(input); } LOG.info("Loaded functions : " + props.keySet() + " from path : " + functionConfigPath); for (Object key : props.keySet()) { String functionName = key.toString(); try { Class<AnomalyFunction> clz = (Class<AnomalyFunction>) Class.forName(props.get(functionName).toString()); Method getFunctionProps = clz.getMethod("getPropertyKeys"); AnomalyFunction anomalyFunction = clz.newInstance(); anomalyFunctionMetadata.put(functionName, getFunctionProps.invoke(anomalyFunction)); } catch (ClassNotFoundException e) { LOG.warn("Unknown class for function : " + functionName); } catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) { LOG.error("Unknown method", e); } catch (InstantiationException e) { LOG.error("Unsupported anomaly function", e); } } } /** * @return map of function name vs function property keys * <p/> * eg. { "WEEK_OVER_WEEK_RULE":["baseline","changeThreshold","averageVolumeThreshold"], * "MIN_MAX_THRESHOLD":["min","max"] } */ @GET @Path("metadata") public Map<String, Object> getAnomalyFunctionMetadata() { return anomalyFunctionMetadata; } /** * @return List of metric functions * <p/> * eg. ["SUM","AVG","COUNT"] */ @GET @Path("metric-function") public MetricAggFunction[] getMetricFunctions() { return MetricAggFunction.values(); } @POST @Path("/analyze") @Consumes(MediaType.APPLICATION_JSON) public Response analyze(AnomalyFunctionDTO anomalyFunctionSpec, @QueryParam("startTime") Long startTime, @QueryParam("endTime") Long endTime) throws Exception { // TODO: replace this with Job/Task framework and job tracker page BaseAnomalyFunction anomalyFunction = anomalyFunctionFactory.fromSpec(anomalyFunctionSpec); AnomalyDetectionInputContextBuilder anomalyDetectionInputContextBuilder = new AnomalyDetectionInputContextBuilder(anomalyFunctionFactory); DateTime windowStart = new DateTime(startTime); DateTime windowEnd = new DateTime(endTime); anomalyDetectionInputContextBuilder.init(anomalyFunctionSpec) .fetchTimeSeriesData(windowStart, windowEnd) .fetchSaclingFactors(windowStart, windowEnd); AnomalyDetectionInputContext anomalyDetectionInputContext = anomalyDetectionInputContextBuilder.build(); Map<DimensionMap, MetricTimeSeries> dimensionKeyMetricTimeSeriesMap = anomalyDetectionInputContext.getDimensionKeyMetricTimeSeriesMap(); List<RawAnomalyResultDTO> anomalyResults = new ArrayList<>(); List<RawAnomalyResultDTO> results = new ArrayList<>(); for (Map.Entry<DimensionMap, MetricTimeSeries> entry : dimensionKeyMetricTimeSeriesMap.entrySet()) { DimensionMap dimensionMap = entry.getKey(); if (entry.getValue().getTimeWindowSet().size() < 2) { LOG.warn("Insufficient data for {} to run anomaly detection function", dimensionMap); continue; } try { // Run algorithm MetricTimeSeries metricTimeSeries = entry.getValue(); LOG.info("Analyzing anomaly function with dimensionKey: {}, windowStart: {}, windowEnd: {}", dimensionMap, startTime, endTime); List<RawAnomalyResultDTO> resultsOfAnEntry = anomalyFunction .analyze(dimensionMap, metricTimeSeries, new DateTime(startTime), new DateTime(endTime), new ArrayList<MergedAnomalyResultDTO>()); if (resultsOfAnEntry.size() != 0) { results.addAll(resultsOfAnEntry); } LOG.info("{} has {} anomalies in window {} to {}", dimensionMap, resultsOfAnEntry.size(), new DateTime(startTime), new DateTime(endTime)); } catch (Exception e) { LOG.error("Could not compute for {}", dimensionMap, e); } } if (results.size() > 0) { List<RawAnomalyResultDTO> validResults = new ArrayList<>(); for (RawAnomalyResultDTO anomaly : results) { if (!anomaly.isDataMissing()) { LOG.info("Found anomaly, sev [{}] start [{}] end [{}]", anomaly.getWeight(), new DateTime(anomaly.getStartTime()), new DateTime(anomaly.getEndTime())); validResults.add(anomaly); } } anomalyResults.addAll(validResults); } return Response.ok(anomalyResults).build(); } }