/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.kafka.common.metrics;
import org.apache.kafka.common.MetricName;
import org.apache.kafka.common.utils.Time;
import org.apache.kafka.common.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
/**
* A registry of sensors and metrics.
* <p>
* A metric is a named, numerical measurement. A sensor is a handle to record numerical measurements as they occur. Each
* Sensor has zero or more associated metrics. For example a Sensor might represent message sizes and we might associate
* with this sensor a metric for the average, maximum, or other statistics computed off the sequence of message sizes
* that are recorded by the sensor.
* <p>
* Usage looks something like this:
*
* <pre>
* // set up metrics:
* Metrics metrics = new Metrics(); // this is the global repository of metrics and sensors
* Sensor sensor = metrics.sensor("message-sizes");
* MetricName metricName = new MetricName("message-size-avg", "producer-metrics");
* sensor.add(metricName, new Avg());
* metricName = new MetricName("message-size-max", "producer-metrics");
* sensor.add(metricName, new Max());
*
* // as messages are sent we record the sizes
* sensor.record(messageSize);
* </pre>
*/
public class Metrics implements Closeable {
private final MetricConfig config;
private final ConcurrentMap<MetricName, KafkaMetric> metrics;
private final ConcurrentMap<String, Sensor> sensors;
private final ConcurrentMap<Sensor, List<Sensor>> childrenSensors;
private final List<MetricsReporter> reporters;
private final Time time;
private final ScheduledThreadPoolExecutor metricsScheduler;
private static final Logger log = LoggerFactory.getLogger(Metrics.class);
/**
* Create a metrics repository with no metric reporters and default configuration.
* Expiration of Sensors is disabled.
*/
public Metrics() {
this(new MetricConfig());
}
/**
* Create a metrics repository with no metric reporters and default configuration.
* Expiration of Sensors is disabled.
*/
public Metrics(Time time) {
this(new MetricConfig(), new ArrayList<MetricsReporter>(0), time);
}
/**
* Create a metrics repository with no metric reporters and the given default configuration.
* Expiration of Sensors is disabled.
*/
public Metrics(MetricConfig defaultConfig, Time time) {
this(defaultConfig, new ArrayList<MetricsReporter>(0), time);
}
/**
* Create a metrics repository with no reporters and the given default config. This config will be used for any
* metric that doesn't override its own config. Expiration of Sensors is disabled.
* @param defaultConfig The default config to use for all metrics that don't override their config
*/
public Metrics(MetricConfig defaultConfig) {
this(defaultConfig, new ArrayList<MetricsReporter>(0), Time.SYSTEM);
}
/**
* Create a metrics repository with a default config and the given metric reporters.
* Expiration of Sensors is disabled.
* @param defaultConfig The default config
* @param reporters The metrics reporters
* @param time The time instance to use with the metrics
*/
public Metrics(MetricConfig defaultConfig, List<MetricsReporter> reporters, Time time) {
this(defaultConfig, reporters, time, false);
}
/**
* Create a metrics repository with a default config, given metric reporters and the ability to expire eligible sensors
* @param defaultConfig The default config
* @param reporters The metrics reporters
* @param time The time instance to use with the metrics
* @param enableExpiration true if the metrics instance can garbage collect inactive sensors, false otherwise
*/
public Metrics(MetricConfig defaultConfig, List<MetricsReporter> reporters, Time time, boolean enableExpiration) {
this.config = defaultConfig;
this.sensors = new ConcurrentHashMap<>();
this.metrics = new ConcurrentHashMap<>();
this.childrenSensors = new ConcurrentHashMap<>();
this.reporters = Utils.notNull(reporters);
this.time = time;
for (MetricsReporter reporter : reporters)
reporter.init(new ArrayList<KafkaMetric>());
// Create the ThreadPoolExecutor only if expiration of Sensors is enabled.
if (enableExpiration) {
this.metricsScheduler = new ScheduledThreadPoolExecutor(1);
// Creating a daemon thread to not block shutdown
this.metricsScheduler.setThreadFactory(new ThreadFactory() {
public Thread newThread(Runnable runnable) {
return Utils.newThread("SensorExpiryThread", runnable, true);
}
});
this.metricsScheduler.scheduleAtFixedRate(new ExpireSensorTask(), 30, 30, TimeUnit.SECONDS);
} else {
this.metricsScheduler = null;
}
addMetric(metricName("count", "kafka-metrics-count", "total number of registered metrics"),
new Measurable() {
@Override
public double measure(MetricConfig config, long now) {
return metrics.size();
}
});
}
/**
* Create a MetricName with the given name, group, description and tags, plus default tags specified in the metric
* configuration. Tag in tags takes precedence if the same tag key is specified in the default metric configuration.
*
* @param name The name of the metric
* @param group logical group name of the metrics to which this metric belongs
* @param description A human-readable description to include in the metric
* @param tags additional key/value attributes of the metric
*/
public MetricName metricName(String name, String group, String description, Map<String, String> tags) {
Map<String, String> combinedTag = new LinkedHashMap<>(config.tags());
combinedTag.putAll(tags);
return new MetricName(name, group, description, combinedTag);
}
/**
* Create a MetricName with the given name, group, description, and default tags
* specified in the metric configuration.
*
* @param name The name of the metric
* @param group logical group name of the metrics to which this metric belongs
* @param description A human-readable description to include in the metric
*/
public MetricName metricName(String name, String group, String description) {
return metricName(name, group, description, new HashMap<String, String>());
}
/**
* Create a MetricName with the given name, group and default tags specified in the metric configuration.
*
* @param name The name of the metric
* @param group logical group name of the metrics to which this metric belongs
*/
public MetricName metricName(String name, String group) {
return metricName(name, group, "", new HashMap<String, String>());
}
/**
* Create a MetricName with the given name, group, description, and keyValue as tags, plus default tags specified in the metric
* configuration. Tag in keyValue takes precedence if the same tag key is specified in the default metric configuration.
*
* @param name The name of the metric
* @param group logical group name of the metrics to which this metric belongs
* @param description A human-readable description to include in the metric
* @param keyValue additional key/value attributes of the metric (must come in pairs)
*/
public MetricName metricName(String name, String group, String description, String... keyValue) {
return metricName(name, group, description, getTags(keyValue));
}
/**
* Create a MetricName with the given name, group and tags, plus default tags specified in the metric
* configuration. Tag in tags takes precedence if the same tag key is specified in the default metric configuration.
*
* @param name The name of the metric
* @param group logical group name of the metrics to which this metric belongs
* @param tags key/value attributes of the metric
*/
public MetricName metricName(String name, String group, Map<String, String> tags) {
return metricName(name, group, "", tags);
}
private static Map<String, String> getTags(String... keyValue) {
if ((keyValue.length % 2) != 0)
throw new IllegalArgumentException("keyValue needs to be specified in pairs");
Map<String, String> tags = new HashMap<String, String>();
for (int i = 0; i < keyValue.length; i += 2)
tags.put(keyValue[i], keyValue[i + 1]);
return tags;
}
public MetricConfig config() {
return config;
}
/**
* Get the sensor with the given name if it exists
* @param name The name of the sensor
* @return Return the sensor or null if no such sensor exists
*/
public Sensor getSensor(String name) {
return this.sensors.get(Utils.notNull(name));
}
/**
* Get or create a sensor with the given unique name and no parent sensors. This uses
* a default recording level of INFO.
* @param name The sensor name
* @return The sensor
*/
public Sensor sensor(String name) {
return this.sensor(name, Sensor.RecordingLevel.INFO);
}
/**
* Get or create a sensor with the given unique name and no parent sensors and with a given
* recording level.
* @param name The sensor name.
* @param recordingLevel The recording level.
* @return The sensor
*/
public Sensor sensor(String name, Sensor.RecordingLevel recordingLevel) {
return sensor(name, null, recordingLevel, (Sensor[]) null);
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor. This uses a default recording level of INFO.
* @param name The name of the sensor
* @param parents The parent sensors
* @return The sensor that is created
*/
public Sensor sensor(String name, Sensor... parents) {
return this.sensor(name, Sensor.RecordingLevel.INFO, parents);
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor.
* @param name The name of the sensor.
* @param parents The parent sensors.
* @param recordingLevel The recording level.
* @return The sensor that is created
*/
public Sensor sensor(String name, Sensor.RecordingLevel recordingLevel, Sensor... parents) {
return sensor(name, null, recordingLevel, parents);
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor. This uses a default recording level of INFO.
* @param name The name of the sensor
* @param config A default configuration to use for this sensor for metrics that don't have their own config
* @param parents The parent sensors
* @return The sensor that is created
*/
public synchronized Sensor sensor(String name, MetricConfig config, Sensor... parents) {
return this.sensor(name, config, Sensor.RecordingLevel.INFO, parents);
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor.
* @param name The name of the sensor
* @param config A default configuration to use for this sensor for metrics that don't have their own config
* @param recordingLevel The recording level.
* @param parents The parent sensors
* @return The sensor that is created
*/
public synchronized Sensor sensor(String name, MetricConfig config, Sensor.RecordingLevel recordingLevel, Sensor... parents) {
return sensor(name, config, Long.MAX_VALUE, recordingLevel, parents);
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor.
* @param name The name of the sensor
* @param config A default configuration to use for this sensor for metrics that don't have their own config
* @param inactiveSensorExpirationTimeSeconds If no value if recorded on the Sensor for this duration of time,
* it is eligible for removal
* @param parents The parent sensors
* @param recordingLevel The recording level.
* @return The sensor that is created
*/
public synchronized Sensor sensor(String name, MetricConfig config, long inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel recordingLevel, Sensor... parents) {
Sensor s = getSensor(name);
if (s == null) {
s = new Sensor(this, name, parents, config == null ? this.config : config, time, inactiveSensorExpirationTimeSeconds, recordingLevel);
this.sensors.put(name, s);
if (parents != null) {
for (Sensor parent : parents) {
List<Sensor> children = childrenSensors.get(parent);
if (children == null) {
children = new ArrayList<>();
childrenSensors.put(parent, children);
}
children.add(s);
}
}
log.debug("Added sensor with name {}", name);
}
return s;
}
/**
* Get or create a sensor with the given unique name and zero or more parent sensors. All parent sensors will
* receive every value recorded with this sensor. This uses a default recording level of INFO.
* @param name The name of the sensor
* @param config A default configuration to use for this sensor for metrics that don't have their own config
* @param inactiveSensorExpirationTimeSeconds If no value if recorded on the Sensor for this duration of time,
* it is eligible for removal
* @param parents The parent sensors
* @return The sensor that is created
*/
public synchronized Sensor sensor(String name, MetricConfig config, long inactiveSensorExpirationTimeSeconds, Sensor... parents) {
return this.sensor(name, config, inactiveSensorExpirationTimeSeconds, Sensor.RecordingLevel.INFO, parents);
}
/**
* Remove a sensor (if it exists), associated metrics and its children.
*
* @param name The name of the sensor to be removed
*/
public void removeSensor(String name) {
Sensor sensor = sensors.get(name);
if (sensor != null) {
List<Sensor> childSensors = null;
synchronized (sensor) {
synchronized (this) {
if (sensors.remove(name, sensor)) {
for (KafkaMetric metric : sensor.metrics())
removeMetric(metric.metricName());
log.debug("Removed sensor with name {}", name);
childSensors = childrenSensors.remove(sensor);
}
}
}
if (childSensors != null) {
for (Sensor childSensor : childSensors)
removeSensor(childSensor.name());
}
}
}
/**
* Add a metric to monitor an object that implements measurable. This metric won't be associated with any sensor.
* This is a way to expose existing values as metrics.
* @param metricName The name of the metric
* @param measurable The measurable that will be measured by this metric
*/
public void addMetric(MetricName metricName, Measurable measurable) {
addMetric(metricName, null, measurable);
}
/**
* Add a metric to monitor an object that implements measurable. This metric won't be associated with any sensor.
* This is a way to expose existing values as metrics.
* @param metricName The name of the metric
* @param config The configuration to use when measuring this measurable
* @param measurable The measurable that will be measured by this metric
*/
public synchronized void addMetric(MetricName metricName, MetricConfig config, Measurable measurable) {
KafkaMetric m = new KafkaMetric(new Object(),
Utils.notNull(metricName),
Utils.notNull(measurable),
config == null ? this.config : config,
time);
registerMetric(m);
}
/**
* Remove a metric if it exists and return it. Return null otherwise. If a metric is removed, `metricRemoval`
* will be invoked for each reporter.
*
* @param metricName The name of the metric
* @return the removed `KafkaMetric` or null if no such metric exists
*/
public synchronized KafkaMetric removeMetric(MetricName metricName) {
KafkaMetric metric = this.metrics.remove(metricName);
if (metric != null) {
for (MetricsReporter reporter : reporters)
reporter.metricRemoval(metric);
}
return metric;
}
/**
* Add a MetricReporter
*/
public synchronized void addReporter(MetricsReporter reporter) {
Utils.notNull(reporter).init(new ArrayList<>(metrics.values()));
this.reporters.add(reporter);
}
synchronized void registerMetric(KafkaMetric metric) {
MetricName metricName = metric.metricName();
if (this.metrics.containsKey(metricName))
throw new IllegalArgumentException("A metric named '" + metricName + "' already exists, can't register another one.");
this.metrics.put(metricName, metric);
for (MetricsReporter reporter : reporters)
reporter.metricChange(metric);
}
/**
* Get all the metrics currently maintained indexed by metricName
*/
public Map<MetricName, KafkaMetric> metrics() {
return this.metrics;
}
public List<MetricsReporter> reporters() {
return this.reporters;
}
public KafkaMetric metric(MetricName metricName) {
return this.metrics.get(metricName);
}
/**
* This iterates over every Sensor and triggers a removeSensor if it has expired
* Package private for testing
*/
class ExpireSensorTask implements Runnable {
public void run() {
for (Map.Entry<String, Sensor> sensorEntry : sensors.entrySet()) {
// removeSensor also locks the sensor object. This is fine because synchronized is reentrant
// There is however a minor race condition here. Assume we have a parent sensor P and child sensor C.
// Calling record on C would cause a record on P as well.
// So expiration time for P == expiration time for C. If the record on P happens via C just after P is removed,
// that will cause C to also get removed.
// Since the expiration time is typically high it is not expected to be a significant concern
// and thus not necessary to optimize
synchronized (sensorEntry.getValue()) {
if (sensorEntry.getValue().hasExpired()) {
log.debug("Removing expired sensor {}", sensorEntry.getKey());
removeSensor(sensorEntry.getKey());
}
}
}
}
}
/* For testing use only. */
Map<Sensor, List<Sensor>> childrenSensors() {
return Collections.unmodifiableMap(childrenSensors);
}
/**
* Close this metrics repository.
*/
@Override
public void close() {
if (this.metricsScheduler != null) {
this.metricsScheduler.shutdown();
try {
this.metricsScheduler.awaitTermination(30, TimeUnit.SECONDS);
} catch (InterruptedException ex) {
// ignore and continue shutdown
Thread.currentThread().interrupt();
}
}
for (MetricsReporter reporter : this.reporters)
reporter.close();
}
}