/*
* Copyright 2013-2015 Rackspace
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.rackspacecloud.blueflood.inputs.handlers;
import com.codahale.metrics.Counter;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ListenableFuture;
import com.rackspacecloud.blueflood.cache.MetadataCache;
import com.rackspacecloud.blueflood.concurrent.ThreadPoolBuilder;
import com.rackspacecloud.blueflood.http.DefaultHandler;
import com.rackspacecloud.blueflood.http.QueryStringDecoderAndRouter;
import com.rackspacecloud.blueflood.http.RouteMatcher;
import com.rackspacecloud.blueflood.inputs.processors.*;
import com.rackspacecloud.blueflood.io.EventsIO;
import com.rackspacecloud.blueflood.service.*;
import com.rackspacecloud.blueflood.tracker.Tracker;
import com.rackspacecloud.blueflood.types.IMetric;
import com.rackspacecloud.blueflood.types.MetricsCollection;
import com.rackspacecloud.blueflood.utils.Metrics;
import com.rackspacecloud.blueflood.utils.ModuleLoader;
import com.rackspacecloud.blueflood.utils.TimeValue;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.*;
import io.netty.channel.group.ChannelGroup;
import io.netty.channel.group.DefaultChannelGroup;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.http.*;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import io.netty.handler.timeout.IdleStateHandler;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.TimeUnit;
public class HttpMetricsIngestionServer {
private static final Logger log = LoggerFactory.getLogger(HttpMetricsIngestionServer.class);
private static TimeValue DEFAULT_TIMEOUT = new TimeValue(5, TimeUnit.SECONDS);
private int httpIngestPort;
private String httpIngestHost;
private Processor processor;
private HttpEventsIngestionHandler httpEventsIngestionHandler;
private final int httpMaxContentLength;
private TimeValue timeout;
private EventLoopGroup acceptorGroup;
private EventLoopGroup workerGroup;
private ChannelGroup allOpenChannels = new DefaultChannelGroup("allOpenChannels", GlobalEventExecutor.INSTANCE);
private int HTTP_CONNECTION_READ_IDLE_TIME_SECONDS =
Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_CONNECTION_READ_IDLE_TIME_SECONDS);
public static boolean EXP_TOKEN_SEARCH_IMPROVEMENTS =
Configuration.getInstance().getBooleanProperty(CoreConfig.ENABLE_TOKEN_SEARCH_IMPROVEMENTS);
private static final boolean ENABLE_PER_TENANT_METRICS =
Configuration.getInstance().getBooleanProperty(CoreConfig.ENABLE_PER_TENANT_METRICS);
/**
* This is HTTP Header name is inserted by RouterMatcher based on the
* router configuration in startServer()
*/
public static final String TENANT_ID_HEADER = "tenantId";
/**
* Constructor. Instantiate Metrics Ingest server
* @param context
*/
public HttpMetricsIngestionServer(ScheduleContext context) {
this.httpIngestPort = Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_INGESTION_PORT);
this.httpIngestHost = Configuration.getInstance().getStringProperty(HttpConfig.HTTP_INGESTION_HOST);
this.timeout = DEFAULT_TIMEOUT; //TODO: make configurable
this.processor = new Processor(context, timeout);
this.httpMaxContentLength = Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_MAX_CONTENT_LENGTH);
int acceptThreads = Configuration.getInstance().getIntegerProperty(HttpConfig.MAX_WRITE_ACCEPT_THREADS);
int workerThreads = Configuration.getInstance().getIntegerProperty(HttpConfig.MAX_WRITE_WORKER_THREADS);
acceptorGroup = new NioEventLoopGroup(acceptThreads); // acceptor threads
workerGroup = new NioEventLoopGroup(workerThreads); // client connections threads
}
/**
* Starts the Ingest server
*
* @throws InterruptedException
*/
public void startServer() throws InterruptedException {
RouteMatcher router = new RouteMatcher();
router.get("/v1.0", new DefaultHandler());
router.post("/v1.0/multitenant/experimental/metrics",
new HttpMultitenantMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v1.0/:tenantId/experimental/metrics",
new HttpMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v1.0/:tenantId/experimental/metrics/statsd",
new HttpAggregatedIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.get("/v2.0", new DefaultHandler());
router.post("/v2.0/:tenantId/ingest/multi",
new HttpMultitenantMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v2.0/:tenantId/ingest",
new HttpMetricsIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v2.0/:tenantId/ingest/aggregated",
new HttpAggregatedIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v2.0/:tenantId/ingest/aggregated/multi",
new HttpAggregatedMultiIngestionHandler(processor, timeout, ENABLE_PER_TENANT_METRICS));
router.post("/v2.0/:tenantId/events", getHttpEventsIngestionHandler());
final RouteMatcher finalRouter = router;
log.info("Starting metrics listener HTTP server on port {}", httpIngestPort);
ServerBootstrap server = new ServerBootstrap();
server.group(acceptorGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel channel) throws Exception {
setupPipeline(channel, finalRouter);
}
});
Channel channel = server.bind(new InetSocketAddress(httpIngestHost, httpIngestPort)).sync().channel();
allOpenChannels.add(channel);
//register the tracker MBean for JMX/jolokia
log.info("Registering tracker service");
Tracker.getInstance().register();
log.info("Token search improvements enabled: " + EXP_TOKEN_SEARCH_IMPROVEMENTS);
}
private void setupPipeline(SocketChannel channel, RouteMatcher router) {
final ChannelPipeline pipeline = channel.pipeline();
pipeline.addLast("logging", new LoggingHandler(LogLevel.TRACE)); //duplex handler
pipeline.addLast("idleStateHandler", new IdleStateHandler(HTTP_CONNECTION_READ_IDLE_TIME_SECONDS, 0, 0)); //duplex handler
pipeline.addLast("eventHandler", new UserDefinedEventHandler()); //duplex handler
pipeline.addLast("encoder", new HttpResponseEncoder()); //outbound handler
pipeline.addLast("decoder", new HttpRequestDecoder() { //inbound handler
// if something bad happens during the decode, assume the client send bad data. return a 400.
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable thr) throws Exception {
log.debug("Exception during decoding", thr);
try {
if (ctx.channel().isWritable()) {
log.debug("request decoder error " + thr.getCause() + " on channel " + ctx.channel().toString());
ctx.channel().write(
new DefaultFullHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.BAD_REQUEST))
.addListener(ChannelFutureListener.CLOSE);
} else {
log.debug("channel " + ctx.channel().toString() + " is no longer writeable, not sending 400 response back to client");
}
} catch (Exception ex) {
// If we are getting exception trying to write,
// don't propagate to caller. It may cause this
// method to be called again and will produce
// stack overflow. So just log it here.
log.debug("Can't write to channel " + ctx.channel().toString(), ex);
}
}
});
pipeline.addLast("inflater", new HttpContentDecompressor()); //inbound handler
pipeline.addLast("chunkaggregator", new HttpObjectAggregator(httpMaxContentLength)); //inbound handler
pipeline.addLast("respdecoder", new HttpResponseDecoder()); //inbound handler
pipeline.addLast("handler", new QueryStringDecoderAndRouter(router)); //inbound handler
}
private HttpEventsIngestionHandler getHttpEventsIngestionHandler() {
if (this.httpEventsIngestionHandler == null) {
this.httpEventsIngestionHandler = new HttpEventsIngestionHandler((EventsIO) ModuleLoader.getInstance(EventsIO.class, CoreConfig.EVENTS_MODULES));
}
return this.httpEventsIngestionHandler;
}
@VisibleForTesting
public void setHttpEventsIngestionHandler(HttpEventsIngestionHandler httpEventsIngestionHandler) {
this.httpEventsIngestionHandler = httpEventsIngestionHandler;
}
static class Processor {
private static int BATCH_SIZE = Configuration.getInstance().getIntegerProperty(CoreConfig.METRIC_BATCH_SIZE);
private static int WRITE_THREADS =
Configuration.getInstance().getIntegerProperty(CoreConfig.METRICS_BATCH_WRITER_THREADS); // metrics will be batched into this many partitions.
private final TypeAndUnitProcessor typeAndUnitProcessor;
private final RollupTypeCacher rollupTypeCacher;
private final DiscoveryWriter discoveryWriter;
private final TokenDiscoveryWriter tokenDiscoveryWriter;
private final BatchWriter batchWriter;
private IncomingMetricMetadataAnalyzer metricMetadataAnalyzer =
new IncomingMetricMetadataAnalyzer(MetadataCache.getInstance());
private int HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS =
Configuration.getInstance().getIntegerProperty(HttpConfig.HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS);
private final Counter bufferedMetrics = Metrics.counter(HttpMetricsIngestionHandler.class, "Buffered Metrics");
private final TimeValue timeout;
Processor(ScheduleContext context, TimeValue timeout) {
this.timeout = timeout;
typeAndUnitProcessor = new TypeAndUnitProcessor(
new ThreadPoolBuilder()
.withName("Metric type and unit processing")
.withCorePoolSize(HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS)
.withMaxPoolSize(HTTP_MAX_TYPE_UNIT_PROCESSOR_THREADS)
.build(),
metricMetadataAnalyzer);
typeAndUnitProcessor.withLogger(log);
batchWriter = new BatchWriter(
new ThreadPoolBuilder()
.withName("Metric Batch Writing")
.withCorePoolSize(WRITE_THREADS)
.withMaxPoolSize(WRITE_THREADS)
.withSynchronousQueue()
.build(),
timeout,
bufferedMetrics,
context
);
batchWriter.withLogger(log);
discoveryWriter =
new DiscoveryWriter(new ThreadPoolBuilder()
.withName("Metric Discovery Writing")
.withCorePoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.DISCOVERY_WRITER_MIN_THREADS))
.withMaxPoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.DISCOVERY_WRITER_MAX_THREADS))
.withUnboundedQueue()
.build());
discoveryWriter.withLogger(log);
tokenDiscoveryWriter =
new TokenDiscoveryWriter(
new ThreadPoolBuilder()
.withName("Metric Token Discovery Writing")
.withCorePoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.TOKEN_DISCOVERY_WRITER_MIN_THREADS))
.withMaxPoolSize(Configuration.getInstance().getIntegerProperty(CoreConfig.TOKEN_DISCOVERY_WRITER_MAX_THREADS))
.withUnboundedQueue()
.build());
tokenDiscoveryWriter.withLogger(log);
// RollupRunnable keeps a static one of these. It would be nice if we could register it and share.
MetadataCache rollupTypeCache = MetadataCache.createLoadingCacheInstance(
new TimeValue(48, TimeUnit.HOURS),
Configuration.getInstance().getIntegerProperty(CoreConfig.MAX_ROLLUP_READ_THREADS));
rollupTypeCacher = new RollupTypeCacher(
new ThreadPoolBuilder().withName("Rollup type persistence").build(),
rollupTypeCache);
rollupTypeCacher.withLogger(log);
}
ListenableFuture<List<Boolean>> apply(MetricsCollection collection) throws Exception {
typeAndUnitProcessor.apply(collection);
rollupTypeCacher.apply(collection);
List<List<IMetric>> batches = collection.splitMetricsIntoBatches(BATCH_SIZE);
discoveryWriter.apply(batches);
if (EXP_TOKEN_SEARCH_IMPROVEMENTS)
tokenDiscoveryWriter.processTokens(batches);
return batchWriter.apply(batches);
}
}
@VisibleForTesting
public void shutdownServer() {
try {
allOpenChannels.close().await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
// Pass
}
acceptorGroup.shutdownGracefully();
workerGroup.shutdownGracefully();
}
}