package com.hubspot.singularity.scheduler; import java.util.List; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.inject.Inject; import com.hubspot.mesos.client.MesosClient; import com.hubspot.mesos.json.MesosTaskMonitorObject; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularitySlaveUsage; import com.hubspot.singularity.SingularityTaskCurrentUsage; import com.hubspot.singularity.SingularityTaskUsage; import com.hubspot.singularity.config.SingularityConfiguration; import com.hubspot.singularity.data.UsageManager; import com.hubspot.singularity.sentry.SingularityExceptionNotifier; public class SingularityUsagePoller extends SingularityLeaderOnlyPoller { private static final Logger LOG = LoggerFactory.getLogger(SingularityUsagePoller.class); private final SingularityConfiguration configuration; private final MesosClient mesosClient; private final UsageManager usageManager; private final SingularityUsageHelper usageHelper; private final SingularityExceptionNotifier exceptionNotifier; @Inject SingularityUsagePoller(SingularityConfiguration configuration, SingularityUsageHelper usageHelper, UsageManager usageManager, MesosClient mesosClient, SingularityExceptionNotifier exceptionNotifier) { super(configuration.getCheckUsageEveryMillis(), TimeUnit.MILLISECONDS); this.configuration = configuration; this.usageHelper = usageHelper; this.mesosClient = mesosClient; this.usageManager = usageManager; this.exceptionNotifier = exceptionNotifier; } @Override public void runActionOnPoll() { final long now = System.currentTimeMillis(); for (SingularitySlave slave : usageHelper.getSlavesToTrackUsageFor()) { long memoryBytesUsed = 0; double cpusUsed = 0; try { List<MesosTaskMonitorObject> allTaskUsage = mesosClient.getSlaveResourceUsage(slave.getHost()); for (MesosTaskMonitorObject taskUsage : allTaskUsage) { String taskId = taskUsage.getSource(); SingularityTaskUsage usage = getUsage(taskUsage); List<SingularityTaskUsage> taskUsages = usageManager.getTaskUsage(taskId); if (taskUsages.size() + 1 > configuration.getNumUsageToKeep()) { usageManager.deleteSpecificTaskUsage(taskId, taskUsages.get(0).getTimestamp()); } usageManager.saveSpecificTaskUsage(taskId, usage); memoryBytesUsed += usage.getMemoryRssBytes(); if (!taskUsages.isEmpty()) { SingularityTaskUsage lastUsage = taskUsages.get(taskUsages.size() - 1); double taskCpusUsed = ((usage.getCpuSeconds() - lastUsage.getCpuSeconds()) / (usage.getTimestamp() - lastUsage.getTimestamp())); SingularityTaskCurrentUsage currentUsage = new SingularityTaskCurrentUsage(usage.getMemoryRssBytes(), now, taskCpusUsed); usageManager.saveCurrentTaskUsage(taskId, currentUsage); cpusUsed += taskCpusUsed; } } SingularitySlaveUsage slaveUsage = new SingularitySlaveUsage(memoryBytesUsed, now, cpusUsed, allTaskUsage.size()); List<Long> slaveTimestamps = usageManager.getSlaveUsageTimestamps(slave.getId()); if (slaveTimestamps.size() + 1 > configuration.getNumUsageToKeep()) { usageManager.deleteSpecificSlaveUsage(slave.getId(), slaveTimestamps.get(0)); } LOG.debug("Saving slave usage {}", slaveUsage); usageManager.saveSpecificSlaveUsageAndSetCurrent(slave.getId(), slaveUsage); } catch (Exception e) { String message = String.format("Could not get slave usage for host %s", slave.getHost()); LOG.error(message, e); exceptionNotifier.notify(message, e); } } } private SingularityTaskUsage getUsage(MesosTaskMonitorObject taskUsage) { double cpuSeconds = taskUsage.getStatistics().getCpusSystemTimeSecs() + taskUsage.getStatistics().getCpusUserTimeSecs(); return new SingularityTaskUsage(taskUsage.getStatistics().getMemRssBytes(), taskUsage.getStatistics().getTimestamp(), cpuSeconds); } }