ThresholdMonitoringReporterTest.java

package org.stagemonitor.alerting;

import com.codahale.metrics.Timer;

import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.stagemonitor.alerting.alerter.AlertSender;
import org.stagemonitor.alerting.alerter.Alerter;
import org.stagemonitor.alerting.alerter.Subscription;
import org.stagemonitor.alerting.check.Check;
import org.stagemonitor.alerting.check.CheckResult;
import org.stagemonitor.alerting.check.Threshold;
import org.stagemonitor.alerting.incident.CheckResults;
import org.stagemonitor.alerting.incident.ConcurrentMapIncidentRepository;
import org.stagemonitor.alerting.incident.Incident;
import org.stagemonitor.alerting.incident.IncidentRepository;
import org.stagemonitor.core.CorePlugin;
import org.stagemonitor.core.MeasurementSession;
import org.stagemonitor.configuration.ConfigurationRegistry;
import org.stagemonitor.core.metrics.metrics2.Metric2Registry;
import org.stagemonitor.core.metrics.metrics2.MetricName;
import org.stagemonitor.core.util.JsonUtils;

import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.stagemonitor.core.metrics.MetricsReporterTestHelper.timer;
import static org.stagemonitor.core.metrics.metrics2.MetricName.name;

public class ThresholdMonitoringReporterTest {

	private final MeasurementSession measurementSession = new MeasurementSession("testApp", "testHost", "testInstance");
	private ThresholdMonitoringReporter thresholdMonitoringReporter;
	private Alerter alerter;
	private IncidentRepository incidentRepository;
	private AlertingPlugin alertingPlugin;

	@Before
	public void setUp() throws Exception {
		ConfigurationRegistry configuration = mock(ConfigurationRegistry.class);
		when(configuration.getConfig(CorePlugin.class)).thenReturn(mock(CorePlugin.class));
		alertingPlugin = mock(AlertingPlugin.class);
		Subscription subscription = new Subscription();
		subscription.setAlerterType("Test Alerter");
		subscription.setAlertOnBackToOk(true);
		subscription.setAlertOnWarn(true);
		subscription.setAlertOnError(true);
		subscription.setAlertOnCritical(true);
		when(alertingPlugin.getSubscriptionsByIds()).thenReturn(Collections.singletonMap("1", subscription));
		when(configuration.getConfig(AlertingPlugin.class)).thenReturn(alertingPlugin);
		alerter = mock(Alerter.class);
		when(alerter.getAlerterType()).thenReturn("Test Alerter");
		when(alerter.isAvailable()).thenReturn(true);
		AlertSender alertSender = new AlertSender(configuration, Collections.singletonList(alerter));

		incidentRepository = spy(new ConcurrentMapIncidentRepository(new ConcurrentHashMap<String, Incident>()));
		thresholdMonitoringReporter = ThresholdMonitoringReporter.forRegistry(new Metric2Registry())
				.alertingPlugin(alertingPlugin)
				.alertSender(alertSender)
				.incidentRepository(incidentRepository)
				.measurementSession(measurementSession)
				.build();
	}

	@Test
	public void testAlerting() throws Exception {
		Check check = createCheckCheckingMean(1, 5);

		when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check));

		checkMetrics();

		ArgumentCaptor<Alerter.AlertArguments> alertArguments = ArgumentCaptor.forClass(Alerter.AlertArguments.class);
		verify(alerter).alert(alertArguments.capture());
		final Incident incident = alertArguments.getValue().getIncident();
		assertEquals(check.getId(), incident.getCheckId());
		assertEquals("Test Timer", incident.getCheckName());
		assertEquals(CheckResult.Status.OK, incident.getOldStatus());
		assertEquals(CheckResult.Status.WARN, incident.getNewStatus());

		assertEquals(1, incident.getCheckResults().size());
		List<CheckResult> checkResults = incident.getCheckResults().iterator().next().getResults();
		assertEquals(2, checkResults.size());

		CheckResult result = checkResults.get(0);
		assertEquals("test_timer,signature=timer3 mean < 5.0 is false", result.getFailingExpression());
		assertEquals(6.0, result.getCurrentValue(), 0);
		assertEquals(CheckResult.Status.WARN, result.getStatus());
	}

	@Test
	public void testAlertAfter2Failures() throws Exception {
		Check check = createCheckCheckingMean(2, 6);

		when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check));

		checkMetrics();
		verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class));
		verify(incidentRepository).createIncident(any(Incident.class));

		checkMetrics();
		verify(alerter).alert(any(Alerter.AlertArguments.class));
		verify(incidentRepository).updateIncident(any(Incident.class));
	}

	@Test
	public void testNoAlertWhenFailureRecovers() throws Exception {
		Check check = createCheckCheckingMean(2, 6);
		when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check));

		// violation
		checkMetrics(7, 0, 0);
		verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class));
		final Incident incident = incidentRepository.getIncidentByCheckId(check.getId());
		assertNotNull(incident);
		assertEquals(CheckResult.Status.OK, incident.getOldStatus());
		assertEquals(CheckResult.Status.WARN, incident.getNewStatus());
		assertNotNull(incident.getFirstFailureAt());
		assertNull(incident.getResolvedAt());
		assertEquals(1, incident.getConsecutiveFailures());
		System.out.println(incident);

		// back to ok
		checkMetrics(1, 0, 0);
		verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class));
		assertNull(incidentRepository.getIncidentByCheckId(check.getId()));
	}

	@Test
	public void testAlertWhenBackToOk() throws Exception {
		Check check = createCheckCheckingMean(1, 6);
		when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check));

		// violation
		checkMetrics(7, 0, 0);
		verify(alerter, times(1)).alert(any(Alerter.AlertArguments.class));
		Incident incident = incidentRepository.getIncidentByCheckId(check.getId());
		assertNotNull(incident);
		assertEquals(CheckResult.Status.OK, incident.getOldStatus());
		assertEquals(CheckResult.Status.WARN, incident.getNewStatus());
		assertNotNull(incident.getFirstFailureAt());
		assertNull(incident.getResolvedAt());
		assertEquals(1, incident.getConsecutiveFailures());
		System.out.println(incident);

		// back to ok
		checkMetrics(1, 0, 0);
		ArgumentCaptor<Alerter.AlertArguments> alertArguments = ArgumentCaptor.forClass(Alerter.AlertArguments.class);
		verify(alerter, times(2)).alert(alertArguments.capture());
		incident = alertArguments.getValue().getIncident();
		assertNotNull(incident);
		assertEquals(CheckResult.Status.WARN, incident.getOldStatus());
		assertEquals(CheckResult.Status.OK, incident.getNewStatus());
		assertNotNull(incident.getFirstFailureAt());
		assertNotNull(incident.getResolvedAt());
	}

	@Test
	public void testDontDeleteIncidentIfThereAreNonOkResultsFromOtherInstances() {
		Check check = createCheckCheckingMean(2, 6);
		when(alertingPlugin.getChecks()).thenReturn(Collections.singletonMap(check.getId(), check));
		incidentRepository.createIncident(
				new Incident(check, new MeasurementSession("testApp", "testHost2", "testInstance"),
						Arrays.asList(new CheckResult("test", 10, CheckResult.Status.CRITICAL))));

		checkMetrics(7, 0, 0);
		verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class));
		verify(incidentRepository).updateIncident(any(Incident.class));
		Incident storedIncident = incidentRepository.getIncidentByCheckId(check.getId());
		assertEquals(CheckResult.Status.CRITICAL, storedIncident.getOldStatus());
		assertEquals(CheckResult.Status.CRITICAL, storedIncident.getNewStatus());
		assertEquals(2, storedIncident.getCheckResults().size());
		assertEquals(2, storedIncident.getVersion());
		boolean containsTestHost = false;
		boolean containsTestHost2 = false;
		for (CheckResults checkResults : storedIncident.getCheckResults()) {
			if (checkResults.getMeasurementSession().getHostName().equals("testHost2")) {
				containsTestHost2 = true;
			} else if (checkResults.getMeasurementSession().getHostName().equals("testHost")) {
				containsTestHost = true;
			}
		}
		assertTrue(containsTestHost);
		assertTrue(containsTestHost2);
		System.out.println(storedIncident);
		System.out.println(JsonUtils.toJson(storedIncident));

		checkMetrics(1, 0, 0);
		verify(alerter, times(0)).alert(any(Alerter.AlertArguments.class));
		verify(incidentRepository, times(0)).deleteIncident(any(Incident.class));
		verify(incidentRepository, times(2)).updateIncident(any(Incident.class));

		storedIncident = incidentRepository.getIncidentByCheckId(check.getId());
		assertEquals(CheckResult.Status.CRITICAL, storedIncident.getOldStatus());
		assertEquals(CheckResult.Status.CRITICAL, storedIncident.getNewStatus());
		assertEquals(1, storedIncident.getCheckResults().size());
		assertEquals(3, storedIncident.getVersion());
	}

	public static Check createCheckCheckingMean(int alertAfterXFailures, long meanMs) {
		Check check = new Check();
		check.setName("Test Timer");
		check.setApplication("testApp");
		check.setTarget(name("test_timer").build());
		check.setAlertAfterXFailures(alertAfterXFailures);
		check.getWarn().add(new Threshold("mean", Threshold.Operator.LESS, meanMs));
		return check;
	}

	private void checkMetrics() {
		checkMetrics(5, 4, 6);
	}

	private void checkMetrics(long timer1Mean, long timer2Mean, long timer3Mean) {
		final Map<MetricName, Timer> timers = new HashMap<>();
		timers.put(name("test_timer").tag("signature", "timer1").build(), timer(TimeUnit.MILLISECONDS.toNanos(timer1Mean)));
		timers.put(name("test_timer").tag("signature", "timer2").build(), timer(TimeUnit.MILLISECONDS.toNanos(timer2Mean)));
		timers.put(name("test_timer").tag("signature", "timer3").build(), timer(TimeUnit.MILLISECONDS.toNanos(timer3Mean)));
		timers.put(name("test_other_timer").tag("signature", "timer4").build(), timer(TimeUnit.MILLISECONDS.toNanos(999)));
		thresholdMonitoringReporter.reportMetrics(
				new HashMap<>(),
				new HashMap<>(),
				new HashMap<>(),
				new HashMap<>(),
				timers
		);
	}
}