/*
* Copyright 2013 Rackspace
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.rackspacecloud.blueflood.service;
import com.rackspacecloud.blueflood.io.astyanax.AShardStateIO;
import com.rackspacecloud.blueflood.io.IntegrationTestBase;
import com.rackspacecloud.blueflood.io.ShardStateIO;
import com.rackspacecloud.blueflood.io.datastax.DShardStateIO;
import com.rackspacecloud.blueflood.rollup.Granularity;
import com.rackspacecloud.blueflood.rollup.SlotKey;
import com.rackspacecloud.blueflood.utils.Clock;
import com.rackspacecloud.blueflood.utils.Util;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.joda.time.Instant;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.mockito.Mockito;
import java.io.IOException;
import java.util.*;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import static org.mockito.Mockito.when;
@RunWith(Parameterized.class)
public class ShardStateIntegrationTest extends IntegrationTestBase {
private ShardStateIO io;
private List<Integer> shards;
private final long ROLLUP_DELAY_MILLIS = 300000;
private final long SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS = 600000;
private final long LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS = 500000;
public ShardStateIntegrationTest(ShardStateIO io, List<Integer> shardsToTest) {
this.io = io;
shards = shardsToTest;
}
@Test
public void testSingleShardManager() {
long time = 1234000L;
ScheduleContext ctx = new ScheduleContext(time, shards);
ShardStateWorker pull = new ShardStatePuller(shards, ctx.getShardStateManager(), this.io);
ShardStateWorker push = new ShardStatePusher(shards, ctx.getShardStateManager(), this.io);
for (long t = time; t < time + 10000000; t += 1000) {
ctx.update(t + 0, shards.get(0));
ctx.update(t + 2000, shards.get(1));
ctx.update(t + 4000, shards.get(2));
ctx.update(t + 6000, shards.get(3));
}
time += 10000000 + 7;
ctx.setCurrentTimeMillis(time);
push.performOperation();
pull.performOperation();
// the numbers we're testing against are the number of slots per granularity in 10000 seconds.
for (Granularity g : Granularity.rollupGranularities()) {
for (int shard : shards) {
if (g == Granularity.MIN_5)
Assert.assertEquals(34, ctx.getSlotStamps(g, shard).size());
else if (g == Granularity.MIN_20)
Assert.assertEquals(9, ctx.getSlotStamps(g, shard).size());
else if (g == Granularity.MIN_60)
Assert.assertEquals(4, ctx.getSlotStamps(g, shard).size());
else if (g == Granularity.MIN_240)
Assert.assertEquals(1, ctx.getSlotStamps(g, shard).size());
else if (g == Granularity.MIN_1440)
Assert.assertEquals(1, ctx.getSlotStamps(g, shard).size());
}
}
}
@Test
public void testRollupFailureForDelayedMetrics() {
long time = 1234000L;
final Clock mockClock = Mockito.mock(Clock.class);
Collection<Integer> managedShards = Lists.newArrayList(0);
ScheduleContext ingestionCtx = new ScheduleContext(time, managedShards, mockClock);
ScheduleContext rollupCtx = new ScheduleContext(time, managedShards, mockClock);
// Shard workers for rollup ctx
ShardStateWorker rollupPuller = new ShardStatePuller(managedShards, rollupCtx.getShardStateManager(), this.io);
ShardStateWorker rollupPusher = new ShardStatePusher(managedShards, rollupCtx.getShardStateManager(), this.io);
// Shard workers for ingest ctx
ShardStateWorker ingestPuller = new ShardStatePuller(managedShards, ingestionCtx.getShardStateManager(), this.io);
ShardStateWorker ingestPusher = new ShardStatePusher(managedShards, ingestionCtx.getShardStateManager(), this.io);
//***************************************************************************************
//* Testing for on-time metrics (ROLLUP_DELAY_MILLIS) *
//***************************************************************************************
when(mockClock.now()).thenReturn(new Instant(time + 1));
ingestionCtx.update(time, 0);
ingestPusher.performOperation(); // Shard state is persisted on ingestion host
rollupPuller.performOperation(); // Shard state is read on rollup host
rollupCtx.setCurrentTimeMillis(time + ROLLUP_DELAY_MILLIS + 1);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, rollupCtx.getScheduledCount());
simulateHierarchicalScheduling(rollupCtx);
//****************************************************************************************
//* Testing for short delay metrics(SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS) *
//****************************************************************************************
rollupPusher.performOperation();
// Delayed metric is received on ingestion host
final long lastIngestTime1 = time + ROLLUP_DELAY_MILLIS + 2;
ingestPuller.performOperation();
when(mockClock.now()).thenReturn(new Instant(lastIngestTime1));
ingestionCtx.update(time + 1, 0);
ingestPusher.performOperation();
rollupPuller.performOperation();
//last ingest time will be the actual system time as it is the last update time. We adjust it to suit this test
for (Granularity g: Granularity.rollupGranularities()) {
UpdateStamp stamp = rollupCtx.getShardStateManager().getSlotStateManager(0, g).getSlotStamps().get(g.slot(time));
stamp.setLastIngestTimestamp(lastIngestTime1);
}
rollupCtx.setCurrentTimeMillis(lastIngestTime1);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals("Re-roll's should not be scheduled yet", 0, rollupCtx.getScheduledCount());
rollupCtx.setCurrentTimeMillis(time + SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS + 2);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, rollupCtx.getScheduledCount());
simulateHierarchicalScheduling(rollupCtx);
//****************************************************************************************
//* Testing for long delay metrics(LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS) *
//****************************************************************************************
rollupPusher.performOperation();
// Delayed metric is received on ingestion host
final long lastIngestTime2 = time + 2 + SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS + 1;
ingestPuller.performOperation();
when(mockClock.now()).thenReturn(new Instant(lastIngestTime2));
ingestionCtx.update(time + 2, 0);
ingestPusher.performOperation();
rollupPuller.performOperation();
//last ingest time will be the actual system time as it is the last update time. We adjust it to suit this test
for (Granularity g: Granularity.rollupGranularities()) {
UpdateStamp stamp = rollupCtx.getShardStateManager().getSlotStateManager(0, g).getSlotStamps().get(g.slot(time));
stamp.setLastIngestTimestamp(lastIngestTime2);
}
rollupCtx.setCurrentTimeMillis(lastIngestTime2);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals("Re-roll's should not be scheduled yet", 0, rollupCtx.getScheduledCount());
rollupCtx.setCurrentTimeMillis(time + 2 + SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS + LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS + 2);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, rollupCtx.getScheduledCount());
simulateHierarchicalScheduling(rollupCtx);
}
private void simulateHierarchicalScheduling(ScheduleContext rollupCtx) {
// Simulate the hierarchical scheduling of slots
int count = 0;
while (rollupCtx.getScheduledCount() > 0) {
SlotKey slot = rollupCtx.getNextScheduled();
rollupCtx.clearFromRunning(slot);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
count += 1;
}
Assert.assertEquals(5, count); // 5 rollup grans should have been scheduled by now
}
@Test
public void testReRollOfDelayedSlotsBeforeRolledStateGetsPersisted() {
long time = 1234000L;
Collection<Integer> managedShards = Lists.newArrayList(0);
ScheduleContext ingestionCtx = new ScheduleContext(time, managedShards);
ScheduleContext rollupCtx = new ScheduleContext(time, managedShards);
// Shard workers for rollup ctx
ShardStateWorker rollupPuller = new ShardStatePuller(managedShards, rollupCtx.getShardStateManager(), this.io);
ShardStateWorker rollupPusher = new ShardStatePusher(managedShards, rollupCtx.getShardStateManager(), this.io);
// Shard workers for ingest ctx
ShardStateWorker ingestPuller = new ShardStatePuller(managedShards, ingestionCtx.getShardStateManager(), this.io);
ShardStateWorker ingestPusher = new ShardStatePusher(managedShards, ingestionCtx.getShardStateManager(), this.io);
ingestionCtx.update(time + 30000, 0);
ingestPusher.performOperation(); // Shard state is persisted on ingestion host
rollupPuller.performOperation(); // Shard state is read on rollup host
rollupCtx.setCurrentTimeMillis(time + 600000);
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, rollupCtx.getScheduledCount());
// Simulate the hierarchical scheduling of slots
simulateAndVerifyHierarchicalScheduling(rollupCtx);
//We are not calling rollupPusher.performOperation(). So rollup state wont be persisted.
// Delayed metric is received on ingestion host
ingestPuller.performOperation();
ingestionCtx.update(time, 0);
ingestPusher.performOperation();
rollupPuller.performOperation();
//SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS is set to a higher value than ROLLUP_DELAY_MILLIS
rollupCtx.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, 720000, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals("Re-roll's should not be scheduled yet", 0, rollupCtx.getScheduledCount());
}
@Test
public void testSetAllCoarserSlotsDirtyForFinerSlot() {
// Tests that the correct coarser slots are set dirty for a finer slot which was seen out-of-order.
// Prior to a bug fix, clearFromRunning would throw NPE because we were looking up coarser slots
// based on the timestamp on the finer slot's UpdateStamp, not based on the relative courser slot from the finer slot
long time = 1386823200000L;
final Collection<Integer> shards = Lists.newArrayList(123);
ScheduleContext ctxA = new ScheduleContext(time, shards);
ctxA.update(time, 123);
ShardStateManager.SlotStateManager slotStateManager20 =
ctxA.getShardStateManager().getSlotStateManager(123, Granularity.MIN_20);
UpdateStamp stamp = slotStateManager20.getSlotStamps().get(518);
stamp.setTimestamp(time + 3600000L); // add one hour
ctxA.clearFromRunning(SlotKey.of(Granularity.MIN_20, 518, 123));
}
@Test
public void testConcurrentShardManagers() {
long time = 1234000L;
// notice how they share shard 5.
final int commonShard = 5;
final List<Integer> shardsA = Lists.newArrayList(shards);
shardsA.add(commonShard);
final List<Integer> shardsB = Lists.newArrayList(shardsA.get(0)*10,
shardsA.get(1)*10,
shardsA.get(2)*10,
shardsA.get(3)*10, commonShard);
Collection<Integer> allShards = new ArrayList<Integer>() {{
for (int i : Iterables.concat(shardsA, shardsB))
add(i);
}};
ScheduleContext ctxA = new ScheduleContext(time, shardsA);
ScheduleContext ctxB = new ScheduleContext(time, shardsB);
ShardStateWorker pushA = new ShardStatePusher(allShards, ctxA.getShardStateManager(), this.io);
ShardStateWorker pullA = new ShardStatePuller(allShards, ctxA.getShardStateManager(), this.io);
ShardStateWorker pushB = new ShardStatePusher(allShards, ctxB.getShardStateManager(), this.io);
ShardStateWorker pullB = new ShardStatePuller(allShards, ctxB.getShardStateManager(), this.io);
// send a few updates to all contexts.
for (ScheduleContext ctx : new ScheduleContext[] { ctxA, ctxB }) {
for (long t = time; t < time + 10000000; t += 1000) {
ctx.update(t + 0, shardsA.get(0));
ctx.update(t + 1000, shardsB.get(0));
ctx.update(t + 2000, shardsA.get(1));
ctx.update(t + 3000, shardsB.get(1));
ctx.update(t + 4000, shardsA.get(2));
ctx.update(t + 5000, shardsB.get(2));
ctx.update(t + 6000, shardsA.get(3));
ctx.update(t + 7000, shardsB.get(3));
}
}
time += 10000000 + 7;
ctxA.setCurrentTimeMillis(time);
ctxB.setCurrentTimeMillis(time);
// simulate a poll() cycle for each.
pushA.performOperation();
pushB.performOperation();
pullA.performOperation();
pullB.performOperation();
// states should be the same.
for (Granularity g : Granularity.rollupGranularities()) {
for (int shard : allShards)
Assert.assertEquals(ctxA.getSlotStamps(g, shard), ctxB.getSlotStamps(g, shard));
}
time += ROLLUP_DELAY_MILLIS; // this pushes us forward at least one slot.
// now do this: update ctxA, do 2 push/pull cycles on each state. they should sill be the same.
ctxA.update(time, shardsA.get(0));
ctxA.update(time, shardsB.get(0));
ctxA.update(time, shardsA.get(1));
ctxA.update(time, shardsB.get(1));
ctxA.setCurrentTimeMillis(time);
ctxB.setCurrentTimeMillis(time);
// states should not be the same in some places.
Assert.assertFalse(ctxA.getSlotStamps(Granularity.MIN_5, shardsA.get(0)).equals(ctxB.getSlotStamps(Granularity.MIN_5, shardsA.get(0))));
Assert.assertFalse(ctxA.getSlotStamps(Granularity.MIN_5, shardsB.get(0)).equals(ctxB.getSlotStamps(Granularity.MIN_5, shardsB.get(0))));
Assert.assertTrue(ctxA.getSlotStamps(Granularity.MIN_5, shardsA.get(2)).equals(ctxB.getSlotStamps(Granularity.MIN_5, shardsA.get(2))));
Assert.assertTrue(ctxA.getSlotStamps(Granularity.MIN_5, shardsB.get(2)).equals(ctxB.getSlotStamps(Granularity.MIN_5, shardsB.get(2))));
// this is where the syncing should happen. Order is important for a valid test. A contains the updates, so
// I want to put that one first. B contains old data and it gets written second. Part of what I'm verifying
// is that B doesn't overwrite A with data that is obviously old.
// A pushes updated data
pushA.performOperation();
// B tries to push old data (should not overwrite what A just did)
pushB.performOperation();
// B pulls new data (should get updates from A).
pullB.performOperation();
// we didn't do a pull on A because if things are broken, it would have pulled the crap data written by B and
// given the false impression that all timestamps are the same.
// states should have synced up and be the same again.
for (Granularity g : Granularity.rollupGranularities()) {
Assert.assertEquals(ctxA.getSlotStamps(g, commonShard), ctxB.getSlotStamps(g, commonShard));
}
}
@Test
// this test illustrates how loading shard state clobbered the knowledge that a shard,slot had already been
// rolled up.
public void testUpdateClobbering() {
long time = 1234L;
final Collection<Integer> shardsA = Lists.newArrayList(1);
final Collection<Integer> shardsB = Lists.newArrayList(2);
Collection<Integer> allShards = new ArrayList<Integer>() {{
for (int i : Iterables.concat(shardsA, shardsB)) add(i);
}};
ScheduleContext ctxA = new ScheduleContext(time, shardsA);
ShardStateWorker pushA = new ShardStatePusher(allShards, ctxA.getShardStateManager(), this.io);
ShardStateWorker pullA = new ShardStatePuller(allShards, ctxA.getShardStateManager(), this.io);
// update.
time += 1000;
ctxA.setCurrentTimeMillis(time);
ctxA.update(time, 1);
// persist.
pushA.performOperation();
// time goes on.
time += 600000;
ctxA.setCurrentTimeMillis(time);
// should be ready to schedule.
ctxA.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, ctxA.getScheduledCount());
// simulate slots getting run.
simulateAndVerifyHierarchicalScheduling(ctxA);
// verify that scheduling doesn't find anything else.
ctxA.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(0, ctxA.getScheduledCount());
// reloading under these circumstances (no updates) should not affect the schedule.
pullA.performOperation();
ctxA.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(0, ctxA.getScheduledCount());
}
@Test
public void testShardOperationsConcurrency() throws InterruptedException {
final long tryFor = 15000;
final AtomicLong time = new AtomicLong(1234L);
final Collection<Integer> shards = Collections.unmodifiableCollection(Util.parseShards("ALL"));
final ScheduleContext ctx = new ScheduleContext(time.get(), shards);
final CountDownLatch latch = new CountDownLatch(2);
final Throwable[] errBucket = new Throwable[2];
Thread pushPull = new Thread() { public void run() {
ShardStateWorker push = new ShardStatePusher(shards, ctx.getShardStateManager(), ShardStateIntegrationTest.this.io);
ShardStateWorker pull = new ShardStatePuller(shards, ctx.getShardStateManager(), ShardStateIntegrationTest.this.io);
push.setPeriod(1);
pull.setPeriod(1);
long startTime = System.currentTimeMillis();
while (System.currentTimeMillis() - startTime < tryFor) {
try {
push.performOperation();
pull.performOperation();
} catch (Throwable th) {
th.printStackTrace();
errBucket[0] = th;
break;
}
}
latch.countDown();
}};
Thread updateIterator = new Thread() { public void run() {
long start = System.currentTimeMillis();
outer: while (System.currentTimeMillis() - start < tryFor) {
for (int shard : shards) {
time.set(time.get() + 30000);
ctx.setCurrentTimeMillis(time.get());
try {
ctx.update(time.get(), shard);
} catch (Throwable th) {
th.printStackTrace();
errBucket[1] = th;
break outer;
}
}
}
latch.countDown();
}};
pushPull.start();
updateIterator.start();
latch.await(tryFor + 2000, TimeUnit.MILLISECONDS);
Assert.assertNull(errBucket[0]);
Assert.assertNull(errBucket[1]);
}
@Test
public void testConvergenceForMultipleIngestors() throws InterruptedException {
final long tryFor = 1000;
final AtomicLong time = new AtomicLong(1234L);
final Collection<Integer> shards = Collections.unmodifiableCollection(Util.parseShards("ALL"));
final List<ScheduleContext> ctxs = Lists.newArrayList(new ScheduleContext(time.get(), shards), new ScheduleContext(time.get(), shards));
final List<ShardStateWorker> workers = Lists.newArrayList(new ShardStatePusher(shards, ctxs.get(0).getShardStateManager(), ShardStateIntegrationTest.this.io),
new ShardStatePuller(shards, ctxs.get(0).getShardStateManager(), ShardStateIntegrationTest.this.io),
new ShardStatePusher(shards, ctxs.get(1).getShardStateManager(), ShardStateIntegrationTest.this.io),
new ShardStatePuller(shards, ctxs.get(1).getShardStateManager(), ShardStateIntegrationTest.this.io));
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean err = new AtomicBoolean(false);
Thread updateIterator = new Thread() { public void run() {
long start = System.currentTimeMillis();
Random rand = new Random();
outer: while (System.currentTimeMillis() - start < tryFor) {
for (int shard : shards) {
time.set(time.get() + 30000);
ScheduleContext ctx = ctxs.get(rand.nextInt(2));
try {
ctx.update(time.get(), shard);
} catch (Throwable th) {
th.printStackTrace();
err.set(true);
break outer;
}
}
}
latch.countDown();
}};
updateIterator.start();
latch.await();
workers.get(0).performOperation();
workers.get(3).performOperation();
workers.get(2).performOperation();
workers.get(1).performOperation();
workers.get(0).performOperation();
workers.get(3).performOperation();
workers.get(2).performOperation();
workers.get(1).performOperation();
Assert.assertFalse(err.get());
for (Granularity gran : Granularity.rollupGranularities()) {
for (int shard : shards) {
Assert.assertEquals(ctxs.get(0).getSlotStamps(gran, shard).size(), ctxs.get(1).getSlotStamps(gran, shard).size());
for (Map.Entry<Integer, UpdateStamp> entry : ctxs.get(0).getSlotStamps(gran, shard).entrySet()) {
Assert.assertEquals(entry.getValue(), ctxs.get(1).getSlotStamps(gran, shard).get(entry.getKey()));
}
}
}
}
@Test
public void testSlotStateConvergence() throws InterruptedException {
int shard = 0;
long time = 1234000L;
long metricTimeUpdate1 = time + 30000;
long metricsTimeUpdate2 = time + 60000;
Collection<Integer> shards = Lists.newArrayList(shard);
List<ShardStateWorker> allWorkers = new ArrayList<ShardStateWorker>(6);
// Ingestor 1
ScheduleContext ctxIngestor1 = new ScheduleContext(time, shards);
ShardStatePuller pullerIngestor1 = new ShardStatePuller(shards, ctxIngestor1.getShardStateManager(), this.io);
ShardStatePusher pusherIngestor1 = new ShardStatePusher(shards, ctxIngestor1.getShardStateManager(), this.io);
allWorkers.add(pullerIngestor1);
allWorkers.add(pusherIngestor1);
// Ingestor 2
ScheduleContext ctxIngestor2 = new ScheduleContext(time, shards);
ShardStatePuller pullerIngestor2 = new ShardStatePuller(shards, ctxIngestor2.getShardStateManager(), this.io);
ShardStatePusher pusherIngestor2 = new ShardStatePusher(shards, ctxIngestor2.getShardStateManager(), this.io);
allWorkers.add(pullerIngestor2);
allWorkers.add(pusherIngestor2);
// Rollup slave
ScheduleContext ctxRollup = new ScheduleContext(time, shards);
ShardStatePuller pullerRollup = new ShardStatePuller(shards, ctxRollup.getShardStateManager(), this.io);
ShardStatePusher pusherRollup = new ShardStatePusher(shards, ctxRollup.getShardStateManager(), this.io);
allWorkers.add(pullerRollup);
allWorkers.add(pusherRollup);
// Updates for same shard come for same slot on different ingestion contexts
ctxIngestor1.update(metricTimeUpdate1, shard);
ctxIngestor2.update(metricsTimeUpdate2, shard);
makeWorkersSyncState(allWorkers);
// After the sync, the higher timestamp should have "won" and ACTIVE slots should have converged on both the ingestion contexts
for (Granularity gran : Granularity.rollupGranularities())
Assert.assertEquals(ctxIngestor1.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
ctxRollup.setCurrentTimeMillis(time + 800000L);
ctxRollup.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, ctxRollup.getScheduledCount());
simulateAndVerifyHierarchicalScheduling(ctxRollup);
makeWorkersSyncState(allWorkers);
// By this point of time, all contexts should have got the ROLLED state
for (Granularity gran : Granularity.rollupGranularities()) { // Check to see if shard states are indeed matching
Assert.assertEquals(ctxIngestor1.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
Assert.assertEquals(ctxRollup.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
}
Map<Integer, UpdateStamp> slotStamps;
// Test for specific state of ROLLED. Because we have already tested that slot states are same across, it also means ROLLED has been stamped on ingestor1 and ingestor2
for (Granularity gran : Granularity.rollupGranularities()) {
int slot = 0;
if (gran == Granularity.MIN_5) slot = 4;
if (gran == Granularity.MIN_20) slot = 1;
slotStamps = ctxRollup.getSlotStamps(gran, shard);
Assert.assertEquals(slotStamps.get(slot).getState(), UpdateStamp.State.Rolled);
Assert.assertEquals(slotStamps.get(slot).getTimestamp(), metricsTimeUpdate2);
}
// Delayed metric test
long delayedMetricTimestamp = time + 45000; // Notice that this is lesser than the last time we rolled the slot
ctxIngestor1.update(delayedMetricTimestamp, shard);
makeWorkersSyncState(allWorkers);
//last ingest time will be the actual system time as it is the last update time. We adjust it to suit this test
for (Granularity g: Granularity.rollupGranularities()) {
UpdateStamp stamp = ctxRollup.getShardStateManager().getSlotStateManager(0, g).getSlotStamps().get(g.slot(delayedMetricTimestamp));
stamp.setLastIngestTimestamp(delayedMetricTimestamp + 1);
}
// Shard state will be the same throughout and will be marked as ACTIVE
for (Granularity gran : Granularity.rollupGranularities()) {
Assert.assertEquals(ctxIngestor1.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
Assert.assertEquals(ctxRollup.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
}
for (Granularity gran : Granularity.rollupGranularities()) {
int slot = 0;
if (gran == Granularity.MIN_5) slot = 4;
if (gran == Granularity.MIN_20) slot = 1;
slotStamps = ctxRollup.getSlotStamps(gran, shard);
Assert.assertEquals(slotStamps.get(slot).getState(), UpdateStamp.State.Active);
Assert.assertEquals(slotStamps.get(slot).getTimestamp(), delayedMetricTimestamp);
}
// Scheduling the slot for rollup will and following the same process as we did before will mark the state as ROLLED again, but notice that it will have the timestamp of delayed metric
ctxRollup.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
Assert.assertEquals(1, ctxRollup.getScheduledCount());
simulateAndVerifyHierarchicalScheduling(ctxRollup);
makeWorkersSyncState(allWorkers);
for (Granularity gran : Granularity.rollupGranularities()) {
Assert.assertEquals(ctxIngestor1.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
Assert.assertEquals(ctxRollup.getSlotStamps(gran, shard), ctxIngestor2.getSlotStamps(gran, shard));
}
for (Granularity gran : Granularity.rollupGranularities()) {
int slot = 0;
if (gran == Granularity.MIN_5) slot = 4;
if (gran == Granularity.MIN_20) slot = 1;
slotStamps = ctxRollup.getSlotStamps(gran, shard);
Assert.assertEquals(slotStamps.get(slot).getState(), UpdateStamp.State.Rolled);
Assert.assertEquals(slotStamps.get(slot).getTimestamp(), delayedMetricTimestamp);
}
}
private void simulateAndVerifyHierarchicalScheduling(ScheduleContext ctxRollup) {
int count = 0;
while (ctxRollup.getScheduledCount() > 0) {
SlotKey slot = ctxRollup.getNextScheduled();
ctxRollup.clearFromRunning(slot);
ctxRollup.scheduleEligibleSlots(ROLLUP_DELAY_MILLIS, SHORT_DELAY_METRICS_ROLLUP_DELAY_MILLIS, LONG_DELAY_METRICS_ROLLUP_WAIT_MILLIS);
count += 1;
}
Assert.assertEquals(5, count); // 5 rollup grans should have been scheduled by now
}
@After
public void cleanupShardStateIO () {
if (this.io instanceof InMemoryShardStateIO) {
((InMemoryShardStateIO) this.io).cleanUp();
}
}
private void makeWorkersSyncState(List<ShardStateWorker> workers) {
// All states should be synced after 2 attempts, irrespective of operation ordering which is proved by shuffling of orders
for (int i=0; i<=2; i++) {
for (ShardStateWorker worker : workers) {
worker.performOperation();
}
}
}
@Parameterized.Parameters
public static Collection<Object[]> getDifferentShardStateIOInstances() {
List<Object[]> instances = new ArrayList<Object[]>();
instances.add(new Object[] { new AShardStateIO(), Lists.newArrayList(1, 2, 3, 4) });
instances.add(new Object[] { new InMemoryShardStateIO(), Lists.newArrayList(5, 6, 7, 8) });
instances.add(new Object[] { new DShardStateIO(), Lists.newArrayList(1, 2, 3, 4) });
return instances;
}
private static class InMemoryShardStateIO implements ShardStateIO {
private Map<Integer, Map<Granularity, Map<Integer, UpdateStamp>>> map = new HashMap<Integer, Map<Granularity, Map<Integer, UpdateStamp>>>();
@Override
public Collection<SlotState> getShardState(int shard) throws IOException {
Map<Granularity, Map<Integer, UpdateStamp>> updates = map.get(shard);
if (updates == null) {
return new ArrayList<SlotState>();
} else {
List<SlotState> states = new ArrayList<SlotState>();
for (Map.Entry<Granularity, Map<Integer, UpdateStamp>> e0 : updates.entrySet()) {
for (Map.Entry<Integer, UpdateStamp> e1 : e0.getValue().entrySet()) {
SlotState state = new SlotState(e0.getKey(), e1.getKey(), e1.getValue().getState());
state.withTimestamp(e1.getValue().getTimestamp())
.withLastUpdatedTimestamp(state.getLastUpdatedTimestamp());
states.add(state);
}
}
return states;
}
}
@Override
public void putShardState(int shard, Map<Granularity, Map<Integer, UpdateStamp>> slotTimes) throws IOException {
map.put(shard, slotTimes);
}
public void cleanUp() {
map.clear();
}
}
}