/**
* Copyright 2016-2017 Seznam.cz, a.s.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package cz.seznam.euphoria.inmem;
import cz.seznam.euphoria.core.client.dataset.windowing.MergingWindowing;
import cz.seznam.euphoria.core.client.dataset.windowing.Window;
import cz.seznam.euphoria.core.client.dataset.windowing.WindowedElement;
import cz.seznam.euphoria.core.client.dataset.windowing.Windowing;
import cz.seznam.euphoria.core.client.functional.BinaryFunction;
import cz.seznam.euphoria.core.client.functional.UnaryFunction;
import cz.seznam.euphoria.core.client.operator.ReduceStateByKey;
import cz.seznam.euphoria.core.client.operator.state.ListStorage;
import cz.seznam.euphoria.core.client.operator.state.ListStorageDescriptor;
import cz.seznam.euphoria.core.client.operator.state.MergingStorageDescriptor;
import cz.seznam.euphoria.core.client.operator.state.State;
import cz.seznam.euphoria.core.client.operator.state.StateFactory;
import cz.seznam.euphoria.core.client.operator.state.StateMerger;
import cz.seznam.euphoria.core.client.operator.state.Storage;
import cz.seznam.euphoria.core.client.operator.state.StorageDescriptor;
import cz.seznam.euphoria.core.client.operator.state.StorageProvider;
import cz.seznam.euphoria.core.client.operator.state.ValueStorage;
import cz.seznam.euphoria.core.client.operator.state.ValueStorageDescriptor;
import cz.seznam.euphoria.core.client.triggers.Trigger;
import cz.seznam.euphoria.core.client.triggers.TriggerContext;
import cz.seznam.euphoria.core.client.util.Pair;
import cz.seznam.euphoria.shaded.guava.com.google.common.base.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toSet;
class ReduceStateByKeyReducer implements Runnable {
private static final Logger LOG = LoggerFactory.getLogger(ReduceStateByKeyReducer.class);
static final class KeyedElementCollector extends WindowedElementCollector<Object> {
private final Object key;
KeyedElementCollector(Collector<Datum> wrap, Window window, Object key,
Supplier<Long> stampSupplier) {
super(wrap, stampSupplier);
this.key = key;
this.window = window;
}
@Override
public void collect(Object elem) {
super.collect(Pair.of(key, elem));
}
} // ~ end of KeyedElementCollector
final class ClearingValueStorage<T> implements ValueStorage<T> {
private final ValueStorage<T> wrap;
private final KeyedWindow<?, ?> scope;
private final StorageDescriptor descriptor;
ClearingValueStorage(ValueStorage<T> wrap,
KeyedWindow<?, ?> scope,
StorageDescriptor descriptor) {
this.wrap = wrap;
this.scope = scope;
this.descriptor = descriptor;
}
@Override
public void clear() {
wrap.clear();
processing.triggerStorage.removeStorage(scope, descriptor);
}
@Override
public void set(T value) {
wrap.set(value);
}
@Override
public T get() {
return wrap.get();
}
} // ~ end of ClearingValueStorage
final class ClearingListStorage<T> implements ListStorage<T> {
private final ListStorage<T> wrap;
private final KeyedWindow scope;
private final StorageDescriptor descriptor;
public ClearingListStorage(ListStorage<T> wrap, KeyedWindow scope,
StorageDescriptor descriptor) {
this.wrap = wrap;
this.scope = scope;
this.descriptor = descriptor;
}
@Override
public void clear() {
wrap.clear();
processing.triggerStorage.removeStorage(scope, descriptor);
}
@Override
public void add(T element) {
wrap.add(element);
}
@Override
public Iterable<T> get() {
return wrap.get();
}
} // ~ end of ClearingListStorage
class ElementTriggerContext implements TriggerContext {
private final KeyedWindow<Window, Object> scope;
ElementTriggerContext(KeyedWindow<Window, Object> scope) {
this.scope = scope;
}
KeyedWindow getScope() {
return scope;
}
@Override
public boolean registerTimer(long stamp, Window window) {
Preconditions.checkState(this.scope.window().equals(window));
return scheduler.scheduleAt(
stamp, this.scope, guardTriggerable(createTriggerHandler()));
}
@Override
public void deleteTimer(long stamp, Window window) {
Preconditions.checkState(this.scope.window().equals(window));
scheduler.cancel(stamp, this.scope);
}
@Override
public long getCurrentTimestamp() {
return scheduler.getCurrentTimestamp();
}
@Override
public <T> ValueStorage<T> getValueStorage(ValueStorageDescriptor<T> descriptor) {
return new ClearingValueStorage<>(
processing.triggerStorage.getValueStorage(this.scope, descriptor),
this.scope,
descriptor);
}
@Override
public <T> ListStorage<T> getListStorage(ListStorageDescriptor<T> descriptor) {
return new ClearingListStorage<>(
processing.triggerStorage.getListStorage(this.scope, descriptor),
this.scope,
descriptor);
}
} // ~ end of ElementTriggerContext
class MergingElementTriggerContext
extends ElementTriggerContext
implements TriggerContext.TriggerMergeContext {
final Collection<KeyedWindow<Window, Object>> mergeSources;
MergingElementTriggerContext(KeyedWindow<Window, Object> target,
Collection<KeyedWindow<Window, Object>> sources) {
super(target);
this.mergeSources = sources;
}
@SuppressWarnings("unchecked")
@Override
public void mergeStoredState(StorageDescriptor storageDescriptor) {
if (!(storageDescriptor instanceof MergingStorageDescriptor)) {
throw new IllegalStateException("Storage descriptor must support merging!");
}
MergingStorageDescriptor descr = (MergingStorageDescriptor) storageDescriptor;
BinaryFunction mergeFn = descr.getMerger();
// create a new instance of storage
Storage merged;
if (descr instanceof ValueStorageDescriptor) {
merged = getValueStorage((ValueStorageDescriptor) descr);
} else if (descr instanceof ListStorageDescriptor) {
merged = getListStorage((ListStorageDescriptor) descr);
} else {
throw new IllegalStateException("Cannot merge states for " + descr);
}
// merge all existing (non null) trigger states
for (KeyedWindow<?, ?> w : this.mergeSources) {
Storage s = processing.triggerStorage.getStorage(w, storageDescriptor);
if (s != null) {
mergeFn.apply(merged, s);
}
}
}
} // ~ end of MergingElementTriggerContext
static final class WindowRegistry {
final Map<Window, Map<Object, State>> windows = new HashMap<>();
final Map<Object, Set<Window>> keyMap = new HashMap<>();
State removeWindowState(KeyedWindow<?, ?> kw) {
Map<Object, State> keys = windows.get(kw.window());
if (keys != null) {
State state = keys.remove(kw.key());
// ~ garbage collect on windows level
if (keys.isEmpty()) {
windows.remove(kw.window());
}
Set<Window> actives = keyMap.get(kw.key());
if (actives != null) {
actives.remove(kw.window());
if (actives.isEmpty()) {
keyMap.remove(kw.key());
}
}
return state;
}
return null;
}
void setWindowState(KeyedWindow<?, ?> kw, State state) {
Map<Object, State> keys = windows.get(kw.window());
if (keys == null) {
windows.put(kw.window(), keys = new HashMap<>());
}
keys.put(kw.key(), state);
Set<Window> actives = keyMap.get(kw.key());
if (actives == null) {
keyMap.put(kw.key(), actives = new HashSet<>());
}
actives.add(kw.window());
}
State getWindowState(KeyedWindow<?, ?> kw) {
return getWindowState(kw.window(), kw.key());
}
State getWindowState(Window window, Object key) {
Map<Object, State> keys = windows.get(window);
if (keys != null) {
return keys.get(key);
}
return null;
}
Map<Object, State> getWindowStates(Window window) {
return windows.get(window);
}
Set<Window> getActivesForKey(Object itemKey) {
return keyMap.get(itemKey);
}
} // ~ end of WindowRegistry
// statistics related to the running operator
final class ProcessingStats {
final ProcessingState processing;
long watermarkPassed = -1;
long maxElementStamp = -1;
long lastLogTime = -1;
ProcessingStats(ProcessingState processing) {
this.processing = processing;
}
void update(long elementStamp) {
watermarkPassed = processing.triggering.getCurrentTimestamp();
if (maxElementStamp < elementStamp) {
maxElementStamp = elementStamp;
}
long now = System.currentTimeMillis();
if (lastLogTime + 5000 < now) {
log();
lastLogTime = now;
}
}
private void log() {
LOG.info("Reducer {} processing stats: at watermark {}, maxElementStamp {}",
new Object[] {
ReduceStateByKeyReducer.this.name,
watermarkPassed,
maxElementStamp});
}
} // ~ end of ProcessingStats
static final class ScopedStorage {
static final class StorageKey {
private final Object itemKey;
private final Window itemWindow;
private final String storeId;
public StorageKey(Object itemKey, Window itemWindow, String storeId) {
this.itemKey = itemKey;
this.itemWindow = itemWindow;
this.storeId = storeId;
}
@Override
public boolean equals(Object o) {
if (o instanceof StorageKey) {
StorageKey that = (StorageKey) o;
return Objects.equals(this.itemKey, that.itemKey)
&& Objects.equals(this.itemWindow, that.itemWindow)
&& Objects.equals(this.storeId, that.storeId);
}
return false;
}
@Override
public int hashCode() {
int result = itemKey != null ? itemKey.hashCode() : 0;
result = 31 * result + (itemWindow != null ? itemWindow.hashCode() : 0);
result = 31 * result + (storeId != null ? storeId.hashCode() : 0);
return result;
}
}
final HashMap<StorageKey, Object> store = new HashMap<>();
final StorageProvider storageProvider;
ScopedStorage(StorageProvider storageProvider) {
this.storageProvider = storageProvider;
}
Storage removeStorage(KeyedWindow scope, StorageDescriptor descriptor) {
StorageKey skey = storageKey(scope, descriptor);
return (Storage) store.remove(skey);
}
Storage getStorage(KeyedWindow scope, StorageDescriptor descriptor) {
StorageKey skey = storageKey(scope, descriptor);
return (Storage) store.get(skey);
}
@SuppressWarnings("unchecked")
<T> ValueStorage<T> getValueStorage(
KeyedWindow scope, ValueStorageDescriptor<T> descriptor)
{
StorageKey skey = storageKey(scope, descriptor);
Storage s = (Storage) store.get(skey);
if (s == null) {
store.put(skey, s = storageProvider.getValueStorage(descriptor));
}
return (ValueStorage<T>) s;
}
@SuppressWarnings("unchecked")
<T> ListStorage<T> getListStorage(
KeyedWindow scope, ListStorageDescriptor<T> descriptor) {
StorageKey skey = storageKey(scope, descriptor);
Storage s = (Storage) store.get(skey);
if (s == null) {
store.put(skey, s = storageProvider.getListStorage(descriptor));
}
return (ListStorage<T>) s;
}
private StorageKey storageKey(KeyedWindow kw, StorageDescriptor desc) {
return new StorageKey(kw.key(), kw.window(), desc.getName());
}
} // ~ end of ScopedStorage
final class ProcessingState {
final boolean allowEarlyEmitting;
final ScopedStorage triggerStorage;
final StorageProvider storageProvider;
final WindowRegistry wRegistry = new WindowRegistry();
final Collector<Datum> stateOutput;
final BlockingQueue<Datum> rawOutput;
final TriggerScheduler<Window, Object> triggering;
final StateFactory stateFactory;
final StateMerger stateMerger;
final ProcessingStats stats = new ProcessingStats(this);
// flushed windows with the time of the flush
private Map<Window, Long> flushedWindows = new HashMap<>();
private ProcessingState(
BlockingQueue<Datum> output,
TriggerScheduler<Window, Object> triggering,
StateFactory stateFactory,
StateMerger stateMerger,
StorageProvider storageProvider,
boolean allowEarlyEmitting) {
this.triggerStorage = new ScopedStorage(storageProvider);
this.storageProvider = storageProvider;
this.stateOutput = InMemExecutor.QueueCollector.wrap(requireNonNull(output));
this.rawOutput = output;
this.triggering = requireNonNull(triggering);
this.stateFactory = requireNonNull(stateFactory);
this.stateMerger = requireNonNull(stateMerger);
this.allowEarlyEmitting = allowEarlyEmitting;
}
Map<Window, Long> takeFlushedWindows() {
if (flushedWindows.isEmpty()) {
return Collections.emptyMap();
}
Map<Window, Long> flushed = flushedWindows;
flushedWindows = new HashMap<>();
return flushed;
}
// ~ signal eos further down the output channel
void closeOutput() {
try {
this.rawOutput.put(Datum.endOfStream());
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
/**
* Flushes (emits result) the specified window.
*/
@SuppressWarnings("unchecked")
void flushWindow(KeyedWindow<?, ?> kw) {
State state = wRegistry.getWindowState(kw);
if (state == null) {
return;
}
state.flush(newCollector(kw));
// ~ remember we flushed the window such that we can emit one
// notification to downstream operators for all keys in this window
flushedWindows.put(kw.window(), getCurrentWatermark());
}
/**
* Purges the specified window.
*/
State purgeWindow(KeyedWindow<?, ?> kw) {
State state = wRegistry.removeWindowState(kw);
if (state == null) {
return null;
}
state.close();
return state;
}
/**
* Flushes and closes all window storages and clear the window registry.
*/
@SuppressWarnings("unchecked")
void flushAndCloseAllWindows() {
for (Map.Entry<Window, Map<Object, State>> windowState : wRegistry.windows.entrySet()) {
for (Map.Entry<Object, State> itemState : windowState.getValue().entrySet()) {
State state = itemState.getValue();
state.flush(newCollector(new KeyedWindow(windowState.getKey(), itemState.getKey())));
state.close();
}
}
wRegistry.windows.clear();
}
@SuppressWarnings("unchecked")
State getWindowStateForUpdate(KeyedWindow<?, ?> kw) {
State state = wRegistry.getWindowState(kw);
if (state == null) {
// ~ if no such window yet ... set it up
state = stateFactory.createState(
storageProvider,
allowEarlyEmitting ? newCollector(kw) : null);
wRegistry.setWindowState(kw, state);
}
return state;
}
private KeyedElementCollector newCollector(KeyedWindow kw) {
return new KeyedElementCollector(
stateOutput, kw.window(), kw.key(),
processing.triggering::getCurrentTimestamp);
}
// ~ returns a freely modifable collection of windows actively
// for the given item key
Set<Window> getActivesForKey(Object itemKey) {
Set<Window> actives = wRegistry.getActivesForKey(itemKey);
if (actives == null || actives.isEmpty()) {
return new HashSet<>();
} else {
return new HashSet<>(actives);
}
}
// ~ merges window states for sources and places it on 'target'
// ~ returns a list of windows which were merged and actually removed
@SuppressWarnings("unchecked")
Set<KeyedWindow<Window, Object>>
mergeWindowStates(Collection<Window> sources, KeyedWindow<Window, Object> target) {
// ~ first find the states to be merged into `target`
List<Pair<Window, State>> merge = new ArrayList<>(sources.size());
for (Window source : sources) {
if (!source.equals(target.window())) {
State state = wRegistry.removeWindowState(new KeyedWindow<>(source, target.key()));
if (state != null) {
merge.add(Pair.of(source, state));
}
}
}
// ~ prepare for the state merge
List<State<?, ?>> statesToMerge = new ArrayList<>(merge.size());
// ~ if any of the states emits any data during the merge, we'll make
// sure it happens in the scope of the merge target window
for (Pair<Window, State> m : merge) {
statesToMerge.add(m.getSecond());
}
// ~ now merge the state and re-assign it to the merge-window
if (!statesToMerge.isEmpty()) {
State targetState = getWindowStateForUpdate(target);
stateMerger.merge(targetState, statesToMerge);
}
// ~ finally return a list of windows which were actually merged and removed
return merge.stream()
.map(Pair::getFirst)
.map(w -> new KeyedWindow<>(w, target.key()))
.collect(toSet());
}
/** Update current timestamp by given watermark. */
void updateStamp(long stamp) {
triggering.updateStamp(stamp);
}
/** Update trigger of given window. */
void onUpstreamWindowTrigger(Window window, long stamp) {
LOG.debug("Updating trigger of window {} to {}", window, stamp);
Map<Object, State> ws = wRegistry.getWindowStates(window);
if (ws == null || ws.isEmpty()) {
return;
}
for (Map.Entry<Object, State> e : ws.entrySet()) {
@SuppressWarnings("unchecked")
KeyedWindow<Window, Object> kw = new KeyedWindow<>(window, e.getKey());
Triggerable<Window, Object> t = guardTriggerable((tstamp, tkw) -> {
flushWindow(tkw);
purgeWindow(tkw);
trigger.onClear(kw.window(), new ElementTriggerContext(tkw));
});
if (!triggering.scheduleAt(stamp, kw, t)) {
LOG.debug("Manually firing already passed flush event for window {}", kw);
t.fire(stamp, kw);
}
}
}
void emitWatermark() {
final long stamp = getCurrentWatermark();
try {
rawOutput.put(Datum.watermark(stamp));
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
} // ~ end of ProcessingState
private final BlockingQueue<Datum> input;
private final BlockingQueue<Datum> output;
private final boolean isAttachedWindowing;
private final Windowing windowing;
private final UnaryFunction keyExtractor;
private final UnaryFunction valueExtractor;
private final WatermarkEmitStrategy watermarkStrategy;
private final String name;
private final Trigger<Window> trigger;
// ~ both of these are guarded by "processing"
private final ProcessingState processing;
private final TriggerScheduler<Window, Object> scheduler;
private long currentElementTime;
@SuppressWarnings("unchecked")
ReduceStateByKeyReducer(ReduceStateByKey operator,
String name,
BlockingQueue<Datum> input,
BlockingQueue<Datum> output,
UnaryFunction keyExtractor,
UnaryFunction valueExtractor,
TriggerScheduler scheduler,
WatermarkEmitStrategy watermarkStrategy,
StorageProvider storageProvider,
boolean allowEarlyEmitting) {
this.name = requireNonNull(name);
this.input = requireNonNull(input);
this.output = requireNonNull(output);
this.isAttachedWindowing = operator.getWindowing() == null;
this.windowing = isAttachedWindowing
? AttachedWindowing.INSTANCE : operator.getWindowing();
this.keyExtractor = requireNonNull(keyExtractor);
this.valueExtractor = requireNonNull(valueExtractor);
this.watermarkStrategy = requireNonNull(watermarkStrategy);
this.trigger = requireNonNull(windowing.getTrigger());
this.scheduler = requireNonNull(scheduler);
this.processing = new ProcessingState(
output, scheduler,
requireNonNull(operator.getStateFactory()),
requireNonNull(operator.getStateMerger()),
storageProvider,
allowEarlyEmitting);
}
<W extends Window, K> Triggerable<W, K> guardTriggerable(Triggerable<W, K> t) {
return ((timestamp, kw) -> {
synchronized (processing) {
t.fire(timestamp, kw);
}
});
}
Triggerable<Window, Object> createTriggerHandler() {
return ((timestamp, kw) -> {
// ~ let trigger know about the time event and process window state
// according to trigger result
ElementTriggerContext ectx = new ElementTriggerContext(kw);
Trigger.TriggerResult result = trigger.onTimer(timestamp, kw.window(), ectx);
handleTriggerResult(result, ectx);
});
}
void handleTriggerResult(
Trigger.TriggerResult result, ElementTriggerContext ctx) {
KeyedWindow scope = ctx.scope;
// Flush window (emit the internal state to output)
if (result.isFlush()) {
processing.flushWindow(scope);
}
// Purge given window (discard internal state and cancel all triggers)
if (result.isPurge()) {
processing.purgeWindow(scope);
trigger.onClear(scope.window(), ctx);
}
// emit a warning about late comers
if (result == Trigger.TriggerResult.PURGE) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Window {} discarded for key {} at current watermark {} with scheduler {}",
new Object[]{ctx.getScope().window(), ctx.getScope().key(),
getCurrentWatermark(), scheduler.getClass()});
}
}
}
@Override
public void run() {
LOG.debug("Started ReduceStateByKeyReducer for operator {}", name);
watermarkStrategy.schedule(processing::emitWatermark);
boolean run = true;
while (run) {
try {
// ~ process incoming data
Datum item = input.take();
// ~ make sure to avoid race-conditions with triggers from another
// thread (i.e. processing-time-trigger-scheduler)
synchronized (processing) {
if (item.isElement()) {
currentElementTime = item.getTimestamp();
processing.stats.update(currentElementTime);
processInput(item);
} else if (item.isEndOfStream()) {
processEndOfStream((Datum.EndOfStream) item);
run = false;
} else if (item.isWatermark()) {
processWatermark((Datum.Watermark) item);
} else if (item.isWindowTrigger()) {
processWindowTrigger((Datum.WindowTrigger) item);
}
// ~ send pending notifications about flushed windows
notifyFlushedWindows();
}
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
break;
}
}
}
private void notifyFlushedWindows() throws InterruptedException {
// ~ send notifications to downstream operators about flushed windows
long max = 0;
for (Map.Entry<Window, Long> w : processing.takeFlushedWindows().entrySet()) {
output.put(Datum.windowTrigger(w.getKey(), w.getValue()));
long flushTime = w.getValue();
if (flushTime > max) {
max = flushTime;
}
}
output.put(Datum.watermark(max));
}
private void processInput(WindowedElement element) {
if (windowing instanceof MergingWindowing) {
processInputMerging(element);
} else {
processInputNonMerging(element);
}
}
@SuppressWarnings("unchecked")
private void processInputNonMerging(WindowedElement element) {
Object item = element.getElement();
Object itemKey = keyExtractor.apply(item);
Object itemValue = valueExtractor.apply(item);
Iterable<Window> windows = windowing.assignWindowsToElement(element);
for (Window window : windows) {
ElementTriggerContext pitctx =
new ElementTriggerContext(new KeyedWindow(window, itemKey));
State windowState = processing.getWindowStateForUpdate(pitctx.getScope());
windowState.add(itemValue);
Trigger.TriggerResult result =
trigger.onElement(getCurrentElementTime(), window, pitctx);
// ~ handle trigger result
handleTriggerResult(result, pitctx);
}
}
@SuppressWarnings("unchecked")
private void processInputMerging(WindowedElement element) {
assert windowing instanceof MergingWindowing;
Object item = element.getElement();
Object itemKey = keyExtractor.apply(item);
Object itemValue = valueExtractor.apply(item);
Iterable<Window> windows = windowing.assignWindowsToElement(element);
for (Window window : windows) {
// ~ first try to merge the new window into the set of existing ones
Set<Window> current = processing.getActivesForKey(itemKey);
current.add(window);
Collection<Pair<Collection<Window>, Window>> cmds =
((MergingWindowing) windowing).mergeWindows(current);
for (Pair<Collection<Window>, Window> cmd : cmds) {
Collection<Window> srcs = cmd.getFirst();
Window trgt = cmd.getSecond();
// ~ if the new window was merged, continue processing the merge
// target as the window the new input element should be placed into
if (srcs.contains(window)) {
window = trgt;
}
// ~ merge window (item) states
Set<KeyedWindow<Window, Object>> merged =
processing.mergeWindowStates(srcs, new KeyedWindow<>(trgt, itemKey));
// ~ merge window trigger states
trigger.onMerge(trgt,
new MergingElementTriggerContext(new KeyedWindow<>(trgt, itemKey), merged));
// ~ clear window trigger states for the merged windows
for (KeyedWindow w : merged) {
trigger.onClear(w.window(), new ElementTriggerContext(w));
}
}
// ~ only now, add the element to the new window
ElementTriggerContext pitctx =
new ElementTriggerContext(new KeyedWindow(window, itemKey));
State windowState = processing.getWindowStateForUpdate(pitctx.getScope());
windowState.add(itemValue);
Trigger.TriggerResult tr =
trigger.onElement(getCurrentElementTime(), window, pitctx);
// ~ handle trigger result
handleTriggerResult(tr, pitctx);
}
}
private void processWatermark(Datum.Watermark watermark) {
// update current stamp
long stamp = watermark.getTimestamp();
processing.updateStamp(stamp);
}
private void processWindowTrigger(Datum.WindowTrigger trigger) {
if (isAttachedWindowing) {
// reregister trigger of given window
// FIXME: move this to windowing itself so that attached windowing
// can be implemented 'natively' as instance of generic windowing
processing.onUpstreamWindowTrigger(trigger.getWindow(), trigger.getTimestamp());
}
}
private void processEndOfStream(Datum.EndOfStream eos) throws InterruptedException {
// ~ flush all registered triggers
scheduler.updateStamp(Long.MAX_VALUE);
// ~ stop triggers - there actually should be none left
scheduler.close();
// close all states
processing.flushAndCloseAllWindows();
processing.closeOutput();
output.put(eos);
}
// retrieve current watermark stamp
private long getCurrentWatermark() {
return scheduler.getCurrentTimestamp();
}
private long getCurrentElementTime() {
return currentElementTime;
}
}