/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.streaming.runtime.operators.windowing;
import static java.util.Objects.requireNonNull;
import org.apache.commons.math3.util.ArithmeticUtils;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.functions.Function;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.functions.KeySelector;
import org.apache.flink.core.fs.FSDataInputStream;
import org.apache.flink.core.fs.FSDataOutputStream;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.streaming.api.operators.AbstractUdfStreamOperator;
import org.apache.flink.streaming.api.operators.OneInputStreamOperator;
import org.apache.flink.streaming.api.operators.TimestampedCollector;
import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
import org.apache.flink.streaming.runtime.tasks.ProcessingTimeCallback;
import org.apache.flink.util.MathUtils;
/**
* Base class for special window operator implementation for windows that fire at the same time for
* all keys.
*
* @deprecated Deprecated in favour of the generic {@link WindowOperator}. This was an
* optimized implementation used for aligned windows.
*/
@Internal
@Deprecated
public abstract class AbstractAlignedProcessingTimeWindowOperator<KEY, IN, OUT, STATE, F extends Function>
extends AbstractUdfStreamOperator<OUT, F>
implements OneInputStreamOperator<IN, OUT>, ProcessingTimeCallback {
private static final long serialVersionUID = 3245500864882459867L;
private static final long MIN_SLIDE_TIME = 50;
// ----- fields for operator parametrization -----
private final Function function;
private final KeySelector<IN, KEY> keySelector;
private final TypeSerializer<KEY> keySerializer;
private final TypeSerializer<STATE> stateTypeSerializer;
private final long windowSize;
private final long windowSlide;
private final long paneSize;
private final int numPanesPerWindow;
// ----- fields for operator functionality -----
private transient AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes;
private transient TimestampedCollector<OUT> out;
private transient RestoredState<IN, KEY, STATE, OUT> restoredState;
private transient long nextEvaluationTime;
private transient long nextSlideTime;
protected AbstractAlignedProcessingTimeWindowOperator(
F function,
KeySelector<IN, KEY> keySelector,
TypeSerializer<KEY> keySerializer,
TypeSerializer<STATE> stateTypeSerializer,
long windowLength,
long windowSlide) {
super(function);
if (windowLength < MIN_SLIDE_TIME) {
throw new IllegalArgumentException("Window length must be at least " + MIN_SLIDE_TIME + " msecs");
}
if (windowSlide < MIN_SLIDE_TIME) {
throw new IllegalArgumentException("Window slide must be at least " + MIN_SLIDE_TIME + " msecs");
}
if (windowLength < windowSlide) {
throw new IllegalArgumentException("The window size must be larger than the window slide");
}
final long paneSlide = ArithmeticUtils.gcd(windowLength, windowSlide);
if (paneSlide < MIN_SLIDE_TIME) {
throw new IllegalArgumentException(String.format(
"Cannot compute window of size %d msecs sliding by %d msecs. " +
"The unit of grouping is too small: %d msecs", windowLength, windowSlide, paneSlide));
}
this.function = requireNonNull(function);
this.keySelector = requireNonNull(keySelector);
this.keySerializer = requireNonNull(keySerializer);
this.stateTypeSerializer = requireNonNull(stateTypeSerializer);
this.windowSize = windowLength;
this.windowSlide = windowSlide;
this.paneSize = paneSlide;
this.numPanesPerWindow = MathUtils.checkedDownCast(windowLength / paneSlide);
}
protected abstract AbstractKeyedTimePanes<IN, KEY, STATE, OUT> createPanes(
KeySelector<IN, KEY> keySelector, Function function);
// ------------------------------------------------------------------------
// startup and shutdown
// ------------------------------------------------------------------------
@Override
public void open() throws Exception {
super.open();
out = new TimestampedCollector<>(output);
// decide when to first compute the window and when to slide it
// the values should align with the start of time (that is, the UNIX epoch, not the big bang)
final long now = getProcessingTimeService().getCurrentProcessingTime();
nextEvaluationTime = now + windowSlide - (now % windowSlide);
nextSlideTime = now + paneSize - (now % paneSize);
final long firstTriggerTime = Math.min(nextEvaluationTime, nextSlideTime);
// check if we restored state and if we need to fire some windows based on that restored state
if (restoredState == null) {
// initial empty state: create empty panes that gather the elements per slide
panes = createPanes(keySelector, function);
}
else {
// restored state
panes = restoredState.panes;
long nextPastEvaluationTime = restoredState.nextEvaluationTime;
long nextPastSlideTime = restoredState.nextSlideTime;
long nextPastTriggerTime = Math.min(nextPastEvaluationTime, nextPastSlideTime);
int numPanesRestored = panes.getNumPanes();
// fire windows from the past as long as there are more panes with data and as long
// as the missed trigger times have not caught up with the presence
while (numPanesRestored > 0 && nextPastTriggerTime < firstTriggerTime) {
// evaluate the window from the past
if (nextPastTriggerTime == nextPastEvaluationTime) {
computeWindow(nextPastTriggerTime);
nextPastEvaluationTime += windowSlide;
}
// evaluate slide from the past
if (nextPastTriggerTime == nextPastSlideTime) {
panes.slidePanes(numPanesPerWindow);
numPanesRestored--;
nextPastSlideTime += paneSize;
}
nextPastTriggerTime = Math.min(nextPastEvaluationTime, nextPastSlideTime);
}
}
// make sure the first window happens
getProcessingTimeService().registerTimer(firstTriggerTime, this);
}
@Override
public void close() throws Exception {
super.close();
// early stop the triggering thread, so it does not attempt to return any more data
stopTriggers();
}
@Override
public void dispose() throws Exception {
super.dispose();
// acquire the lock during shutdown, to prevent trigger calls at the same time
// fail-safe stop of the triggering thread (in case of an error)
stopTriggers();
// release the panes. panes may still be null if dispose is called
// after open() failed
if (panes != null) {
panes.dispose();
}
}
private void stopTriggers() {
// reset the action timestamps. this makes sure any pending triggers will not evaluate
nextEvaluationTime = -1L;
nextSlideTime = -1L;
}
// ------------------------------------------------------------------------
// Receiving elements and triggers
// ------------------------------------------------------------------------
@Override
public void processElement(StreamRecord<IN> element) throws Exception {
panes.addElementToLatestPane(element.getValue());
}
@Override
public void onProcessingTime(long timestamp) throws Exception {
// first we check if we actually trigger the window function
if (timestamp == nextEvaluationTime) {
// compute and output the results
computeWindow(timestamp);
nextEvaluationTime += windowSlide;
}
// check if we slide the panes by one. this may happen in addition to the
// window computation, or just by itself
if (timestamp == nextSlideTime) {
panes.slidePanes(numPanesPerWindow);
nextSlideTime += paneSize;
}
long nextTriggerTime = Math.min(nextEvaluationTime, nextSlideTime);
getProcessingTimeService().registerTimer(nextTriggerTime, this);
}
private void computeWindow(long timestamp) throws Exception {
out.setAbsoluteTimestamp(timestamp);
panes.truncatePanes(numPanesPerWindow);
panes.evaluateWindow(out, new TimeWindow(timestamp - windowSize, timestamp), this);
}
// ------------------------------------------------------------------------
// Checkpointing
// ------------------------------------------------------------------------
@Override
public void snapshotState(FSDataOutputStream out, long checkpointId, long timestamp) throws Exception {
super.snapshotState(out, checkpointId, timestamp);
// we write the panes with the key/value maps into the stream, as well as when this state
// should have triggered and slided
DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out);
outView.writeLong(nextEvaluationTime);
outView.writeLong(nextSlideTime);
panes.writeToOutput(outView, keySerializer, stateTypeSerializer);
outView.flush();
}
@Override
public void restoreState(FSDataInputStream in) throws Exception {
super.restoreState(in);
DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(in);
final long nextEvaluationTime = inView.readLong();
final long nextSlideTime = inView.readLong();
AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes = createPanes(keySelector, function);
panes.readFromInput(inView, keySerializer, stateTypeSerializer);
restoredState = new RestoredState<>(panes, nextEvaluationTime, nextSlideTime);
}
// ------------------------------------------------------------------------
// Property access (for testing)
// ------------------------------------------------------------------------
public long getWindowSize() {
return windowSize;
}
public long getWindowSlide() {
return windowSlide;
}
public long getPaneSize() {
return paneSize;
}
public int getNumPanesPerWindow() {
return numPanesPerWindow;
}
public long getNextEvaluationTime() {
return nextEvaluationTime;
}
public long getNextSlideTime() {
return nextSlideTime;
}
// ------------------------------------------------------------------------
// Utilities
// ------------------------------------------------------------------------
@Override
public String toString() {
return "Window (processing time) (length=" + windowSize + ", slide=" + windowSlide + ')';
}
// ------------------------------------------------------------------------
// ------------------------------------------------------------------------
private static final class RestoredState<IN, KEY, STATE, OUT> {
final AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes;
final long nextEvaluationTime;
final long nextSlideTime;
RestoredState(AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes, long nextEvaluationTime, long nextSlideTime) {
this.panes = panes;
this.nextEvaluationTime = nextEvaluationTime;
this.nextSlideTime = nextSlideTime;
}
}
}