/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.flink.api.common;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.flink.annotation.Internal;
import org.apache.flink.api.common.cache.DistributedCache.DistributedCacheEntry;
import org.apache.flink.api.common.operators.GenericDataSinkBase;
import org.apache.flink.api.common.operators.Operator;
import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.core.fs.Path;
import org.apache.flink.util.Visitable;
import org.apache.flink.util.Visitor;
import static org.apache.flink.util.Preconditions.checkNotNull;
import static org.apache.flink.util.Preconditions.checkArgument;
/**
* This class represents Flink programs, in the form of dataflow plans.
*
* <p>The dataflow is referenced by the data sinks, from which all connected
* operators of the data flow can be reached via backwards traversal</p>.
*/
@Internal
public class Plan implements Visitable<Operator<?>> {
/**
* A collection of all sinks in the plan. Since the plan is traversed from the sinks to the sources, this
* collection must contain all the sinks.
*/
protected final List<GenericDataSinkBase<?>> sinks = new ArrayList<>(4);
/** The name of the job. */
protected String jobName;
/** The default parallelism to use for nodes that have no explicitly specified parallelism. */
protected int defaultParallelism = ExecutionConfig.PARALLELISM_DEFAULT;
/** Hash map for files in the distributed cache: registered name to cache entry. */
protected HashMap<String, DistributedCacheEntry> cacheFile = new HashMap<>();
/** Config object for runtime execution parameters. */
protected ExecutionConfig executionConfig;
/** The ID of the Job that this dataflow plan belongs to */
private JobID jobId;
private long sessionTimeout;
// ------------------------------------------------------------------------
/**
* Creates a new dataflow plan with the given name, describing the data flow that ends at the
* given data sinks.
*
* <p>If not all of the sinks of a data flow are given to the plan, the flow might
* not be translated entirely.</p>
*
* @param sinks The collection will the sinks of the job's data flow.
* @param jobName The name to display for the job.
*/
public Plan(Collection<? extends GenericDataSinkBase<?>> sinks, String jobName) {
this(sinks, jobName, ExecutionConfig.PARALLELISM_DEFAULT);
}
/**
* Creates a new program plan with the given name and default parallelism, describing the data flow that ends
* at the given data sinks.
* <p>
* If not all of the sinks of a data flow are given to the plan, the flow might
* not be translated entirely.
*
* @param sinks The collection will the sinks of the job's data flow.
* @param jobName The name to display for the job.
* @param defaultParallelism The default parallelism for the job.
*/
public Plan(Collection<? extends GenericDataSinkBase<?>> sinks, String jobName, int defaultParallelism) {
this.sinks.addAll(sinks);
this.jobName = jobName;
this.defaultParallelism = defaultParallelism;
}
/**
* Creates a new program plan with the given name, containing initially a single data sink.
* <p>
* If not all of the sinks of a data flow are given, the flow might
* not be translated entirely, but only the parts of the flow reachable by traversing backwards
* from the given data sinks.
*
* @param sink The data sink of the data flow.
* @param jobName The name to display for the job.
*/
public Plan(GenericDataSinkBase<?> sink, String jobName) {
this(sink, jobName, ExecutionConfig.PARALLELISM_DEFAULT);
}
/**
* Creates a new program plan with the given name and default parallelism, containing initially a single data
* sink.
* <p>
* If not all of the sinks of a data flow are given, the flow might
* not be translated entirely, but only the parts of the flow reachable by traversing backwards
* from the given data sinks.
*
* @param sink The data sink of the data flow.
* @param jobName The name to display for the job.
* @param defaultParallelism The default parallelism for the job.
*/
public Plan(GenericDataSinkBase<?> sink, String jobName, int defaultParallelism) {
this(Collections.<GenericDataSinkBase<?>>singletonList(sink), jobName, defaultParallelism);
}
/**
* Creates a new program plan, describing the data flow that ends at the
* given data sinks. The display name for the job is generated using a timestamp.
* <p>
* If not all of the sinks of a data flow are given, the flow might
* not be translated entirely, but only the parts of the flow reachable by traversing backwards
* from the given data sinks.
*
* @param sinks The collection will the sinks of the data flow.
*/
public Plan(Collection<? extends GenericDataSinkBase<?>> sinks) {
this(sinks, ExecutionConfig.PARALLELISM_DEFAULT);
}
/**
* Creates a new program plan with the given default parallelism, describing the data flow that ends at the
* given data sinks. The display name for the job is generated using a timestamp.
* <p>
* If not all of the sinks of a data flow are given, the flow might
* not be translated entirely, but only the parts of the flow reachable by traversing backwards
* from the given data sinks.
*
* @param sinks The collection will the sinks of the data flow.
* @param defaultParallelism The default parallelism for the job.
*/
public Plan(Collection<? extends GenericDataSinkBase<?>> sinks, int defaultParallelism) {
this(sinks, "Flink Job at " + Calendar.getInstance().getTime(), defaultParallelism);
}
/**
* Creates a new program plan with single data sink.
* The display name for the job is generated using a timestamp.
* <p>
* If not all of the sinks of a data flow are given to the plan, the flow might
* not be translated entirely.
*
* @param sink The data sink of the data flow.
*/
public Plan(GenericDataSinkBase<?> sink) {
this(sink, ExecutionConfig.PARALLELISM_DEFAULT);
}
/**
* Creates a new program plan with single data sink and the given default parallelism.
* The display name for the job is generated using a timestamp.
* <p>
* If not all of the sinks of a data flow are given to the plan, the flow might
* not be translated entirely.
*
* @param sink The data sink of the data flow.
* @param defaultParallelism The default parallelism for the job.
*/
public Plan(GenericDataSinkBase<?> sink, int defaultParallelism) {
this(sink, "Flink Job at " + Calendar.getInstance().getTime(), defaultParallelism);
}
// ------------------------------------------------------------------------
/**
* Adds a data sink to the set of sinks in this program.
*
* @param sink The data sink to add.
*/
public void addDataSink(GenericDataSinkBase<?> sink) {
checkNotNull(jobName, "The data sink must not be null.");
if (!this.sinks.contains(sink)) {
this.sinks.add(sink);
}
}
/**
* Gets all the data sinks of this job.
*
* @return All sinks of the program.
*/
public Collection<? extends GenericDataSinkBase<?>> getDataSinks() {
return this.sinks;
}
/**
* Gets the name of this job.
*
* @return The name of the job.
*/
public String getJobName() {
return this.jobName;
}
/**
* Sets the jobName for this Plan.
*
* @param jobName The jobName to set.
*/
public void setJobName(String jobName) {
checkNotNull(jobName, "The job name must not be null.");
this.jobName = jobName;
}
/**
* Gets the ID of the job that the dataflow plan belongs to.
* If this ID is not set, then the dataflow represents its own
* independent job.
*
* @return The ID of the job that the dataflow plan belongs to.
*/
public JobID getJobId() {
return jobId;
}
/**
* Sets the ID of the job that the dataflow plan belongs to.
* If this ID is set to {@code null}, then the dataflow represents its own
* independent job.
*
* @param jobId The ID of the job that the dataflow plan belongs to.
*/
public void setJobId(JobID jobId) {
this.jobId = jobId;
}
public void setSessionTimeout(long sessionTimeout) {
this.sessionTimeout = sessionTimeout;
}
public long getSessionTimeout() {
return sessionTimeout;
}
/**
* Gets the default parallelism for this job. That degree is always used when an operator
* is not explicitly given a parallelism.
*
* @return The default parallelism for the plan.
*/
public int getDefaultParallelism() {
return this.defaultParallelism;
}
/**
* Sets the default parallelism for this plan. That degree is always used when an operator
* is not explicitly given a parallelism.
*
* @param defaultParallelism The default parallelism for the plan.
*/
public void setDefaultParallelism(int defaultParallelism) {
checkArgument(defaultParallelism >= 1 || defaultParallelism == ExecutionConfig.PARALLELISM_DEFAULT,
"The default parallelism must be positive, or ExecutionConfig.PARALLELISM_DEFAULT if the system should use the globally configured default.");
this.defaultParallelism = defaultParallelism;
}
/**
* Gets the optimizer post-pass class for this job. The post-pass typically creates utility classes
* for data types and is specific to a particular data model (record, tuple, Scala, ...)
*
* @return The name of the class implementing the optimizer post-pass.
*/
public String getPostPassClassName() {
return "org.apache.flink.optimizer.postpass.JavaApiPostPass";
}
/**
* Gets the execution config object.
*
* @return The execution config object.
*/
public ExecutionConfig getExecutionConfig() {
if(executionConfig == null) {
throw new RuntimeException("Execution config has not been set properly for this plan");
}
return executionConfig;
}
/**
* Sets the runtime config object defining execution parameters.
*
* @param executionConfig The execution config to use.
*/
public void setExecutionConfig(ExecutionConfig executionConfig) {
this.executionConfig = executionConfig;
}
// ------------------------------------------------------------------------
/**
* Traverses the job depth first from all data sinks on towards the sources.
*
* @see Visitable#accept(Visitor)
*/
@Override
public void accept(Visitor<Operator<?>> visitor) {
for (GenericDataSinkBase<?> sink : this.sinks) {
sink.accept(visitor);
}
}
/**
* register cache files in program level
* @param entry contains all relevant information
* @param name user defined name of that file
* @throws java.io.IOException
*/
public void registerCachedFile(String name, DistributedCacheEntry entry) throws IOException {
if (!this.cacheFile.containsKey(name)) {
try {
URI u = new URI(entry.filePath);
if (!u.getPath().startsWith("/")) {
u = new File(entry.filePath).toURI();
}
FileSystem fs = FileSystem.get(u);
if (fs.exists(new Path(u.getPath()))) {
this.cacheFile.put(name, new DistributedCacheEntry(u.toString(), entry.isExecutable));
} else {
throw new IOException("File " + u.toString() + " doesn't exist.");
}
} catch (URISyntaxException ex) {
throw new IOException("Invalid path: " + entry.filePath, ex);
}
} else {
throw new IOException("cache file " + name + "already exists!");
}
}
/**
* return the registered caches files
* @return Set of (name, filePath) pairs
*/
public Set<Entry<String,DistributedCacheEntry>> getCachedFiles() {
return this.cacheFile.entrySet();
}
public int getMaximumParallelism() {
MaxDopVisitor visitor = new MaxDopVisitor();
accept(visitor);
return Math.max(visitor.maxDop, this.defaultParallelism);
}
// --------------------------------------------------------------------------------------------
private static final class MaxDopVisitor implements Visitor<Operator<?>> {
private int maxDop = -1;
@Override
public boolean preVisit(Operator<?> visitable) {
this.maxDop = Math.max(this.maxDop, visitable.getParallelism());
return true;
}
@Override
public void postVisit(Operator<?> visitable) {}
}
}