/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.flink.yarn; import akka.actor.ActorRef; import org.apache.flink.api.java.tuple.Tuple2; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.clusterframework.ContaineredTaskManagerParameters; import org.apache.flink.runtime.clusterframework.messages.NotifyResourceStarted; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.yarn.messages.NotifyWhenResourcesRegistered; import org.apache.flink.yarn.messages.RequestNumberOfRegisteredResources; import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.client.api.AMRMClient; import org.apache.hadoop.yarn.client.api.NMClient; import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; import org.apache.hadoop.yarn.conf.YarnConfiguration; import java.util.Comparator; import java.util.PriorityQueue; public class TestingYarnFlinkResourceManager extends YarnFlinkResourceManager { private final PriorityQueue<Tuple2<Integer, ActorRef>> waitingQueue = new PriorityQueue<>(32, new Comparator<Tuple2<Integer, ActorRef>>() { @Override public int compare(Tuple2<Integer, ActorRef> o1, Tuple2<Integer, ActorRef> o2) { return o1.f0 - o2.f0; } }); public TestingYarnFlinkResourceManager( Configuration flinkConfig, YarnConfiguration yarnConfig, LeaderRetrievalService leaderRetrievalService, String applicationMasterHostName, String webInterfaceURL, ContaineredTaskManagerParameters taskManagerParameters, ContainerLaunchContext taskManagerLaunchContext, int yarnHeartbeatIntervalMillis, int maxFailedContainers, int numInitialTaskManagers) { super(flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers); } public TestingYarnFlinkResourceManager( Configuration flinkConfig, YarnConfiguration yarnConfig, LeaderRetrievalService leaderRetrievalService, String applicationMasterHostName, String webInterfaceURL, ContaineredTaskManagerParameters taskManagerParameters, ContainerLaunchContext taskManagerLaunchContext, int yarnHeartbeatIntervalMillis, int maxFailedContainers, int numInitialTaskManagers, YarnResourceManagerCallbackHandler callbackHandler, AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient, NMClient nodeManagerClient) { super(flinkConfig, yarnConfig, leaderRetrievalService, applicationMasterHostName, webInterfaceURL, taskManagerParameters, taskManagerLaunchContext, yarnHeartbeatIntervalMillis, maxFailedContainers, numInitialTaskManagers, callbackHandler, resourceManagerClient, nodeManagerClient); } @Override protected void handleMessage(Object message) { if (message instanceof RequestNumberOfRegisteredResources) { getSender().tell(getNumberOfStartedTaskManagers(), getSelf()); } else if (message instanceof NotifyWhenResourcesRegistered) { NotifyWhenResourcesRegistered notifyMessage = (NotifyWhenResourcesRegistered) message; if (getNumberOfStartedTaskManagers() >= notifyMessage.getNumberResources()) { getSender().tell(true, getSelf()); } else { waitingQueue.offer(Tuple2.of(notifyMessage.getNumberResources(), getSender())); } } else if (message instanceof NotifyResourceStarted) { super.handleMessage(message); while (!waitingQueue.isEmpty() && waitingQueue.peek().f0 <= getNumberOfStartedTaskManagers()) { ActorRef receiver = waitingQueue.poll().f1; receiver.tell(true, getSelf()); } } else { super.handleMessage(message); } } }