/* * Copyright 2013 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.springframework.yarn.test.support; import java.io.File; import java.io.IOException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; import org.springframework.yarn.test.context.YarnCluster; /** * Standalone simple mini cluster having Yarn * and Hdfs nodes. * * @author Janne Valkealahti * */ public class StandaloneYarnCluster implements YarnCluster { private final static Log log = LogFactory.getLog(StandaloneYarnCluster.class); /** Yarn specific mini cluster */ private MiniYARNCluster yarnCluster = null; /** Hdfs specific mini cluster */ private MiniDFSCluster dfsCluster = null; /** Unique cluster name */ private final String clusterName; /** Configuration build at runtime */ private Configuration configuration; /** Monitor sync for start and stop */ private final Object startupShutdownMonitor = new Object(); /** Flag for cluster state */ private boolean started; /** Number of nodes for yarn and dfs */ private int nodes = 1; /** * Instantiates a mini cluster with default * cluster node count. * * @param clusterName the unique cluster name */ public StandaloneYarnCluster(String clusterName) { this.clusterName = clusterName; } /** * Instantiates a mini cluster with given * cluster node count. * * @param clusterName the unique cluster name * @param nodes the node count */ public StandaloneYarnCluster(String clusterName, int nodes) { this.clusterName = clusterName; this.nodes = nodes; } @Override public Configuration getConfiguration() { return configuration; } @Override public void start() throws IOException { log.info("Checking if cluster=" + clusterName + " needs to be started"); synchronized (this.startupShutdownMonitor) { if (started) { return; } // SHDP-309 hack to set hadoop.security.token.service.use_ip // for what we want because 2.3.0 added core-site.xml in // jar container MiniYARNCluster which sets this value to false // while default is true. // we try to add new default resource which should override // one from a core-site.xml Configuration.addDefaultResource("shdp-site.xml"); log.info("Starting cluster=" + clusterName); configuration = new YarnConfiguration(); configuration.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, "target/" + clusterName + "-dfs"); dfsCluster = new MiniDFSCluster.Builder(configuration). numDataNodes(nodes). build(); yarnCluster = new MiniYARNCluster(clusterName, nodes, 1, 1); yarnCluster.init(configuration); yarnCluster.start(); log.info("Started cluster=" + clusterName); started = true; } } @Override public void stop() { log.info("Checking if cluster=" + clusterName + " needs to be stopped"); synchronized (this.startupShutdownMonitor) { if (!started) { return; } if (yarnCluster != null) { yarnCluster.stop(); yarnCluster = null; } if (dfsCluster != null) { dfsCluster.shutdown(); dfsCluster = null; } log.info("Stopped cluster=" + clusterName); started = false; } } @Override public File getYarnWorkDir() { return yarnCluster != null ? yarnCluster.getTestWorkDir() : null; } /** * Sets a number of nodes for cluster. Every node * will act as yarn and dfs role. Default is one node. * * @param nodes the number of nodes */ public void setNodes(int nodes) { this.nodes = nodes; } }