/**
* Copyright 2010 The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.master;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.NavigableSet;
import java.util.Random;
import java.util.TreeMap;
import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HServerAddress;
import org.apache.hadoop.hbase.HServerInfo;
/**
* Makes decisions about the placement and movement of Regions across
* RegionServers.
*
* <p>Cluster-wide load balancing will occur only when there are no regions in
* transition and according to a fixed period of a time using {@link #balanceCluster(Map)}.
*
* <p>Inline region placement with {@link #immediateAssignment} can be used when
* the Master needs to handle closed regions that it currently does not have
* a destination set for. This can happen during master failover.
*
* <p>On cluster startup, bulk assignment can be used to determine
* locations for all Regions in a cluster.
*
* <p>This classes produces plans for the {@link AssignmentManager} to execute.
*/
public class LoadBalancer {
private static final Log LOG = LogFactory.getLog(LoadBalancer.class);
private static final Random rand = new Random();
/**
* Generate a global load balancing plan according to the specified map of
* server information to the most loaded regions of each server.
*
* The load balancing invariant is that all servers are within 1 region of the
* average number of regions per server. If the average is an integer number,
* all servers will be balanced to the average. Otherwise, all servers will
* have either floor(average) or ceiling(average) regions.
*
* The algorithm is currently implemented as such:
*
* <ol>
* <li>Determine the two valid numbers of regions each server should have,
* <b>MIN</b>=floor(average) and <b>MAX</b>=ceiling(average).
*
* <li>Iterate down the most loaded servers, shedding regions from each so
* each server hosts exactly <b>MAX</b> regions. Stop once you reach a
* server that already has <= <b>MAX</b> regions.
*
* <li>Iterate down the least loaded servers, assigning regions so each server
* has exactly </b>MIN</b> regions. Stop once you reach a server that
* already has >= <b>MIN</b> regions.
*
* Regions being assigned to underloaded servers are those that were shed
* in the previous step. It is possible that there were not enough
* regions shed to fill each underloaded server to <b>MIN</b>. If so we
* end up with a number of regions required to do so, <b>neededRegions</b>.
*
* It is also possible that we were able fill each underloaded but ended
* up with regions that were unassigned from overloaded servers but that
* still do not have assignment.
*
* If neither of these conditions hold (no regions needed to fill the
* underloaded servers, no regions leftover from overloaded servers),
* we are done and return. Otherwise we handle these cases below.
*
* <li>If <b>neededRegions</b> is non-zero (still have underloaded servers),
* we iterate the most loaded servers again, shedding a single server from
* each (this brings them from having <b>MAX</b> regions to having
* <b>MIN</b> regions).
*
* <li>We now definitely have more regions that need assignment, either from
* the previous step or from the original shedding from overloaded servers.
*
* Iterate the least loaded servers filling each to <b>MIN</b>.
*
* <li>If we still have more regions that need assignment, again iterate the
* least loaded servers, this time giving each one (filling them to
* </b>MAX</b>) until we run out.
*
* <li>All servers will now either host <b>MIN</b> or <b>MAX</b> regions.
*
* In addition, any server hosting >= <b>MAX</b> regions is guaranteed
* to end up with <b>MAX</b> regions at the end of the balancing. This
* ensures the minimal number of regions possible are moved.
* </ol>
*
* TODO: We can at-most reassign the number of regions away from a particular
* server to be how many they report as most loaded.
* Should we just keep all assignment in memory? Any objections?
* Does this mean we need HeapSize on HMaster? Or just careful monitor?
* (current thinking is we will hold all assignments in memory)
*
* @param clusterState Map of regionservers and their load/region information to
* a list of their most loaded regions
* @return a list of regions to be moved, including source and destination,
* or null if cluster is already balanced
*/
public List<RegionPlan> balanceCluster(
Map<HServerInfo,List<HRegionInfo>> clusterState) {
long startTime = System.currentTimeMillis();
// Make a map sorted by load and count regions
TreeMap<HServerInfo,List<HRegionInfo>> serversByLoad =
new TreeMap<HServerInfo,List<HRegionInfo>>(
new HServerInfo.LoadComparator());
int numServers = clusterState.size();
if (numServers == 0) {
LOG.debug("numServers=0 so skipping load balancing");
return null;
}
int numRegions = 0;
// Iterate so we can count regions as we build the map
for(Map.Entry<HServerInfo, List<HRegionInfo>> server:
clusterState.entrySet()) {
server.getKey().getLoad().setNumberOfRegions(server.getValue().size());
numRegions += server.getKey().getLoad().getNumberOfRegions();
serversByLoad.put(server.getKey(), server.getValue());
}
// Check if we even need to do any load balancing
float average = (float)numRegions / numServers; // for logging
int min = numRegions / numServers;
int max = numRegions % numServers == 0 ? min : min + 1;
if(serversByLoad.lastKey().getLoad().getNumberOfRegions() <= max &&
serversByLoad.firstKey().getLoad().getNumberOfRegions() >= min) {
// Skipped because no server outside (min,max) range
LOG.info("Skipping load balancing. servers=" + numServers + " " +
"regions=" + numRegions + " average=" + average + " " +
"mostloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions() +
" leastloaded=" + serversByLoad.lastKey().getLoad().getNumberOfRegions());
return null;
}
// Balance the cluster
// TODO: Look at data block locality or a more complex load to do this
List<RegionPlan> regionsToMove = new ArrayList<RegionPlan>();
int regionidx = 0; // track the index in above list for setting destination
// Walk down most loaded, pruning each to the max
int serversOverloaded = 0;
Map<HServerInfo,BalanceInfo> serverBalanceInfo =
new TreeMap<HServerInfo,BalanceInfo>();
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
serversByLoad.descendingMap().entrySet()) {
HServerInfo serverInfo = server.getKey();
int regionCount = serverInfo.getLoad().getNumberOfRegions();
if(regionCount <= max) {
serverBalanceInfo.put(serverInfo, new BalanceInfo(0, 0));
break;
}
serversOverloaded++;
List<HRegionInfo> regions = server.getValue();
int numToOffload = Math.min(regionCount - max, regions.size());
int numTaken = 0;
for (HRegionInfo hri: regions) {
// Don't rebalance meta regions.
if (hri.isMetaRegion()) continue;
regionsToMove.add(new RegionPlan(hri, serverInfo, null));
numTaken++;
if (numTaken >= numToOffload) break;
}
serverBalanceInfo.put(serverInfo,
new BalanceInfo(numToOffload, (-1)*numTaken));
}
// Walk down least loaded, filling each to the min
int serversUnderloaded = 0; // number of servers that get new regions
int neededRegions = 0; // number of regions needed to bring all up to min
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
serversByLoad.entrySet()) {
int regionCount = server.getKey().getLoad().getNumberOfRegions();
if(regionCount >= min) {
break;
}
serversUnderloaded++;
int numToTake = min - regionCount;
int numTaken = 0;
while(numTaken < numToTake && regionidx < regionsToMove.size()) {
regionsToMove.get(regionidx).setDestination(server.getKey());
numTaken++;
regionidx++;
}
serverBalanceInfo.put(server.getKey(), new BalanceInfo(0, numTaken));
// If we still want to take some, increment needed
if(numTaken < numToTake) {
neededRegions += (numToTake - numTaken);
}
}
// If none needed to fill all to min and none left to drain all to max,
// we are done
if(neededRegions == 0 && regionidx == regionsToMove.size()) {
long endTime = System.currentTimeMillis();
LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
"Moving " + regionsToMove.size() + " regions off of " +
serversOverloaded + " overloaded servers onto " +
serversUnderloaded + " less loaded servers");
return regionsToMove;
}
// Need to do a second pass.
// Either more regions to assign out or servers that are still underloaded
// If we need more to fill min, grab one from each most loaded until enough
if (neededRegions != 0) {
// Walk down most loaded, grabbing one from each until we get enough
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
serversByLoad.descendingMap().entrySet()) {
BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
int idx =
balanceInfo == null ? 0 : balanceInfo.getNextRegionForUnload();
if (idx >= server.getValue().size()) break;
HRegionInfo region = server.getValue().get(idx);
if (region.isMetaRegion()) continue; // Don't move meta regions.
regionsToMove.add(new RegionPlan(region, server.getKey(), null));
if(--neededRegions == 0) {
// No more regions needed, done shedding
break;
}
}
}
// Now we have a set of regions that must be all assigned out
// Assign each underloaded up to the min, then if leftovers, assign to max
// Walk down least loaded, assigning to each to fill up to min
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
serversByLoad.entrySet()) {
int regionCount = server.getKey().getLoad().getNumberOfRegions();
if (regionCount >= min) break;
BalanceInfo balanceInfo = serverBalanceInfo.get(server.getKey());
if(balanceInfo != null) {
regionCount += balanceInfo.getNumRegionsAdded();
}
if(regionCount >= min) {
continue;
}
int numToTake = min - regionCount;
int numTaken = 0;
while(numTaken < numToTake && regionidx < regionsToMove.size()) {
regionsToMove.get(regionidx).setDestination(server.getKey());
numTaken++;
regionidx++;
}
}
// If we still have regions to dish out, assign underloaded to max
if(regionidx != regionsToMove.size()) {
for(Map.Entry<HServerInfo, List<HRegionInfo>> server :
serversByLoad.entrySet()) {
int regionCount = server.getKey().getLoad().getNumberOfRegions();
if(regionCount >= max) {
break;
}
regionsToMove.get(regionidx).setDestination(server.getKey());
regionidx++;
if(regionidx == regionsToMove.size()) {
break;
}
}
}
long endTime = System.currentTimeMillis();
if (regionidx != regionsToMove.size() || neededRegions != 0) {
// Emit data so can diagnose how balancer went astray.
LOG.warn("regionidx=" + regionidx + ", regionsToMove=" + regionsToMove.size() +
", numServers=" + numServers + ", serversOverloaded=" + serversOverloaded +
", serversUnderloaded=" + serversUnderloaded);
StringBuilder sb = new StringBuilder();
for (Map.Entry<HServerInfo, List<HRegionInfo>> e: clusterState.entrySet()) {
if (sb.length() > 0) sb.append(", ");
sb.append(e.getKey().getServerName());
sb.append(" ");
sb.append(e.getValue().size());
}
LOG.warn("Input " + sb.toString());
}
// All done!
LOG.info("Calculated a load balance in " + (endTime-startTime) + "ms. " +
"Moving " + regionsToMove.size() + " regions off of " +
serversOverloaded + " overloaded servers onto " +
serversUnderloaded + " less loaded servers");
return regionsToMove;
}
/**
* Stores additional per-server information about the regions added/removed
* during the run of the balancing algorithm.
*
* For servers that receive additional regions, we are not updating the number
* of regions in HServerInfo once we decide to reassign regions to a server,
* but we need this information later in the algorithm. This is stored in
* <b>numRegionsAdded</b>.
*
* For servers that shed regions, we need to track which regions we have
* already shed. <b>nextRegionForUnload</b> contains the index in the list
* of regions on the server that is the next to be shed.
*/
private static class BalanceInfo {
private final int nextRegionForUnload;
private final int numRegionsAdded;
public BalanceInfo(int nextRegionForUnload, int numRegionsAdded) {
this.nextRegionForUnload = nextRegionForUnload;
this.numRegionsAdded = numRegionsAdded;
}
public int getNextRegionForUnload() {
return nextRegionForUnload;
}
public int getNumRegionsAdded() {
return numRegionsAdded;
}
}
/**
* Generates a bulk assignment plan to be used on cluster startup using a
* simple round-robin assignment.
* <p>
* Takes a list of all the regions and all the servers in the cluster and
* returns a map of each server to the regions that it should be assigned.
* <p>
* Currently implemented as a round-robin assignment. Same invariant as
* load balancing, all servers holding floor(avg) or ceiling(avg).
*
* TODO: Use block locations from HDFS to place regions with their blocks
*
* @param regions all regions
* @param servers all servers
* @return map of server to the regions it should take, or null if no
* assignment is possible (ie. no regions or no servers)
*/
public static Map<HServerInfo,List<HRegionInfo>> roundRobinAssignment(
List<HRegionInfo> regions, List<HServerInfo> servers) {
if(regions.size() == 0 || servers.size() == 0) {
return null;
}
Map<HServerInfo,List<HRegionInfo>> assignments =
new TreeMap<HServerInfo,List<HRegionInfo>>();
int numRegions = regions.size();
int numServers = servers.size();
int max = (int)Math.ceil((float)numRegions/numServers);
int serverIdx = 0;
for(HServerInfo server : servers) {
List<HRegionInfo> serverRegions = new ArrayList<HRegionInfo>(max);
for(int i=serverIdx;i<regions.size();i+=numServers) {
serverRegions.add(regions.get(i));
}
assignments.put(server, serverRegions);
serverIdx++;
}
return assignments;
}
/**
* Generates a bulk assignment startup plan, attempting to reuse the existing
* assignment information from META, but adjusting for the specified list of
* available/online servers available for assignment.
* <p>
* Takes a map of all regions to their existing assignment from META. Also
* takes a list of online servers for regions to be assigned to. Attempts to
* retain all assignment, so in some instances initial assignment will not be
* completely balanced.
* <p>
* Any leftover regions without an existing server to be assigned to will be
* assigned randomly to available servers.
* @param regions regions and existing assignment from meta
* @param servers available servers
* @return map of servers and regions to be assigned to them
*/
public static Map<HServerInfo, List<HRegionInfo>> retainAssignment(
Map<HRegionInfo, HServerAddress> regions, List<HServerInfo> servers) {
Map<HServerInfo, List<HRegionInfo>> assignments =
new TreeMap<HServerInfo, List<HRegionInfo>>();
// Build a map of server addresses to server info so we can match things up
Map<HServerAddress, HServerInfo> serverMap =
new TreeMap<HServerAddress, HServerInfo>();
for (HServerInfo server : servers) {
serverMap.put(server.getServerAddress(), server);
assignments.put(server, new ArrayList<HRegionInfo>());
}
for (Map.Entry<HRegionInfo, HServerAddress> region : regions.entrySet()) {
HServerAddress hsa = region.getValue();
HServerInfo server = hsa == null? null: serverMap.get(hsa);
if (server != null) {
assignments.get(server).add(region.getKey());
} else {
assignments.get(servers.get(rand.nextInt(assignments.size()))).add(
region.getKey());
}
}
return assignments;
}
/**
* Find the block locations for all of the files for the specified region.
*
* Returns an ordered list of hosts that are hosting the blocks for this
* region. The weight of each host is the sum of the block lengths of all
* files on that host, so the first host in the list is the server which
* holds the most bytes of the given region's HFiles.
*
* TODO: Make this work. Need to figure out how to match hadoop's hostnames
* given for block locations with our HServerAddress.
* TODO: Use the right directory for the region
* TODO: Use getFileBlockLocations on the files not the directory
*
* @param fs the filesystem
* @param region region
* @return ordered list of hosts holding blocks of the specified region
* @throws IOException if any filesystem errors
*/
@SuppressWarnings("unused")
private List<String> getTopBlockLocations(FileSystem fs, HRegionInfo region)
throws IOException {
String encodedName = region.getEncodedName();
Path path = new Path("/hbase/table/" + encodedName);
FileStatus status = fs.getFileStatus(path);
BlockLocation [] blockLocations =
fs.getFileBlockLocations(status, 0, status.getLen());
Map<HostAndWeight,HostAndWeight> hostWeights =
new TreeMap<HostAndWeight,HostAndWeight>(new HostAndWeight.HostComparator());
for(BlockLocation bl : blockLocations) {
String [] hosts = bl.getHosts();
long len = bl.getLength();
for(String host : hosts) {
HostAndWeight haw = hostWeights.get(host);
if(haw == null) {
haw = new HostAndWeight(host, len);
hostWeights.put(haw, haw);
} else {
haw.addWeight(len);
}
}
}
NavigableSet<HostAndWeight> orderedHosts = new TreeSet<HostAndWeight>(
new HostAndWeight.WeightComparator());
orderedHosts.addAll(hostWeights.values());
List<String> topHosts = new ArrayList<String>(orderedHosts.size());
for(HostAndWeight haw : orderedHosts.descendingSet()) {
topHosts.add(haw.getHost());
}
return topHosts;
}
/**
* Stores the hostname and weight for that hostname.
*
* This is used when determining the physical locations of the blocks making
* up a region.
*
* To make a prioritized list of the hosts holding the most data of a region,
* this class is used to count the total weight for each host. The weight is
* currently just the size of the file.
*/
private static class HostAndWeight {
private final String host;
private long weight;
public HostAndWeight(String host, long weight) {
this.host = host;
this.weight = weight;
}
public void addWeight(long weight) {
this.weight += weight;
}
public String getHost() {
return host;
}
public long getWeight() {
return weight;
}
private static class HostComparator implements Comparator<HostAndWeight> {
@Override
public int compare(HostAndWeight l, HostAndWeight r) {
return l.getHost().compareTo(r.getHost());
}
}
private static class WeightComparator implements Comparator<HostAndWeight> {
@Override
public int compare(HostAndWeight l, HostAndWeight r) {
if(l.getWeight() == r.getWeight()) {
return l.getHost().compareTo(r.getHost());
}
return l.getWeight() < r.getWeight() ? -1 : 1;
}
}
}
/**
* Generates an immediate assignment plan to be used by a new master for
* regions in transition that do not have an already known destination.
*
* Takes a list of regions that need immediate assignment and a list of
* all available servers. Returns a map of regions to the server they
* should be assigned to.
*
* This method will return quickly and does not do any intelligent
* balancing. The goal is to make a fast decision not the best decision
* possible.
*
* Currently this is random.
*
* @param regions
* @param servers
* @return map of regions to the server it should be assigned to
*/
public static Map<HRegionInfo,HServerInfo> immediateAssignment(
List<HRegionInfo> regions, List<HServerInfo> servers) {
Map<HRegionInfo,HServerInfo> assignments =
new TreeMap<HRegionInfo,HServerInfo>();
for(HRegionInfo region : regions) {
assignments.put(region, servers.get(rand.nextInt(servers.size())));
}
return assignments;
}
public static HServerInfo randomAssignment(List<HServerInfo> servers) {
if (servers == null || servers.isEmpty()) {
LOG.warn("Wanted to do random assignment but no servers to assign to");
return null;
}
return servers.get(rand.nextInt(servers.size()));
}
/**
* Stores the plan for the move of an individual region.
*
* Contains info for the region being moved, info for the server the region
* should be moved from, and info for the server the region should be moved
* to.
*
* The comparable implementation of this class compares only the region
* information and not the source/dest server info.
*/
public static class RegionPlan implements Comparable<RegionPlan> {
private final HRegionInfo hri;
private final HServerInfo source;
private HServerInfo dest;
/**
* Instantiate a plan for a region move, moving the specified region from
* the specified source server to the specified destination server.
*
* Destination server can be instantiated as null and later set
* with {@link #setDestination(HServerInfo)}.
*
* @param hri region to be moved
* @param source regionserver region should be moved from
* @param dest regionserver region should be moved to
*/
public RegionPlan(final HRegionInfo hri, HServerInfo source, HServerInfo dest) {
this.hri = hri;
this.source = source;
this.dest = dest;
}
/**
* Set the destination server for the plan for this region.
*/
public void setDestination(HServerInfo dest) {
this.dest = dest;
}
/**
* Get the source server for the plan for this region.
* @return server info for source
*/
public HServerInfo getSource() {
return source;
}
/**
* Get the destination server for the plan for this region.
* @return server info for destination
*/
public HServerInfo getDestination() {
return dest;
}
/**
* Get the encoded region name for the region this plan is for.
* @return Encoded region name
*/
public String getRegionName() {
return this.hri.getEncodedName();
}
public HRegionInfo getRegionInfo() {
return this.hri;
}
/**
* Compare the region info.
* @param o region plan you are comparing against
*/
@Override
public int compareTo(RegionPlan o) {
return getRegionName().compareTo(o.getRegionName());
}
@Override
public String toString() {
return "hri=" + this.hri.getRegionNameAsString() + ", src=" +
(this.source == null? "": this.source.getServerName()) +
", dest=" + (this.dest == null? "": this.dest.getServerName());
}
}
}