/**
* Copyright The Apache Software Foundation
*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.util;
import com.google.common.annotations.VisibleForTesting;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
/**
* Class for determining the "size" of a class, an attempt to calculate the
* actual bytes that an object of this class will occupy in memory
*
* The core of this class is taken from the Derby project
*/
@InterfaceAudience.Private
public class ClassSize {
private static final Log LOG = LogFactory.getLog(ClassSize.class);
/** Array overhead */
public static final int ARRAY;
/** Overhead for ArrayList(0) */
public static final int ARRAYLIST;
/** Overhead for LinkedList(0) */
public static final int LINKEDLIST;
/** Overhead for a single entry in LinkedList */
public static final int LINKEDLIST_ENTRY;
/** Overhead for ByteBuffer */
public static final int BYTE_BUFFER;
/** Overhead for an Integer */
public static final int INTEGER;
/** Overhead for entry in map */
public static final int MAP_ENTRY;
/** Object overhead is minimum 2 * reference size (8 bytes on 64-bit) */
public static final int OBJECT;
/** Reference size is 8 bytes on 64-bit, 4 bytes on 32-bit */
public static final int REFERENCE;
/** String overhead */
public static final int STRING;
/** Overhead for TreeMap */
public static final int TREEMAP;
/** Overhead for ConcurrentHashMap */
public static final int CONCURRENT_HASHMAP;
/** Overhead for ConcurrentHashMap.Entry */
public static final int CONCURRENT_HASHMAP_ENTRY;
/** Overhead for ConcurrentHashMap.Segment */
public static final int CONCURRENT_HASHMAP_SEGMENT;
/** Overhead for ConcurrentSkipListMap */
public static final int CONCURRENT_SKIPLISTMAP;
/** Overhead for ConcurrentSkipListMap Entry */
public static final int CONCURRENT_SKIPLISTMAP_ENTRY;
/** Overhead for CellArrayMap */
public static final int CELL_ARRAY_MAP;
/** Overhead for Cell Array Entry */
public static final int CELL_ARRAY_MAP_ENTRY;
/** Overhead for ReentrantReadWriteLock */
public static final int REENTRANT_LOCK;
/** Overhead for AtomicLong */
public static final int ATOMIC_LONG;
/** Overhead for AtomicInteger */
public static final int ATOMIC_INTEGER;
/** Overhead for AtomicBoolean */
public static final int ATOMIC_BOOLEAN;
/** Overhead for AtomicReference */
public static final int ATOMIC_REFERENCE;
/** Overhead for CopyOnWriteArraySet */
public static final int COPYONWRITE_ARRAYSET;
/** Overhead for CopyOnWriteArrayList */
public static final int COPYONWRITE_ARRAYLIST;
/** Overhead for timerange */
public static final int TIMERANGE;
/** Overhead for TimeRangeTracker */
public static final int TIMERANGE_TRACKER;
/** Overhead for CellSkipListSet */
public static final int CELL_SET;
public static final int STORE_SERVICES;
/**
* MemoryLayout abstracts details about the JVM object layout. Default implementation is used in
* case Unsafe is not available.
*/
private static class MemoryLayout {
int headerSize() {
return 2 * oopSize();
}
int arrayHeaderSize() {
return (int) align(3 * oopSize());
}
/**
* Return the size of an "ordinary object pointer". Either 4 or 8, depending on 32/64 bit,
* and CompressedOops
*/
int oopSize() {
return is32BitJVM() ? 4 : 8;
}
/**
* Aligns a number to 8.
* @param num number to align to 8
* @return smallest number >= input that is a multiple of 8
*/
public long align(long num) {
//The 7 comes from that the alignSize is 8 which is the number of bytes
//stored and sent together
return ((num + 7) >> 3) << 3;
}
long sizeOf(byte[] b, int len) {
return align(arrayHeaderSize() + len);
}
}
/**
* UnsafeLayout uses Unsafe to guesstimate the object-layout related parameters like object header
* sizes and oop sizes
* See HBASE-15950.
*/
private static class UnsafeLayout extends MemoryLayout {
@SuppressWarnings("unused")
private static final class HeaderSize {
private byte a;
}
public UnsafeLayout() {
}
@Override
int headerSize() {
try {
return (int) UnsafeAccess.theUnsafe.objectFieldOffset(
HeaderSize.class.getDeclaredField("a"));
} catch (NoSuchFieldException | SecurityException e) {
LOG.error(e);
}
return super.headerSize();
}
@Override
int arrayHeaderSize() {
return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class);
}
@Override
@SuppressWarnings("static-access")
int oopSize() {
// Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each
// element is allocated in an Object[].
return UnsafeAccess.theUnsafe.ARRAY_OBJECT_INDEX_SCALE;
}
@Override
@SuppressWarnings("static-access")
long sizeOf(byte[] b, int len) {
return align(arrayHeaderSize() + len * UnsafeAccess.theUnsafe.ARRAY_BYTE_INDEX_SCALE);
}
}
private static MemoryLayout getMemoryLayout() {
// Have a safeguard in case Unsafe estimate is wrong. This is static context, there is
// no configuration, so we look at System property.
String enabled = System.getProperty("hbase.memorylayout.use.unsafe");
if (UnsafeAvailChecker.isAvailable() && (enabled == null || Boolean.parseBoolean(enabled))) {
LOG.debug("Using Unsafe to estimate memory layout");
return new UnsafeLayout();
}
LOG.debug("Not using Unsafe to estimate memory layout");
return new MemoryLayout();
}
private static final MemoryLayout memoryLayout = getMemoryLayout();
private static final boolean USE_UNSAFE_LAYOUT = (memoryLayout instanceof UnsafeLayout);
@VisibleForTesting
public static boolean useUnsafeLayout() {
return USE_UNSAFE_LAYOUT;
}
/**
* Method for reading the arc settings and setting overheads according
* to 32-bit or 64-bit architecture.
*/
static {
REFERENCE = memoryLayout.oopSize();
OBJECT = memoryLayout.headerSize();
ARRAY = memoryLayout.arrayHeaderSize();
ARRAYLIST = align(OBJECT + REFERENCE + (2 * Bytes.SIZEOF_INT)) + align(ARRAY);
LINKEDLIST = align(OBJECT + (2 * Bytes.SIZEOF_INT) + (2 * REFERENCE));
LINKEDLIST_ENTRY = align(OBJECT + (2 * REFERENCE));
//noinspection PointlessArithmeticExpression
BYTE_BUFFER = align(OBJECT + REFERENCE +
(5 * Bytes.SIZEOF_INT) +
(3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG) + align(ARRAY);
INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
MAP_ENTRY = align(OBJECT + 5 * REFERENCE + Bytes.SIZEOF_BOOLEAN);
TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + 7 * REFERENCE);
// STRING is different size in jdk6 and jdk7. Just use what we estimate as size rather than
// have a conditional on whether jdk7.
STRING = (int) estimateBase(String.class, false);
// CONCURRENT_HASHMAP is different size in jdk6 and jdk7; it looks like its different between
// 23.6-b03 and 23.0-b21. Just use what we estimate as size rather than have a conditional on
// whether jdk7.
CONCURRENT_HASHMAP = (int) estimateBase(ConcurrentHashMap.class, false);
CONCURRENT_HASHMAP_ENTRY = align(REFERENCE + OBJECT + (3 * REFERENCE) +
(2 * Bytes.SIZEOF_INT));
CONCURRENT_HASHMAP_SEGMENT = align(REFERENCE + OBJECT +
(3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_FLOAT + ARRAY);
// The size changes from jdk7 to jdk8, estimate the size rather than use a conditional
CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false);
// CELL_ARRAY_MAP is the size of an instance of CellArrayMap class, which extends
// CellFlatMap class. CellArrayMap object containing a ref to an Array, so
// OBJECT + REFERENCE + ARRAY
// CellFlatMap object contains two integers, one boolean and one reference to object, so
// 2*INT + BOOLEAN + REFERENCE
CELL_ARRAY_MAP = align(OBJECT + 2*Bytes.SIZEOF_INT + Bytes.SIZEOF_BOOLEAN
+ ARRAY + 2*REFERENCE);
CONCURRENT_SKIPLISTMAP_ENTRY = align(
align(OBJECT + (3 * REFERENCE)) + /* one node per entry */
align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */
// REFERENCE in the CellArrayMap all the rest is counted in KeyValue.heapSize()
CELL_ARRAY_MAP_ENTRY = align(REFERENCE);
REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE));
ATOMIC_LONG = align(OBJECT + Bytes.SIZEOF_LONG);
ATOMIC_INTEGER = align(OBJECT + Bytes.SIZEOF_INT);
ATOMIC_BOOLEAN = align(OBJECT + Bytes.SIZEOF_BOOLEAN);
ATOMIC_REFERENCE = align(OBJECT + REFERENCE);
COPYONWRITE_ARRAYSET = align(OBJECT + REFERENCE);
COPYONWRITE_ARRAYLIST = align(OBJECT + (2 * REFERENCE) + ARRAY);
TIMERANGE = align(ClassSize.OBJECT + Bytes.SIZEOF_LONG * 2 + Bytes.SIZEOF_BOOLEAN);
TIMERANGE_TRACKER = align(ClassSize.OBJECT + 2 * REFERENCE);
CELL_SET = align(OBJECT + REFERENCE);
STORE_SERVICES = align(OBJECT + REFERENCE + ATOMIC_LONG);
}
/**
* The estimate of the size of a class instance depends on whether the JVM
* uses 32 or 64 bit addresses, that is it depends on the size of an object
* reference. It is a linear function of the size of a reference, e.g.
* 24 + 5*r where r is the size of a reference (usually 4 or 8 bytes).
*
* This method returns the coefficients of the linear function, e.g. {24, 5}
* in the above example.
*
* @param cl A class whose instance size is to be estimated
* @param debug debug flag
* @return an array of 3 integers. The first integer is the size of the
* primitives, the second the number of arrays and the third the number of
* references.
*/
@SuppressWarnings("unchecked")
private static int [] getSizeCoefficients(Class cl, boolean debug) {
int primitives = 0;
int arrays = 0;
int references = 0;
int index = 0;
for ( ; null != cl; cl = cl.getSuperclass()) {
Field[] field = cl.getDeclaredFields();
if (null != field) {
for (Field aField : field) {
if (Modifier.isStatic(aField.getModifiers())) continue;
Class fieldClass = aField.getType();
if (fieldClass.isArray()) {
arrays++;
references++;
} else if (!fieldClass.isPrimitive()) {
references++;
} else {// Is simple primitive
String name = fieldClass.getName();
if (name.equals("int") || name.equals("I"))
primitives += Bytes.SIZEOF_INT;
else if (name.equals("long") || name.equals("J"))
primitives += Bytes.SIZEOF_LONG;
else if (name.equals("boolean") || name.equals("Z"))
primitives += Bytes.SIZEOF_BOOLEAN;
else if (name.equals("short") || name.equals("S"))
primitives += Bytes.SIZEOF_SHORT;
else if (name.equals("byte") || name.equals("B"))
primitives += Bytes.SIZEOF_BYTE;
else if (name.equals("char") || name.equals("C"))
primitives += Bytes.SIZEOF_CHAR;
else if (name.equals("float") || name.equals("F"))
primitives += Bytes.SIZEOF_FLOAT;
else if (name.equals("double") || name.equals("D"))
primitives += Bytes.SIZEOF_DOUBLE;
}
if (debug) {
if (LOG.isDebugEnabled()) {
LOG.debug("" + index + " " + aField.getName() + " " + aField.getType());
}
}
index++;
}
}
}
return new int [] {primitives, arrays, references};
}
/**
* Estimate the static space taken up by a class instance given the
* coefficients returned by getSizeCoefficients.
*
* @param coeff the coefficients
*
* @param debug debug flag
* @return the size estimate, in bytes
*/
private static long estimateBaseFromCoefficients(int [] coeff, boolean debug) {
long prealign_size = OBJECT + coeff[0] + coeff[2] * REFERENCE;
// Round up to a multiple of 8
long size = align(prealign_size) + align(coeff[1] * ARRAY);
if (debug) {
if (LOG.isDebugEnabled()) {
LOG.debug("Primitives=" + coeff[0] + ", arrays=" + coeff[1] +
", references=" + coeff[2] + ", refSize " + REFERENCE +
", size=" + size + ", prealign_size=" + prealign_size);
}
}
return size;
}
/**
* Estimate the static space taken up by the fields of a class. This includes
* the space taken up by by references (the pointer) but not by the referenced
* object. So the estimated size of an array field does not depend on the size
* of the array. Similarly the size of an object (reference) field does not
* depend on the object.
*
* @param cl class
* @param debug debug flag
* @return the size estimate in bytes.
*/
@SuppressWarnings("unchecked")
public static long estimateBase(Class cl, boolean debug) {
return estimateBaseFromCoefficients( getSizeCoefficients(cl, debug), debug);
}
/**
* Aligns a number to 8.
* @param num number to align to 8
* @return smallest number >= input that is a multiple of 8
*/
public static int align(int num) {
return (int)(align((long)num));
}
/**
* Aligns a number to 8.
* @param num number to align to 8
* @return smallest number >= input that is a multiple of 8
*/
public static long align(long num) {
return memoryLayout.align(num);
}
/**
* Determines if we are running in a 32-bit JVM. Some unit tests need to
* know this too.
*/
public static boolean is32BitJVM() {
final String model = System.getProperty("sun.arch.data.model");
return model != null && model.equals("32");
}
public static long sizeOf(byte[] b, int len) {
return memoryLayout.sizeOf(b, len);
}
}