/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.server.namenode;
import java.io.DataInput;
import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DeprecatedUTF8;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.ShortWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
/**
* Static utility functions for serializing various pieces of data in the correct
* format for the FSImage file.
*
* Some members are currently public for the benefit of the Offline Image Viewer
* which is located outside of this package. These members should be made
* package-protected when the OIV is refactored.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class FSImageSerialization {
// Static-only class
private FSImageSerialization() {}
/**
* In order to reduce allocation, we reuse some static objects. However, the methods
* in this class should be thread-safe since image-saving is multithreaded, so
* we need to keep the static objects in a thread-local.
*/
static private final ThreadLocal<TLData> TL_DATA =
new ThreadLocal<TLData>() {
@Override
protected TLData initialValue() {
return new TLData();
}
};
/**
* Simple container "struct" for threadlocal data.
*/
static private final class TLData {
final DeprecatedUTF8 U_STR = new DeprecatedUTF8();
final ShortWritable U_SHORT = new ShortWritable();
final LongWritable U_LONG = new LongWritable();
final FsPermission FILE_PERM = new FsPermission((short) 0);
}
// Helper function that reads in an INodeUnderConstruction
// from the input stream
//
static INodeFileUnderConstruction readINodeUnderConstruction(
DataInputStream in) throws IOException {
byte[] name = readBytes(in);
short blockReplication = in.readShort();
long modificationTime = in.readLong();
long preferredBlockSize = in.readLong();
int numBlocks = in.readInt();
BlockInfo[] blocks = new BlockInfo[numBlocks];
Block blk = new Block();
int i = 0;
for (; i < numBlocks-1; i++) {
blk.readFields(in);
blocks[i] = new BlockInfo(blk, blockReplication);
}
// last block is UNDER_CONSTRUCTION
if(numBlocks > 0) {
blk.readFields(in);
blocks[i] = new BlockInfoUnderConstruction(
blk, blockReplication, BlockUCState.UNDER_CONSTRUCTION, null);
}
PermissionStatus perm = PermissionStatus.read(in);
String clientName = readString(in);
String clientMachine = readString(in);
// These locations are not used at all
int numLocs = in.readInt();
DatanodeDescriptor[] locations = new DatanodeDescriptor[numLocs];
for (i = 0; i < numLocs; i++) {
locations[i] = new DatanodeDescriptor();
locations[i].readFields(in);
}
return new INodeFileUnderConstruction(name,
blockReplication,
modificationTime,
preferredBlockSize,
blocks,
perm,
clientName,
clientMachine,
null);
}
// Helper function that writes an INodeUnderConstruction
// into the input stream
//
static void writeINodeUnderConstruction(DataOutputStream out,
INodeFileUnderConstruction cons,
String path)
throws IOException {
writeString(path, out);
out.writeShort(cons.getReplication());
out.writeLong(cons.getModificationTime());
out.writeLong(cons.getPreferredBlockSize());
int nrBlocks = cons.getBlocks().length;
out.writeInt(nrBlocks);
for (int i = 0; i < nrBlocks; i++) {
cons.getBlocks()[i].write(out);
}
cons.getPermissionStatus().write(out);
writeString(cons.getClientName(), out);
writeString(cons.getClientMachine(), out);
out.writeInt(0); // do not store locations of last block
}
/*
* Save one inode's attributes to the image.
*/
static void saveINode2Image(INode node,
DataOutputStream out) throws IOException {
byte[] name = node.getLocalNameBytes();
out.writeShort(name.length);
out.write(name);
FsPermission filePerm = TL_DATA.get().FILE_PERM;
if (node.isDirectory()) {
out.writeShort(0); // replication
out.writeLong(node.getModificationTime());
out.writeLong(0); // access time
out.writeLong(0); // preferred block size
out.writeInt(-1); // # of blocks
out.writeLong(node.getNsQuota());
out.writeLong(node.getDsQuota());
filePerm.fromShort(node.getFsPermissionShort());
PermissionStatus.write(out, node.getUserName(),
node.getGroupName(),
filePerm);
} else if (node.isLink()) {
out.writeShort(0); // replication
out.writeLong(0); // modification time
out.writeLong(0); // access time
out.writeLong(0); // preferred block size
out.writeInt(-2); // # of blocks
Text.writeString(out, ((INodeSymlink)node).getLinkValue());
filePerm.fromShort(node.getFsPermissionShort());
PermissionStatus.write(out, node.getUserName(),
node.getGroupName(),
filePerm);
} else {
INodeFile fileINode = (INodeFile)node;
out.writeShort(fileINode.getReplication());
out.writeLong(fileINode.getModificationTime());
out.writeLong(fileINode.getAccessTime());
out.writeLong(fileINode.getPreferredBlockSize());
Block[] blocks = fileINode.getBlocks();
out.writeInt(blocks.length);
for (Block blk : blocks)
blk.write(out);
filePerm.fromShort(fileINode.getFsPermissionShort());
PermissionStatus.write(out, fileINode.getUserName(),
fileINode.getGroupName(),
filePerm);
}
}
// This should be reverted to package private once the ImageLoader
// code is moved into this package. This method should not be called
// by other code.
@SuppressWarnings("deprecation")
public static String readString(DataInputStream in) throws IOException {
DeprecatedUTF8 ustr = TL_DATA.get().U_STR;
ustr.readFields(in);
return ustr.toString();
}
static String readString_EmptyAsNull(DataInputStream in) throws IOException {
final String s = readString(in);
return s.isEmpty()? null: s;
}
@SuppressWarnings("deprecation")
static void writeString(String str, DataOutputStream out) throws IOException {
DeprecatedUTF8 ustr = TL_DATA.get().U_STR;
ustr.set(str);
ustr.write(out);
}
/** read the long value */
static long readLong(DataInputStream in) throws IOException {
LongWritable ustr = TL_DATA.get().U_LONG;
ustr.readFields(in);
return ustr.get();
}
/** write the long value */
static void writeLong(long value, DataOutputStream out) throws IOException {
LongWritable uLong = TL_DATA.get().U_LONG;
uLong.set(value);
uLong.write(out);
}
/** read short value */
static short readShort(DataInputStream in) throws IOException {
ShortWritable uShort = TL_DATA.get().U_SHORT;
uShort.readFields(in);
return uShort.get();
}
/** write short value */
static void writeShort(short value, DataOutputStream out) throws IOException {
ShortWritable uShort = TL_DATA.get().U_SHORT;
uShort.set(value);
uShort.write(out);
}
// Same comments apply for this method as for readString()
@SuppressWarnings("deprecation")
public static byte[] readBytes(DataInputStream in) throws IOException {
DeprecatedUTF8 ustr = TL_DATA.get().U_STR;
ustr.readFields(in);
int len = ustr.getLength();
byte[] bytes = new byte[len];
System.arraycopy(ustr.getBytes(), 0, bytes, 0, len);
return bytes;
}
/**
* Reading the path from the image and converting it to byte[][] directly
* this saves us an array copy and conversions to and from String
* @param in
* @return the array each element of which is a byte[] representation
* of a path component
* @throws IOException
*/
@SuppressWarnings("deprecation")
public static byte[][] readPathComponents(DataInputStream in)
throws IOException {
DeprecatedUTF8 ustr = TL_DATA.get().U_STR;
ustr.readFields(in);
return DFSUtil.bytes2byteArray(ustr.getBytes(),
ustr.getLength(), (byte) Path.SEPARATOR_CHAR);
}
/**
* DatanodeImage is used to store persistent information
* about datanodes into the fsImage.
*/
static class DatanodeImage implements Writable {
DatanodeDescriptor node = new DatanodeDescriptor();
static void skipOne(DataInput in) throws IOException {
DatanodeImage nodeImage = new DatanodeImage();
nodeImage.readFields(in);
}
/////////////////////////////////////////////////
// Writable
/////////////////////////////////////////////////
/**
* Public method that serializes the information about a
* Datanode to be stored in the fsImage.
*/
public void write(DataOutput out) throws IOException {
new DatanodeID(node).write(out);
out.writeLong(node.getCapacity());
out.writeLong(node.getRemaining());
out.writeLong(node.getLastUpdate());
out.writeInt(node.getXceiverCount());
}
/**
* Public method that reads a serialized Datanode
* from the fsImage.
*/
public void readFields(DataInput in) throws IOException {
DatanodeID id = new DatanodeID();
id.readFields(in);
long capacity = in.readLong();
long remaining = in.readLong();
long lastUpdate = in.readLong();
int xceiverCount = in.readInt();
// update the DatanodeDescriptor with the data we read in
node.updateRegInfo(id);
node.setStorageID(id.getStorageID());
node.setCapacity(capacity);
node.setRemaining(remaining);
node.setLastUpdate(lastUpdate);
node.setXceiverCount(xceiverCount);
}
}
}