TextBSPFileOutputFormat.java

/**
 * CopyRight by Chinamobile
 * 
 * TextBSPFileOutputFormat.java
 */

package com.chinamobile.bcbsp.io;

import java.io.DataOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;

import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.io.Text;

import com.chinamobile.bcbsp.Constants;
import com.chinamobile.bcbsp.util.BSPJob;
import com.chinamobile.bcbsp.util.StaffAttemptID;

/**
 * TextBSPFileOutputFormat
 * 
 * An example that extends BSPFileOutputFormat<Text,Text>.
 * 
 * @author
 * @version 
 */
public class TextBSPFileOutputFormat extends BSPFileOutputFormat<Text, Text> {

    public static class LineRecordWriter extends RecordWriter<Text, Text> {
        private static final String utf8 = "UTF-8";
        private static final byte[] newline;
        static {
            try {
                newline = "\n".getBytes(utf8);
            } catch (UnsupportedEncodingException uee) {
                throw new IllegalArgumentException("can't find " + utf8
                        + " encoding");
            }
        }

        protected DataOutputStream out;
        private final byte[] keyValueSeparator;

        public LineRecordWriter(DataOutputStream out, String keyValueSeparator) {
            this.out = out;
            try {
                this.keyValueSeparator = keyValueSeparator.getBytes(utf8);
            } catch (UnsupportedEncodingException uee) {
                throw new IllegalArgumentException("can't find " + utf8
                        + " encoding");
            }
        }

        public LineRecordWriter(DataOutputStream out) {
            this(out, Constants.KV_SPLIT_FLAG);
        }

        /**
         * Write the object to the byte stream, handling Text as a special case.
         * 
         * @param o
         *            the object to print
         * @throws IOException
         *             if the write throws, we pass it on
         */
        private void writeObject(Object o) throws IOException {
            if (o instanceof Text) {
                Text to = ( Text ) o;
                out.write(to.getBytes(), 0, to.getLength());
            } else {
                out.write(o.toString().getBytes(utf8));
            }
        }

        @Override
        public void write(Text key, Text value) throws IOException {
            boolean nullKey = (key == null);
            boolean nullValue = (value == null);
            if (nullKey && nullValue) {
                return;
            }
            if (!nullKey) {
                writeObject(key);
            }
            if (!(nullKey || nullValue)) {
                out.write(keyValueSeparator);
            }
            if (!nullValue) {
                writeObject(value);
            }
            out.write(newline);
        }

        @Override
        public void close(BSPJob job) throws IOException {
            out.close();
        }
    }

    @Override
    public RecordWriter<Text, Text> getRecordWriter(BSPJob job,
            StaffAttemptID staffId) throws IOException, InterruptedException {
        Path file = getOutputPath(job, staffId);
        FileSystem fs = file.getFileSystem(job.getConf());
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter(fileOut);
    }

    @Override
    public RecordWriter<Text, Text> getRecordWriter(BSPJob job,
            StaffAttemptID staffId, Path writePath) throws IOException, InterruptedException {
        Path file = getOutputPath(staffId, writePath);
        FileSystem fs = file.getFileSystem(job.getConf());
        FSDataOutputStream fileOut = fs.create(file, false);
        return new LineRecordWriter(fileOut);
    }
}