TrevniAppender.java

/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.tajo.storage.trevni;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.common.TajoDataTypes.Type;
import org.apache.tajo.storage.FileAppender;
import org.apache.tajo.storage.TableStatistics;
import org.apache.tajo.storage.Tuple;
import org.apache.trevni.ColumnFileMetaData;
import org.apache.trevni.ColumnFileWriter;
import org.apache.trevni.ColumnMetaData;
import org.apache.trevni.ValueType;

import java.io.FileNotFoundException;
import java.io.IOException;

public class TrevniAppender extends FileAppender {
  private FileSystem fs;
  private ColumnFileWriter writer;
  private FSDataOutputStream fos;

  private TableStatistics stats = null;
  private boolean flushed = false;

  public TrevniAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException {
    super(conf, schema, meta, path);
  }

  public void init() throws IOException {
    fs = path.getFileSystem(conf);

    if (!fs.exists(path.getParent())) {
      throw new FileNotFoundException(path.toString());
    }

    fos = fs.create(path);

    ColumnMetaData [] trevniMetas =
        new ColumnMetaData[schema.size()];
    int i = 0;
    for (Column column : schema.getColumns()) {
      trevniMetas[i++] = new ColumnMetaData(column.getSimpleName(),
          getType(column.getDataType().getType()));
    }

    writer = new ColumnFileWriter(createFileMeta(), trevniMetas);

    if (enabledStats) {
      this.stats = new TableStatistics(this.schema);
    }

    super.init();
  }

  private ColumnFileMetaData createFileMeta() {
    return new ColumnFileMetaData()
        .setCodec("null")
        .setChecksum("null");
  }

  private static ValueType getType(Type type) {
    switch (type) {
      case BOOLEAN:
        return ValueType.INT;
      case BIT:
        return ValueType.INT;
      case CHAR:
        return ValueType.STRING;
      case INT2:
        return ValueType.INT;
      case INT4:
        return ValueType.INT;
      case INT8:
        return ValueType.LONG;
      case FLOAT4:
        return ValueType.FLOAT;
      case FLOAT8:
        return ValueType.DOUBLE;
      case TEXT:
        return ValueType.STRING;
      case BLOB:
        return ValueType.BYTES;
      case INET4:
        return ValueType.BYTES;
      case INET6:
        return ValueType.BYTES;
      case PROTOBUF:
        return ValueType.BYTES;
      case NULL_TYPE:
        return ValueType.NULL;
      default:
        return null;
    }
  }

  @Override
  public long getOffset() throws IOException {
    return 0;
  }

  @Override
  public void addTuple(Tuple t) throws IOException {
    Column col;
    writer.startRow();
    for (int i = 0; i < schema.size(); i++) {
      if (enabledStats) {
        stats.analyzeField(i, t.get(i));
      }

      if (!t.isNull(i)) {
        col = schema.getColumn(i);
        switch (col.getDataType().getType()) {
          case NULL_TYPE:
            break;
          case BOOLEAN:
          case BIT:
          case INT2:
          case INT4:
            writer.writeValue(t.get(i).asInt4(), i);
            break;
          case INT8:
            writer.writeValue(t.get(i).asInt8(), i);
            break;
          case FLOAT4:
            writer.writeValue(t.get(i).asFloat4(), i);
            break;
          case FLOAT8:
            writer.writeValue(t.get(i).asFloat8(), i);
            break;
          case CHAR:
          case TEXT:
            writer.writeValue(t.get(i).asChars(), i);
            break;
          case PROTOBUF:
          case BLOB:
          case INET4:
          case INET6:
            writer.writeValue(t.get(i).asByteArray(), i);
            break;

          default:
            break;
        }
      }
    }
    writer.endRow();

    // Statistical section
    if (enabledStats) {
      stats.incrementRow();
    }
  }

  @Override
  public void flush() throws IOException {
    if (!flushed) {
      writer.writeTo(fos);
      fos.flush();
      flushed = true;
    }
  }

  @Override
  public void close() throws IOException {
    flush();
    IOUtils.closeQuietly(fos);
  }

  @Override
  public TableStats getStats() {
    if (enabledStats) {
      return stats.getTableStat();
    } else {
      return null;
    }
  }
}