/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.core.segment.store; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.linkedin.pinot.common.segment.ReadMode; import com.linkedin.pinot.core.segment.index.SegmentMetadataImpl; import com.linkedin.pinot.core.segment.memory.PinotDataBuffer; import java.io.File; import java.io.IOException; import java.nio.channels.FileChannel; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; class FilePerIndexDirectory extends ColumnIndexDirectory { private static Logger LOGGER = LoggerFactory.getLogger(FilePerIndexDirectory.class); private Map<IndexKey, PinotDataBuffer> indexBuffers = new HashMap<>(); protected FilePerIndexDirectory(File segmentDirectory, SegmentMetadataImpl metadata, ReadMode readMode) { super(segmentDirectory, metadata, readMode); } @Override public PinotDataBuffer getDictionaryBufferFor(String column) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.DICTIONARY); return getReadBufferFor(key); } @Override public PinotDataBuffer newDictionaryBuffer(String column, int sizeBytes) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.DICTIONARY); return getWriteBufferFor(key, sizeBytes); } @Override public PinotDataBuffer getForwardIndexBufferFor(String column) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.FORWARD_INDEX); return getReadBufferFor(key); } @Override public PinotDataBuffer newForwardIndexBuffer(String column, int sizeBytes) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.FORWARD_INDEX); return getWriteBufferFor(key, sizeBytes); } @Override public PinotDataBuffer getInvertedIndexBufferFor(String column) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.INVERTED_INDEX); return getReadBufferFor(key); } @Override public PinotDataBuffer newInvertedIndexBuffer(String column, int sizeBytes) throws IOException { IndexKey key = new IndexKey(column, ColumnIndexType.INVERTED_INDEX); return getWriteBufferFor(key, sizeBytes); } @Override public boolean hasIndexFor(String column, ColumnIndexType type) { File indexFile = getFileFor(column, type); return indexFile.exists(); } @Override public void close() { for (Map.Entry<IndexKey, PinotDataBuffer> keyBuffers : indexBuffers.entrySet()) { keyBuffers.getValue().close(); } indexBuffers = null; } @Override public void removeIndex(String columnName, ColumnIndexType indexType) { File indexFile = getFileFor(columnName, indexType); indexFile.delete(); } @Override public boolean isIndexRemovalSupported() { return true; } private PinotDataBuffer getReadBufferFor(IndexKey key) throws IOException { if (indexBuffers.containsKey(key)) { return indexBuffers.get(key).duplicate(); } File filename = getFileFor(key.name, key.type); PinotDataBuffer buffer = mapForReads(filename, key.type.toString() + ".reader"); indexBuffers.put(key, buffer); return buffer.duplicate(); } private PinotDataBuffer getWriteBufferFor(IndexKey key, int sizeBytes) throws IOException { if (indexBuffers.containsKey(key)) { return indexBuffers.get(key).duplicate(); } File filename = getFileFor(key.name, key.type); PinotDataBuffer buffer = mapForWrites(filename, sizeBytes, key.type.toString() + ".writer"); indexBuffers.put(key, buffer); return buffer.duplicate(); } @VisibleForTesting File getFileFor(String column, ColumnIndexType indexType) { String filename; switch (indexType) { case DICTIONARY: filename = metadata.getDictionaryFileName(column, metadata.getVersion()); break; case FORWARD_INDEX: filename = metadata.getForwardIndexFileName(column, metadata.getVersion()); break; case INVERTED_INDEX: filename = metadata.getBitmapInvertedIndexFileName(column, metadata.getVersion()); break; default: throw new UnsupportedOperationException("Unknown index type: " + indexType.toString()); } return new File(segmentDirectory, filename); } private PinotDataBuffer mapForWrites(File file, int sizeBytes, String context) throws IOException { Preconditions.checkNotNull(file); Preconditions.checkArgument(sizeBytes >= 0 && sizeBytes < Integer.MAX_VALUE, "File size must be less than 2GB, file: " + file); Preconditions.checkState(!file.exists(), "File: " + file + " already exists"); String allocContext = allocationContext(file, context); // always mmap for writes return PinotDataBuffer.fromFile(file, 0, sizeBytes, ReadMode.mmap, FileChannel.MapMode.READ_WRITE, allocContext); } private PinotDataBuffer mapForReads(File file, String context) throws IOException { Preconditions.checkNotNull(file); Preconditions.checkNotNull(context); Preconditions.checkArgument(file.exists(), "File: " + file + " must exist"); Preconditions.checkArgument(file.isFile(), "File: " + file + " must be a regular file"); String allocationContext = allocationContext(file, context); return PinotDataBuffer .fromFile(file, readMode, FileChannel.MapMode.READ_ONLY, allocationContext); } }