/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.spi;
import com.facebook.presto.spi.block.Block;
import com.facebook.presto.spi.block.DictionaryBlock;
import com.facebook.presto.spi.block.DictionaryId;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static com.facebook.presto.spi.block.DictionaryId.randomDictionaryId;
import static java.lang.Math.min;
import static java.util.Objects.requireNonNull;
public class Page
{
private final Block[] blocks;
private final int positionCount;
private final AtomicLong sizeInBytes = new AtomicLong(-1);
private final AtomicLong retainedSizeInBytes = new AtomicLong(-1);
public Page(Block... blocks)
{
this(determinePositionCount(blocks), blocks);
}
public Page(int positionCount, Block... blocks)
{
requireNonNull(blocks, "blocks is null");
this.blocks = Arrays.copyOf(blocks, blocks.length);
this.positionCount = positionCount;
}
public int getChannelCount()
{
return blocks.length;
}
public int getPositionCount()
{
return positionCount;
}
public long getSizeInBytes()
{
long sizeInBytes = this.sizeInBytes.get();
if (sizeInBytes < 0) {
sizeInBytes = 0;
for (Block block : blocks) {
sizeInBytes += block.getSizeInBytes();
}
this.sizeInBytes.set(sizeInBytes);
}
return sizeInBytes;
}
public long getRetainedSizeInBytes()
{
long retainedSizeInBytes = this.retainedSizeInBytes.get();
if (retainedSizeInBytes < 0) {
retainedSizeInBytes = 0;
for (Block block : blocks) {
retainedSizeInBytes += block.getRetainedSizeInBytes();
}
this.retainedSizeInBytes.set(retainedSizeInBytes);
}
return retainedSizeInBytes;
}
public Block[] getBlocks()
{
return blocks.clone();
}
public Block getBlock(int channel)
{
return blocks[channel];
}
public Page getRegion(int positionOffset, int length)
{
if (positionOffset < 0 || length < 0 || positionOffset + length > positionCount) {
throw new IndexOutOfBoundsException("Invalid position " + positionOffset + " in page with " + positionCount + " positions");
}
int channelCount = getChannelCount();
Block[] slicedBlocks = new Block[channelCount];
for (int i = 0; i < channelCount; i++) {
slicedBlocks[i] = blocks[i].getRegion(positionOffset, length);
}
return new Page(length, slicedBlocks);
}
public void compact()
{
if (getRetainedSizeInBytes() <= getSizeInBytes()) {
return;
}
for (int i = 0; i < blocks.length; i++) {
Block block = blocks[i];
if (block instanceof DictionaryBlock) {
continue;
}
if (block.getSizeInBytes() < block.getRetainedSizeInBytes()) {
// Copy the block to compact its size
Block compactedBlock = block.copyRegion(0, block.getPositionCount());
blocks[i] = compactedBlock;
}
}
Map<DictionaryId, DictionaryBlockIndexes> dictionaryBlocks = getRelatedDictionaryBlocks();
for (DictionaryBlockIndexes blockIndexes : dictionaryBlocks.values()) {
List<DictionaryBlock> compactBlocks = compactRelatedBlocks(blockIndexes.getBlocks());
List<Integer> indexes = blockIndexes.getIndexes();
for (int i = 0; i < compactBlocks.size(); i++) {
blocks[indexes.get(i)] = compactBlocks.get(i);
}
}
long retainedSize = 0;
for (Block block : blocks) {
retainedSize += block.getRetainedSizeInBytes();
}
retainedSizeInBytes.set(retainedSize);
}
private Map<DictionaryId, DictionaryBlockIndexes> getRelatedDictionaryBlocks()
{
Map<DictionaryId, DictionaryBlockIndexes> relatedDictionaryBlocks = new HashMap<>();
for (int i = 0; i < blocks.length; i++) {
Block block = blocks[i];
if (block instanceof DictionaryBlock) {
DictionaryBlock dictionaryBlock = (DictionaryBlock) block;
relatedDictionaryBlocks.computeIfAbsent(dictionaryBlock.getDictionarySourceId(), id -> new DictionaryBlockIndexes())
.addBlock(dictionaryBlock, i);
}
}
return relatedDictionaryBlocks;
}
private static List<DictionaryBlock> compactRelatedBlocks(List<DictionaryBlock> blocks)
{
DictionaryBlock firstDictionaryBlock = blocks.get(0);
Block dictionary = firstDictionaryBlock.getDictionary();
int positionCount = firstDictionaryBlock.getPositionCount();
int dictionarySize = dictionary.getPositionCount();
// determine which dictionary entries are referenced and build a reindex for them
List<Integer> dictionaryPositionsToCopy = new ArrayList<>(min(dictionarySize, positionCount));
int[] remapIndex = new int[dictionarySize];
Arrays.fill(remapIndex, -1);
int newIndex = 0;
for (int i = 0; i < positionCount; i++) {
int position = firstDictionaryBlock.getId(i);
if (remapIndex[position] == -1) {
dictionaryPositionsToCopy.add(position);
remapIndex[position] = newIndex;
newIndex++;
}
}
// entire dictionary is referenced
if (dictionaryPositionsToCopy.size() == dictionarySize) {
return blocks;
}
// compact the dictionaries
int[] newIds = getNewIds(positionCount, firstDictionaryBlock, remapIndex);
List<DictionaryBlock> outputDictionaryBlocks = new ArrayList<>(blocks.size());
DictionaryId newDictionaryId = randomDictionaryId();
for (DictionaryBlock dictionaryBlock : blocks) {
if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) {
throw new IllegalArgumentException("dictionarySourceIds must be the same");
}
try {
Block compactDictionary = dictionaryBlock.getDictionary().copyPositions(dictionaryPositionsToCopy);
outputDictionaryBlocks.add(new DictionaryBlock(positionCount, compactDictionary, newIds, true, newDictionaryId));
}
catch (UnsupportedOperationException e) {
// ignore if copy positions is not supported for the dictionary
outputDictionaryBlocks.add(dictionaryBlock);
}
}
return outputDictionaryBlocks;
}
private static int[] getNewIds(int positionCount, DictionaryBlock dictionaryBlock, int[] remapIndex)
{
int[] newIds = new int[positionCount];
for (int i = 0; i < positionCount; i++) {
int newId = remapIndex[dictionaryBlock.getId(i)];
if (newId == -1) {
throw new IllegalStateException("reference to a non-existent key");
}
newIds[i] = newId;
}
return newIds;
}
/**
* Assures that all data for the block is in memory.
* <p>
* This allows streaming data sources to skip sections that are not
* accessed in a query.
*/
public void assureLoaded()
{
for (Block block : blocks) {
block.assureLoaded();
}
}
@Override
public String toString()
{
StringBuilder builder = new StringBuilder("Page{");
builder.append("positions=").append(positionCount);
builder.append(", channels=").append(getChannelCount());
builder.append('}');
builder.append("@").append(Integer.toHexString(System.identityHashCode(this)));
return builder.toString();
}
private static int determinePositionCount(Block... blocks)
{
requireNonNull(blocks, "blocks is null");
if (blocks.length == 0) {
throw new IllegalArgumentException("blocks is empty");
}
return blocks[0].getPositionCount();
}
private static class DictionaryBlockIndexes
{
private final List<DictionaryBlock> blocks = new ArrayList<>();
private final List<Integer> indexes = new ArrayList<>();
public void addBlock(DictionaryBlock block, int index)
{
blocks.add(block);
indexes.add(index);
}
public List<DictionaryBlock> getBlocks()
{
return blocks;
}
public List<Integer> getIndexes()
{
return indexes;
}
}
}