/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.controller.validation; import java.io.File; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.base.Preconditions; import com.linkedin.pinot.common.config.AbstractTableConfig; import com.linkedin.pinot.common.config.QuotaConfig; import com.linkedin.pinot.controller.api.restlet.resources.TableSizeReader; import javax.annotation.Nonnegative; import javax.annotation.Nonnull; /** * Class to check if a new segment is within the configured storage quota for the table * */ public class StorageQuotaChecker { private static final Logger LOGGER = LoggerFactory.getLogger(StorageQuotaChecker.class); private final TableSizeReader tableSizeReader; private final AbstractTableConfig tableConfig; public StorageQuotaChecker(AbstractTableConfig tableConfig, TableSizeReader tableSizeReader) { this.tableConfig = tableConfig; this.tableSizeReader = tableSizeReader; } public class QuotaCheckerResponse { public boolean isSegmentWithinQuota; public String reason; QuotaCheckerResponse(boolean isSegmentWithinQuota, String reason) { this.isSegmentWithinQuota = isSegmentWithinQuota; this.reason = reason; } } /** * check if the segment represented by segmentFile is within the storage quota * @param segmentFile untarred segment. This should not be null. * segmentFile must exist on disk and must be a directory * @param tableNameWithType table name without type (OFFLINE/REALTIME) information * @param segmentName name of the segment being added * @param timeoutMsec timeout in milliseconds for reading table sizes from server * */ public QuotaCheckerResponse isSegmentStorageWithinQuota(@Nonnull File segmentFile, @Nonnull String tableNameWithType, @Nonnull String segmentName, @Nonnegative int timeoutMsec) { Preconditions.checkNotNull(segmentFile); Preconditions.checkNotNull(tableNameWithType); Preconditions.checkNotNull(segmentName); Preconditions.checkArgument(timeoutMsec > 0, "Timeout value must be > 0, input: %s", timeoutMsec); Preconditions.checkArgument(segmentFile.exists(), "Segment file: %s does not exist", segmentFile); Preconditions.checkArgument(segmentFile.isDirectory(), "Segment file: %s is not a directory", segmentFile); // 1. Read table config // 2. read table size from all the servers // 3. update predicted segment sizes // 4. is the updated size within quota QuotaConfig quotaConfig = tableConfig.getQuotaConfig(); int numReplicas = tableConfig.getValidationConfig().getReplicationNumber(); final String tableName = tableConfig.getTableName(); if (quotaConfig == null) { // no quota configuration...so ignore for backwards compatibility return new QuotaCheckerResponse(true, "Quota configuration not set for table: " +tableNameWithType); } long allowedStorageBytes = numReplicas * quotaConfig.storageSizeBytes(); if (allowedStorageBytes < 0) { return new QuotaCheckerResponse(true, "Storage quota is not configured for table: " + tableNameWithType); } long incomingSegmentSizeBytes = FileUtils.sizeOfDirectory(segmentFile); // read table size TableSizeReader.TableSubTypeSizeDetails tableSubtypeSize = tableSizeReader.getTableSubtypeSize(tableNameWithType, timeoutMsec); // If the segment exists(refresh), get the existing size TableSizeReader.SegmentSizeDetails sizeDetails = tableSubtypeSize.segments.get(segmentName); long existingSegmentSizeBytes = sizeDetails != null ? sizeDetails.estimatedSizeInBytes : 0; long estimatedFinalSizeBytes = tableSubtypeSize.estimatedSizeInBytes - existingSegmentSizeBytes + incomingSegmentSizeBytes; if (estimatedFinalSizeBytes <= allowedStorageBytes) { return new QuotaCheckerResponse(true, String.format("Estimated size: %d bytes is within the configured quota of %d (bytes) for table %s. Incoming segment size: %d (bytes)", estimatedFinalSizeBytes, allowedStorageBytes, tableName, incomingSegmentSizeBytes) ); } else { return new QuotaCheckerResponse(false, String.format("Estimated size: %d bytes exceeds the configured quota of %d (bytes) for table %s. Incoming segment size: %d (bytes)", estimatedFinalSizeBytes, allowedStorageBytes, tableName, incomingSegmentSizeBytes)); } } }