/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.routing; import com.linkedin.pinot.common.config.AbstractTableConfig; import com.linkedin.pinot.common.config.TableNameBuilder; import com.linkedin.pinot.common.metadata.ZKMetadataProvider; import com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata; import com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType; import com.linkedin.pinot.common.utils.time.TimeUtils; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.helix.ZNRecord; import org.apache.helix.model.ExternalView; import org.apache.helix.store.zk.ZkHelixPropertyStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class HelixExternalViewBasedTimeBoundaryService implements TimeBoundaryService { private static final Logger LOGGER = LoggerFactory.getLogger(HelixExternalViewBasedTimeBoundaryService.class); private static final String DAYS_SINCE_EPOCH = "daysSinceEpoch"; private static final String HOURS_SINCE_EPOCH = "hoursSinceEpoch"; private static final String MINUTES_SINCE_EPOCH = "minutesSinceEpoch"; private static final String SECONDS_SINCE_EPOCH = "secondsSinceEpoch"; private final ZkHelixPropertyStore<ZNRecord> _propertyStore; private final Map<String, TimeBoundaryInfo> _timeBoundaryInfoMap = new HashMap<String, TimeBoundaryInfo>(); public HelixExternalViewBasedTimeBoundaryService(ZkHelixPropertyStore<ZNRecord> propertyStore) { _propertyStore = propertyStore; } public synchronized void updateTimeBoundaryService(ExternalView externalView) { if (_propertyStore == null) { return; } String tableName = externalView.getResourceName(); // Do nothing for realtime table. if (TableNameBuilder.getTableTypeFromTableName(tableName) == TableType.REALTIME) { return; } Set<String> offlineSegmentsServing = externalView.getPartitionSet(); if (offlineSegmentsServing.isEmpty()) { LOGGER.info("Skipping updating time boundary service for table '{}' with no offline segments.", tableName); return; } AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, tableName); String timeType = offlineTableConfig.getValidationConfig().getTimeType(); TimeUnit tableTimeUnit = getTimeUnitFromString(timeType); if (tableTimeUnit == null) { LOGGER.info("Skipping updating time boundary service for table '{}' with null timeUnit, config time type: {}.", tableName, timeType); return; } // Bulk reading all segment zk-metadata at once is more efficient than reading one at a time. List<OfflineSegmentZKMetadata> segmentZKMetadataList = ZKMetadataProvider.getOfflineSegmentZKMetadataListForTable(_propertyStore, tableName); long maxTimeValue = computeMaxSegmentEndTimeForTable(segmentZKMetadataList, tableTimeUnit); TimeBoundaryInfo timeBoundaryInfo = new TimeBoundaryInfo(); timeBoundaryInfo.setTimeColumn(offlineTableConfig.getValidationConfig().getTimeColumnName()); timeBoundaryInfo.setTimeValue(Long.toString(maxTimeValue)); _timeBoundaryInfoMap.put(tableName, timeBoundaryInfo); LOGGER.info("Updated time boundary service for table '{}', maxTime: {}", tableName, maxTimeValue); } /** * Compute maximum end time across a list of segment zk-metadata. * * @param segmentZKMetadataList List of Segment zk metadata for which to compute the max end time. * @param tableTimeUnit Time Unit for table * @return Max end time across all segments. */ private long computeMaxSegmentEndTimeForTable(List<OfflineSegmentZKMetadata> segmentZKMetadataList, TimeUnit tableTimeUnit) { long maxTimeValue = -1; for (OfflineSegmentZKMetadata metadata : segmentZKMetadataList) { long endTime = metadata.getEndTime(); if (endTime <= 0) { continue; } // Convert all segment times to table's time unit, before comparison. TimeUnit segmentTimeUnit = metadata.getTimeUnit(); if (segmentTimeUnit != null) { endTime = tableTimeUnit.convert(endTime, segmentTimeUnit); } maxTimeValue = Math.max(maxTimeValue, endTime); } return maxTimeValue; } private TimeUnit getTimeUnitFromString(String timeTypeString) { // If input data does not have a time column, no need to fire an exception. if ((timeTypeString == null) || timeTypeString.isEmpty()) { return null; } TimeUnit timeUnit = TimeUtils.timeUnitFromString(timeTypeString); // Check legacy time formats if (timeUnit == null) { if (timeTypeString.equalsIgnoreCase(DAYS_SINCE_EPOCH)) { timeUnit = TimeUnit.DAYS; } if (timeTypeString.equalsIgnoreCase(HOURS_SINCE_EPOCH)) { timeUnit = TimeUnit.HOURS; } if (timeTypeString.equalsIgnoreCase(MINUTES_SINCE_EPOCH)) { timeUnit = TimeUnit.MINUTES; } if (timeTypeString.equalsIgnoreCase(SECONDS_SINCE_EPOCH)) { timeUnit = TimeUnit.SECONDS; } } if (timeUnit == null) { throw new RuntimeException("Not supported time type for: " + timeTypeString); } return timeUnit; } @Override public void remove(String tableName) { _timeBoundaryInfoMap.remove(tableName); } @Override public TimeBoundaryInfo getTimeBoundaryInfoFor(String table) { return _timeBoundaryInfoMap.get(table); } }