/** * Copyright (C) 2014-2016 LinkedIn Corp. (pinot-core@linkedin.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.linkedin.pinot.tools; import com.linkedin.pinot.common.config.AbstractTableConfig; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.helix.PropertyPathConfig; import org.apache.helix.PropertyType; import org.apache.helix.ZNRecord; import org.apache.helix.manager.zk.ZKHelixAdmin; import org.apache.helix.manager.zk.ZNRecordSerializer; import org.apache.helix.model.IdealState; import org.apache.helix.store.zk.ZkHelixPropertyStore; import org.kohsuke.args4j.Option; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class UpdateSegmentState extends AbstractBaseCommand implements Command { private static final Logger LOGGER = LoggerFactory.getLogger(UpdateSegmentState.class); private static final String CmdName = "UpdateSegmentState"; private static final String fromState = "OFFLINE"; private static final String toState = "ONLINE"; static final String DEFAULT_ZK_ADDRESS = "localhost:2181"; static final String DEFAULT_CLUSTER_NAME = "PinotCluster"; @Option(name = "-zkAddress", required = false, metaVar = "<http>", usage = "Http address of Zookeeper.") private String _zkAddress = DEFAULT_ZK_ADDRESS; @Option(name = "-clusterName", required = false, metaVar = "<String>", usage = "Pinot cluster name.") private String _clusterName = DEFAULT_CLUSTER_NAME; @Option(name = "-tenantName", required = false, metaVar = "<string>", usage = "Name of tenant.") private String _tenantName; @Option(name = "-tableName", required = false, metaVar = "<string>", usage = "Name of the table (e.g. foo_table_OFFLINE).") private String _tableName; @Option(name = "-fix", required = false, metaVar = "<boolean>", usage = "Update IDEALSTATE values (OFFLINE->ONLINE).") private boolean _fix = false; @Option(name = "-help", required = false, help = true, aliases={"-h", "--h", "--help"}, usage = "Print this message.") private boolean _help = false; public UpdateSegmentState() { super(); } @Override public boolean getHelp() { return _help; } @Override public String getName() { return CmdName; } @Override public String toString() { String retString = CmdName + " -zkAddress " + _zkAddress + " -clusterName " + _clusterName; if (_tableName != null) { retString += " -tableName " + _tableName; } else { retString += " -tenanName " + _tenantName; } if (_fix) { retString += " -fix"; } return retString; } @Override public String description() { return "Audit the IDEALSTATE for the segments of a table (or all tables of a tenant). Optionally update segment state from OFFLINE to ONLINE"; } public UpdateSegmentState setZkAddress(String zkAddress) { _zkAddress = zkAddress; return this; } public UpdateSegmentState setClusterName(String clusterName) { _clusterName = clusterName; return this; } public UpdateSegmentState setTenantName(String tenantName) { _tenantName = tenantName; return this; } public UpdateSegmentState setTableName(String tableName) { _tableName = tableName; return this; } public UpdateSegmentState setOverwrite(boolean fix) { _fix = fix; return this; } private ZKHelixAdmin _helixAdmin; private ZkHelixPropertyStore<ZNRecord> _propertyStore; private void init() { LOGGER.info("Trying to connect to " + _zkAddress + " cluster " + _clusterName); _helixAdmin = new ZKHelixAdmin(_zkAddress); ZNRecordSerializer serializer = new ZNRecordSerializer(); String path = PropertyPathConfig.getPath(PropertyType.PROPERTYSTORE, _clusterName); _propertyStore = new ZkHelixPropertyStore<>(_zkAddress, serializer, path); } public List<String> getAllTenantTables() throws Exception { String tableConfigPath = "/CONFIGS/TABLE"; List<ZNRecord> tableConfigs = _propertyStore.getChildren(tableConfigPath, null, 0); List<String> tables = new ArrayList<>(128); for (ZNRecord znRecord : tableConfigs) { AbstractTableConfig tableConfig = AbstractTableConfig.fromZnRecord(znRecord); if (tableConfig.getTenantConfig().getServer().equals(_tenantName)) { tables.add(tableConfig.getTableName()); } } return tables; } public void fixTableIdealState(String tableName) throws Exception { IdealState idealState = _helixAdmin.getResourceIdealState(_clusterName, tableName); if (idealState == null) { LOGGER.info("No IDEALSTATE found for table " + tableName); return; } Map<String, Map<String, String>> mapFieldsIS = idealState.getRecord().getMapFields(); int nChanges = 0; for (String segment : mapFieldsIS.keySet()) { Map<String, String> mapIS = mapFieldsIS.get(segment); for (String server : mapIS.keySet()) { String state = mapIS.get(server); if (state.equals(fromState)) { if (_fix) { mapIS.put(server, toState); } else { LOGGER.info("Table:" + tableName + ",Segment:" + segment + ",Server:" + server + ":" + fromState); } nChanges++; } } } if (nChanges == 0) { LOGGER.info("No segments detected in " + fromState + " state for table " + tableName); } else { if (_fix) { LOGGER.info("Replacing IDEALSTATE for table " + tableName + " with " + nChanges + " changes"); _helixAdmin.setResourceIdealState(_clusterName, tableName, idealState); } else { LOGGER.info("Detected " + nChanges + " instances in " + fromState + " in table " + tableName); } } } @Override public boolean execute() throws Exception { if (_tableName == null && _tenantName == null) { LOGGER.error("One of -tableName or -tenantName must be specified."); return false; } if (_tableName != null && _tenantName != null) { LOGGER.error("Exactly one of -tenantName and -tableName be specified"); return false; } init(); if (_tenantName != null) { // Do this for all tenant tables LOGGER.info("Working on all tables for tenant " + _tenantName); List<String> tableNames = getAllTenantTables(); LOGGER.info("Found " + tableNames.size() + " tables for tenant " + _tenantName); if (tableNames.size() > 0) { for (String tableName : tableNames) { fixTableIdealState(tableName); } } } else { LOGGER.info("Working on table " + _tableName); fixTableIdealState(_tableName); } return true; } }