/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.alibaba.jstorm.config; import backtype.storm.utils.NimbusClientWrapper; import com.alibaba.jstorm.callback.RunnableCallback; import com.alibaba.jstorm.client.ConfigExtension; import com.alibaba.jstorm.utils.JStormUtils; import com.alibaba.jstorm.utils.LoadConf; import com.google.common.collect.MapDifference; import com.google.common.collect.Maps; import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Random; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * @author Cody (weiyue.wy@alibaba-inc.com) * @since 2.1.1 */ public class SupervisorRefreshConfig extends RunnableCallback { private static final Logger LOG = LoggerFactory.getLogger(SupervisorRefreshConfig.class); /** * storm conf, excluding yarn config */ private Map stormConf = new HashMap(); private Map supervisorConf = new HashMap<>(); /** * storm yaml string, excluding yarn config */ private String stormYaml; private NimbusClientWrapper nimbusClientWrapper; private final Random random = new Random(System.currentTimeMillis()); private boolean enableSync; private final Integer refreshInterval; private final YarnConfigBlacklist yarnConfigBlacklist; /** * yarn config */ private final String retainedYarnConfig; public SupervisorRefreshConfig(Map conf) { LOG.info("init SupervisorRefreshConfig thread..."); this.yarnConfigBlacklist = YarnConfigBlacklist.getInstance(conf); try { this.enableSync = ConfigExtension.getClusterConfSyncEnabled(conf); String rawYaml = FileUtils.readFileToString(new File(LoadConf.getStormYamlPath())); this.stormYaml = JStormUtils.trimEnd(yarnConfigBlacklist.filterConfigIfNecessary(rawYaml)); this.stormConf.putAll(conf); this.stormConf.putAll(LoadConf.loadYamlFromString(this.stormYaml)); this.supervisorConf.putAll(LoadConf.loadYamlFromString(this.stormYaml)); this.retainedYarnConfig = yarnConfigBlacklist.getRetainedConfig(rawYaml); LOG.info("retained yarn config:\n============================\n{}", retainedYarnConfig); } catch (IOException ex) { LOG.error("failed to read local storm.yaml!", ex); throw new RuntimeException(ex); } // check nimbus config every 20 ~ 30 sec this.refreshInterval = random.nextInt(10) + 20; LOG.info("done."); } @Override public void run() { try { if (!enableSync) { return; } if (this.nimbusClientWrapper == null) { this.nimbusClientWrapper = new NimbusClientWrapper(); try { this.nimbusClientWrapper.init(this.stormConf); } catch (Exception ex) { LOG.error("init nimbus client wrapper error, maybe nimbus is not alive."); } } String nimbusYaml = JStormUtils.trimEnd(yarnConfigBlacklist.filterConfigIfNecessary( this.nimbusClientWrapper.getClient().getStormRawConf())); Map nimbusConf = LoadConf.loadYamlFromString(nimbusYaml); if (nimbusYaml != null && !this.supervisorConf.equals(nimbusConf)) { Map newConf = LoadConf.loadYamlFromString(nimbusYaml); if (newConf == null) { LOG.error("received invalid storm.yaml, skip..."); } else { MapDifference<?, ?> diff = Maps.difference(this.supervisorConf, nimbusConf); LOG.debug("conf diff, left only:{}, right only:{}", diff.entriesOnlyOnLeft(), diff.entriesOnlyOnRight()); LOG.debug("received nimbus config update, new config:\n{}", nimbusYaml); this.stormYaml = nimbusYaml; // append yarn config nimbusYaml = JStormUtils.trimEnd(nimbusYaml + "\n" + retainedYarnConfig); // backup config & overwrite current storm.yaml RefreshableComponents.refresh(newConf); LoadConf.backupAndOverwriteStormYaml(nimbusYaml); } } } catch (Exception ex) { LOG.error("failed to get nimbus conf, maybe nimbus is not alive."); this.nimbusClientWrapper.reconnect(); } } @Override public Object getResult() { return refreshInterval; } }