/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.interpreter.remote; import java.util.*; import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.GUI; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.display.Input; import org.apache.zeppelin.interpreter.*; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterContext; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResult; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterResultMessage; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.scheduler.Scheduler; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; /** * Proxy for Interpreter instance that runs on separate process */ public class RemoteInterpreter extends Interpreter { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreter.class); private final RemoteInterpreterProcessListener remoteInterpreterProcessListener; private final ApplicationEventListener applicationEventListener; private Gson gson = new Gson(); private String interpreterRunner; private String interpreterPath; private String localRepoPath; private String className; private String sessionKey; private FormType formType; private boolean initialized; private Map<String, String> env; private int connectTimeout; private int maxPoolSize; private String host; private int port; private String userName; private Boolean isUserImpersonate; private int outputLimit = Constants.ZEPPELIN_INTERPRETER_OUTPUT_LIMIT; private String interpreterGroupName; /** * Remote interpreter and manage interpreter process */ public RemoteInterpreter(Properties property, String sessionKey, String className, String interpreterRunner, String interpreterPath, String localRepoPath, int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate, int outputLimit, String interpreterGroupName) { super(property); this.sessionKey = sessionKey; this.className = className; initialized = false; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; this.localRepoPath = localRepoPath; env = getEnvFromInterpreterProperty(property); this.connectTimeout = connectTimeout; this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; this.userName = userName; this.isUserImpersonate = isUserImpersonate; this.outputLimit = outputLimit; this.interpreterGroupName = interpreterGroupName; } /** * Connect to existing process */ public RemoteInterpreter(Properties property, String sessionKey, String className, String host, int port, String localRepoPath, int connectTimeout, int maxPoolSize, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate, int outputLimit) { super(property); this.sessionKey = sessionKey; this.className = className; initialized = false; this.host = host; this.port = port; this.localRepoPath = localRepoPath; this.connectTimeout = connectTimeout; this.maxPoolSize = maxPoolSize; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; this.userName = userName; this.isUserImpersonate = isUserImpersonate; this.outputLimit = outputLimit; } // VisibleForTesting public RemoteInterpreter(Properties property, String sessionKey, String className, String interpreterRunner, String interpreterPath, String localRepoPath, Map<String, String> env, int connectTimeout, RemoteInterpreterProcessListener remoteInterpreterProcessListener, ApplicationEventListener appListener, String userName, Boolean isUserImpersonate) { super(property); this.className = className; this.sessionKey = sessionKey; this.interpreterRunner = interpreterRunner; this.interpreterPath = interpreterPath; this.localRepoPath = localRepoPath; env.putAll(getEnvFromInterpreterProperty(property)); this.env = env; this.connectTimeout = connectTimeout; this.maxPoolSize = 10; this.remoteInterpreterProcessListener = remoteInterpreterProcessListener; this.applicationEventListener = appListener; this.userName = userName; this.isUserImpersonate = isUserImpersonate; } private Map<String, String> getEnvFromInterpreterProperty(Properties property) { Map<String, String> env = new HashMap<>(); for (Object key : property.keySet()) { if (RemoteInterpreterUtils.isEnvString((String) key)) { env.put((String) key, property.getProperty((String) key)); } } return env; } @Override public String getClassName() { return className; } private boolean connectToExistingProcess() { return host != null && port > 0; } public RemoteInterpreterProcess getInterpreterProcess() { InterpreterGroup intpGroup = getInterpreterGroup(); if (intpGroup == null) { return null; } synchronized (intpGroup) { if (intpGroup.getRemoteInterpreterProcess() == null) { RemoteInterpreterProcess remoteProcess; if (connectToExistingProcess()) { remoteProcess = new RemoteInterpreterRunningProcess( connectTimeout, remoteInterpreterProcessListener, applicationEventListener, host, port); } else { // create new remote process remoteProcess = new RemoteInterpreterManagedProcess( interpreterRunner, interpreterPath, localRepoPath, env, connectTimeout, remoteInterpreterProcessListener, applicationEventListener, interpreterGroupName); } intpGroup.setRemoteInterpreterProcess(remoteProcess); } return intpGroup.getRemoteInterpreterProcess(); } } public synchronized void init() { if (initialized == true) { return; } RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); final InterpreterGroup interpreterGroup = getInterpreterGroup(); interpreterProcess.setMaxPoolSize( Math.max(this.maxPoolSize, interpreterProcess.getMaxPoolSize())); String groupId = interpreterGroup.getId(); synchronized (interpreterProcess) { Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } boolean broken = false; try { logger.info("Create remote interpreter {}", getClassName()); if (localRepoPath != null) { property.put("zeppelin.interpreter.localRepo", localRepoPath); } property.put("zeppelin.interpreter.output.limit", Integer.toString(outputLimit)); client.createInterpreter(groupId, sessionKey, getClassName(), (Map) property, userName); // Push angular object loaded from JSON file to remote interpreter if (!interpreterGroup.isAngularRegistryPushed()) { pushAngularObjectRegistryToRemote(client); interpreterGroup.setAngularRegistryPushed(true); } } catch (TException e) { logger.error("Failed to create interpreter: {}", getClassName()); throw new InterpreterException(e); } finally { // TODO(jongyoul): Fixed it when not all of interpreter in same interpreter group are broken interpreterProcess.releaseClient(client, broken); } } initialized = true; } @Override public void open() { InterpreterGroup interpreterGroup = getInterpreterGroup(); synchronized (interpreterGroup) { // initialize all interpreters in this interpreter group List<Interpreter> interpreters = interpreterGroup.get(sessionKey); // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however, // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it // doesn't call open method, it's not open. It causes problem while running intp.close() // In case of Spark, this method initializes all of interpreters and init() method increases // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all // other interpreters doesn't do anything because those LazyInterpreters aren't open. // But for now, we have to initialise all of interpreters for some reasons. // See Interpreter.getInterpreterInTheSameSessionByClassName(String) RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); if (!initialized) { // reference per session interpreterProcess.reference(interpreterGroup, userName, isUserImpersonate); } for (Interpreter intp : new ArrayList<>(interpreters)) { Interpreter p = intp; while (p instanceof WrappedInterpreter) { p = ((WrappedInterpreter) p).getInnerInterpreter(); } try { ((RemoteInterpreter) p).init(); } catch (InterpreterException e) { logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup", p.getClassName()); interpreters.remove(p); } } } } @Override public void close() { InterpreterGroup interpreterGroup = getInterpreterGroup(); synchronized (interpreterGroup) { // close all interpreters in this session List<Interpreter> interpreters = interpreterGroup.get(sessionKey); // TODO(jl): this open method is called by LazyOpenInterpreter.open(). It, however, // initializes all of interpreters with same sessionKey. But LazyOpenInterpreter assumes if it // doesn't call open method, it's not open. It causes problem while running intp.close() // In case of Spark, this method initializes all of interpreters and init() method increases // reference count of RemoteInterpreterProcess. But while closing this interpreter group, all // other interpreters doesn't do anything because those LazyInterpreters aren't open. // But for now, we have to initialise all of interpreters for some reasons. // See Interpreter.getInterpreterInTheSameSessionByClassName(String) if (initialized) { // dereference per session getInterpreterProcess().dereference(); } for (Interpreter intp : new ArrayList<>(interpreters)) { Interpreter p = intp; while (p instanceof WrappedInterpreter) { p = ((WrappedInterpreter) p).getInnerInterpreter(); } try { ((RemoteInterpreter) p).closeInterpreter(); } catch (InterpreterException e) { logger.error("Failed to initialize interpreter: {}. Remove it from interpreterGroup", p.getClassName()); interpreters.remove(p); } } } } public void closeInterpreter() { if (this.initialized == false) { return; } RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; boolean broken = false; try { client = interpreterProcess.getClient(); if (client != null) { client.close(sessionKey, className); } } catch (TException e) { broken = true; throw new InterpreterException(e); } catch (Exception e1) { throw new InterpreterException(e1); } finally { if (client != null) { interpreterProcess.releaseClient(client, broken); } this.initialized = false; } } @Override public InterpreterResult interpret(String st, InterpreterContext context) { if (logger.isDebugEnabled()) { logger.debug("st:\n{}", st); } FormType form = getFormType(); RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } InterpreterContextRunnerPool interpreterContextRunnerPool = interpreterProcess .getInterpreterContextRunnerPool(); List<InterpreterContextRunner> runners = context.getRunners(); if (runners != null && runners.size() != 0) { // assume all runners in this InterpreterContext have the same note id String noteId = runners.get(0).getNoteId(); interpreterContextRunnerPool.clear(noteId); interpreterContextRunnerPool.addAll(noteId, runners); } boolean broken = false; try { final GUI currentGUI = context.getGui(); RemoteInterpreterResult remoteResult = client.interpret( sessionKey, className, st, convert(context)); Map<String, Object> remoteConfig = (Map<String, Object>) gson.fromJson( remoteResult.getConfig(), new TypeToken<Map<String, Object>>() { }.getType()); context.getConfig().clear(); context.getConfig().putAll(remoteConfig); if (form == FormType.NATIVE) { GUI remoteGui = GUI.fromJson(remoteResult.getGui()); currentGUI.clear(); currentGUI.setParams(remoteGui.getParams()); currentGUI.setForms(remoteGui.getForms()); } else if (form == FormType.SIMPLE) { final Map<String, Input> currentForms = currentGUI.getForms(); final Map<String, Object> currentParams = currentGUI.getParams(); final GUI remoteGUI = GUI.fromJson(remoteResult.getGui()); final Map<String, Input> remoteForms = remoteGUI.getForms(); final Map<String, Object> remoteParams = remoteGUI.getParams(); currentForms.putAll(remoteForms); currentParams.putAll(remoteParams); } InterpreterResult result = convert(remoteResult); return result; } catch (TException e) { broken = true; throw new InterpreterException(e); } finally { interpreterProcess.releaseClient(client, broken); } } @Override public void cancel(InterpreterContext context) { RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } boolean broken = false; try { client.cancel(sessionKey, className, convert(context)); } catch (TException e) { broken = true; throw new InterpreterException(e); } finally { interpreterProcess.releaseClient(client, broken); } } @Override public FormType getFormType() { open(); if (formType != null) { return formType; } RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } boolean broken = false; try { formType = FormType.valueOf(client.getFormType(sessionKey, className)); return formType; } catch (TException e) { broken = true; throw new InterpreterException(e); } finally { interpreterProcess.releaseClient(client, broken); } } @Override public int getProgress(InterpreterContext context) { RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); if (interpreterProcess == null || !interpreterProcess.isRunning()) { return 0; } Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } boolean broken = false; try { return client.getProgress(sessionKey, className, convert(context)); } catch (TException e) { broken = true; throw new InterpreterException(e); } finally { interpreterProcess.releaseClient(client, broken); } } @Override public List<InterpreterCompletion> completion(String buf, int cursor, InterpreterContext interpreterContext) { RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); Client client = null; try { client = interpreterProcess.getClient(); } catch (Exception e1) { throw new InterpreterException(e1); } boolean broken = false; try { List completion = client.completion(sessionKey, className, buf, cursor, convert(interpreterContext)); return completion; } catch (TException e) { broken = true; throw new InterpreterException(e); } finally { interpreterProcess.releaseClient(client, broken); } } @Override public Scheduler getScheduler() { int maxConcurrency = maxPoolSize; RemoteInterpreterProcess interpreterProcess = getInterpreterProcess(); if (interpreterProcess == null) { return null; } else { return SchedulerFactory.singleton().createOrGetRemoteScheduler( RemoteInterpreter.class.getName() + sessionKey + interpreterProcess.hashCode(), sessionKey, interpreterProcess, maxConcurrency); } } private String getInterpreterGroupKey(InterpreterGroup interpreterGroup) { return interpreterGroup.getId(); } private RemoteInterpreterContext convert(InterpreterContext ic) { return new RemoteInterpreterContext(ic.getNoteId(), ic.getParagraphId(), ic.getReplName(), ic.getParagraphTitle(), ic.getParagraphText(), gson.toJson(ic.getAuthenticationInfo()), gson.toJson(ic.getConfig()), gson.toJson(ic.getGui()), gson.toJson(ic.getRunners())); } private InterpreterResult convert(RemoteInterpreterResult result) { InterpreterResult r = new InterpreterResult( InterpreterResult.Code.valueOf(result.getCode())); for (RemoteInterpreterResultMessage m : result.getMsg()) { r.add(InterpreterResult.Type.valueOf(m.getType()), m.getData()); } return r; } /** * Push local angular object registry to * remote interpreter. This method should be * call ONLY inside the init() method */ void pushAngularObjectRegistryToRemote(Client client) throws TException { final AngularObjectRegistry angularObjectRegistry = this.getInterpreterGroup() .getAngularObjectRegistry(); if (angularObjectRegistry != null && angularObjectRegistry.getRegistry() != null) { final Map<String, Map<String, AngularObject>> registry = angularObjectRegistry .getRegistry(); logger.info("Push local angular object registry from ZeppelinServer to" + " remote interpreter group {}", this.getInterpreterGroup().getId()); final java.lang.reflect.Type registryType = new TypeToken<Map<String, Map<String, AngularObject>>>() { }.getType(); Gson gson = new Gson(); client.angularRegistryPush(gson.toJson(registry, registryType)); } } public Map<String, String> getEnv() { return env; } public void setEnv(Map<String, String> env) { this.env = env; } public void addEnv(Map<String, String> env) { if (this.env == null) { this.env = new HashMap<>(); } this.env.putAll(env); } //Only for test public String getInterpreterRunner() { return interpreterRunner; } }