/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.interpreter.remote; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; import org.apache.thrift.TException; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.helium.ApplicationEventListener; import org.apache.zeppelin.interpreter.InterpreterContextRunner; import org.apache.zeppelin.interpreter.InterpreterGroup; import org.apache.zeppelin.interpreter.InterpreterResult; import org.apache.zeppelin.interpreter.RemoteZeppelinServerResource; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEvent; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterEventType; import org.apache.zeppelin.interpreter.thrift.RemoteInterpreterService.Client; import org.apache.zeppelin.interpreter.thrift.ZeppelinServerResourceParagraphRunner; import org.apache.zeppelin.resource.Resource; import org.apache.zeppelin.resource.ResourceId; import org.apache.zeppelin.resource.ResourcePool; import org.apache.zeppelin.resource.ResourceSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; /** * Processes message from RemoteInterpreter process */ public class RemoteInterpreterEventPoller extends Thread { private static final Logger logger = LoggerFactory.getLogger(RemoteInterpreterEventPoller.class); private final ScheduledExecutorService appendService = Executors.newSingleThreadScheduledExecutor(); private final RemoteInterpreterProcessListener listener; private final ApplicationEventListener appListener; private volatile boolean shutdown; private RemoteInterpreterProcess interpreterProcess; private InterpreterGroup interpreterGroup; Gson gson = new Gson(); public RemoteInterpreterEventPoller( RemoteInterpreterProcessListener listener, ApplicationEventListener appListener) { this.listener = listener; this.appListener = appListener; shutdown = false; } public void setInterpreterProcess(RemoteInterpreterProcess interpreterProcess) { this.interpreterProcess = interpreterProcess; } public void setInterpreterGroup(InterpreterGroup interpreterGroup) { this.interpreterGroup = interpreterGroup; } @Override public void run() { Client client = null; AppendOutputRunner runner = new AppendOutputRunner(listener); ScheduledFuture<?> appendFuture = appendService.scheduleWithFixedDelay( runner, 0, AppendOutputRunner.BUFFER_TIME_MS, TimeUnit.MILLISECONDS); while (!shutdown) { // wait and retry if (!interpreterProcess.isRunning()) { try { Thread.sleep(1000); } catch (InterruptedException e) { // nothing to do } continue; } try { client = interpreterProcess.getClient(); } catch (Exception e1) { logger.error("Can't get RemoteInterpreterEvent", e1); waitQuietly(); continue; } RemoteInterpreterEvent event = null; boolean broken = false; try { event = client.getEvent(); } catch (TException e) { broken = true; logger.error("Can't get RemoteInterpreterEvent", e); waitQuietly(); continue; } finally { interpreterProcess.releaseClient(client, broken); } AngularObjectRegistry angularObjectRegistry = interpreterGroup.getAngularObjectRegistry(); try { if (event.getType() != RemoteInterpreterEventType.NO_OP) { logger.debug("Receive message from RemoteInterpreter Process: " + event.toString()); } if (event.getType() == RemoteInterpreterEventType.NO_OP) { continue; } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_ADD) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); angularObjectRegistry.add(angularObject.getName(), angularObject.get(), angularObject.getNoteId(), angularObject.getParagraphId()); } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_UPDATE) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); AngularObject localAngularObject = angularObjectRegistry.get( angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId()); if (localAngularObject instanceof RemoteAngularObject) { // to avoid ping-pong loop ((RemoteAngularObject) localAngularObject).set( angularObject.get(), true, false); } else { localAngularObject.set(angularObject.get()); } } else if (event.getType() == RemoteInterpreterEventType.ANGULAR_OBJECT_REMOVE) { AngularObject angularObject = gson.fromJson(event.getData(), AngularObject.class); angularObjectRegistry.remove(angularObject.getName(), angularObject.getNoteId(), angularObject.getParagraphId()); } else if (event.getType() == RemoteInterpreterEventType.RUN_INTERPRETER_CONTEXT_RUNNER) { InterpreterContextRunner runnerFromRemote = gson.fromJson( event.getData(), RemoteInterpreterContextRunner.class); listener.onRemoteRunParagraph( runnerFromRemote.getNoteId(), runnerFromRemote.getParagraphId()); } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_POOL_GET_ALL) { ResourceSet resourceSet = getAllResourcePoolExcept(); sendResourcePoolResponseGetAll(resourceSet); } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_GET) { String resourceIdString = event.getData(); ResourceId resourceId = gson.fromJson(resourceIdString, ResourceId.class); logger.debug("RESOURCE_GET {} {}", resourceId.getResourcePoolId(), resourceId.getName()); Object o = getResource(resourceId); sendResourceResponseGet(resourceId, o); } else if (event.getType() == RemoteInterpreterEventType.RESOURCE_INVOKE_METHOD) { String message = event.getData(); InvokeResourceMethodEventMessage invokeMethodMessage = gson.fromJson(message, InvokeResourceMethodEventMessage.class); Object ret = invokeResourceMethod(invokeMethodMessage); sendInvokeMethodResult(invokeMethodMessage, ret); } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_APPEND) { // on output append Map<String, String> outputAppend = gson.fromJson( event.getData(), new TypeToken<Map<String, Object>>() {}.getType()); String noteId = (String) outputAppend.get("noteId"); String paragraphId = (String) outputAppend.get("paragraphId"); int index = Integer.parseInt(outputAppend.get("index")); String outputToAppend = (String) outputAppend.get("data"); String appId = (String) outputAppend.get("appId"); if (appId == null) { runner.appendBuffer(noteId, paragraphId, index, outputToAppend); } else { appListener.onOutputAppend(noteId, paragraphId, index, appId, outputToAppend); } } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE_ALL) { Map<String, Object> outputUpdate = gson.fromJson( event.getData(), new TypeToken<Map<String, Object>>() {}.getType()); String noteId = (String) outputUpdate.get("noteId"); String paragraphId = (String) outputUpdate.get("paragraphId"); // clear the output listener.onOutputClear(noteId, paragraphId); List<Map<String, String>> messages = (List<Map<String, String>>) outputUpdate.get("messages"); if (messages != null) { for (int i = 0; i < messages.size(); i++) { Map<String, String> m = messages.get(i); InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) m.get("type")); String outputToUpdate = (String) m.get("data"); listener.onOutputUpdated(noteId, paragraphId, i, type, outputToUpdate); } } } else if (event.getType() == RemoteInterpreterEventType.OUTPUT_UPDATE) { // on output update Map<String, String> outputAppend = gson.fromJson( event.getData(), new TypeToken<Map<String, Object>>() {}.getType()); String noteId = (String) outputAppend.get("noteId"); String paragraphId = (String) outputAppend.get("paragraphId"); int index = Integer.parseInt(outputAppend.get("index")); InterpreterResult.Type type = InterpreterResult.Type.valueOf((String) outputAppend.get("type")); String outputToUpdate = (String) outputAppend.get("data"); String appId = (String) outputAppend.get("appId"); if (appId == null) { listener.onOutputUpdated(noteId, paragraphId, index, type, outputToUpdate); } else { appListener.onOutputUpdated(noteId, paragraphId, index, appId, type, outputToUpdate); } } else if (event.getType() == RemoteInterpreterEventType.APP_STATUS_UPDATE) { // on output update Map<String, String> appStatusUpdate = gson.fromJson( event.getData(), new TypeToken<Map<String, String>>() {}.getType()); String noteId = appStatusUpdate.get("noteId"); String paragraphId = appStatusUpdate.get("paragraphId"); String appId = appStatusUpdate.get("appId"); String status = appStatusUpdate.get("status"); appListener.onStatusChange(noteId, paragraphId, appId, status); } else if (event.getType() == RemoteInterpreterEventType.REMOTE_ZEPPELIN_SERVER_RESOURCE) { RemoteZeppelinServerResource reqResourceBody = gson.fromJson( event.getData(), RemoteZeppelinServerResource.class); progressRemoteZeppelinControlEvent( reqResourceBody.getResourceType(), listener, reqResourceBody); } else if (event.getType() == RemoteInterpreterEventType.META_INFOS) { Map<String, String> metaInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() { }.getType()); String settingId = RemoteInterpreterUtils. getInterpreterSettingId(interpreterGroup.getId()); listener.onMetaInfosReceived(settingId, metaInfos); } else if (event.getType() == RemoteInterpreterEventType.PARA_INFOS) { Map<String, String> paraInfos = gson.fromJson(event.getData(), new TypeToken<Map<String, String>>() { }.getType()); String noteId = paraInfos.get("noteId"); String paraId = paraInfos.get("paraId"); String settingId = RemoteInterpreterUtils. getInterpreterSettingId(interpreterGroup.getId()); if (noteId != null && paraId != null && settingId != null) { listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos); } } logger.debug("Event from remote process {}", event.getType()); } catch (Exception e) { logger.error("Can't handle event " + event, e); } } try { clearUnreadEvents(interpreterProcess.getClient()); } catch (Exception e1) { logger.error("Can't get RemoteInterpreterEvent", e1); } if (appendFuture != null) { appendFuture.cancel(true); } } private void clearUnreadEvents(Client client) throws TException { while (client.getEvent().getType() != RemoteInterpreterEventType.NO_OP) {} } private void progressRemoteZeppelinControlEvent( RemoteZeppelinServerResource.Type resourceType, RemoteInterpreterProcessListener remoteWorksEventListener, RemoteZeppelinServerResource reqResourceBody) throws Exception { boolean broken = false; final Gson gson = new Gson(); final String eventOwnerKey = reqResourceBody.getOwnerKey(); Client interpreterServerMain = null; try { interpreterServerMain = interpreterProcess.getClient(); final Client eventClient = interpreterServerMain; if (resourceType == RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS) { final List<ZeppelinServerResourceParagraphRunner> remoteRunners = new LinkedList<>(); ZeppelinServerResourceParagraphRunner reqRunnerContext = new ZeppelinServerResourceParagraphRunner(); Map<String, Object> reqResourceMap = (Map<String, Object>) reqResourceBody.getData(); String noteId = (String) reqResourceMap.get("noteId"); String paragraphId = (String) reqResourceMap.get("paragraphId"); reqRunnerContext.setNoteId(noteId); reqRunnerContext.setParagraphId(paragraphId); RemoteInterpreterProcessListener.RemoteWorksEventListener callBackEvent = new RemoteInterpreterProcessListener.RemoteWorksEventListener() { @Override public void onFinished(Object resultObject) { boolean clientBroken = false; if (resultObject != null && resultObject instanceof List) { List<InterpreterContextRunner> runnerList = (List<InterpreterContextRunner>) resultObject; for (InterpreterContextRunner r : runnerList) { remoteRunners.add( new ZeppelinServerResourceParagraphRunner(r.getNoteId(), r.getParagraphId()) ); } final RemoteZeppelinServerResource resResource = new RemoteZeppelinServerResource(); resResource.setOwnerKey(eventOwnerKey); resResource.setResourceType(RemoteZeppelinServerResource.Type.PARAGRAPH_RUNNERS); resResource.setData(remoteRunners); try { eventClient.onReceivedZeppelinResource(gson.toJson(resResource)); } catch (Exception e) { clientBroken = true; logger.error("Can't get RemoteInterpreterEvent", e); waitQuietly(); } finally { interpreterProcess.releaseClient(eventClient, clientBroken); } } } @Override public void onError() { logger.info("onGetParagraphRunners onError"); } }; remoteWorksEventListener.onGetParagraphRunners( reqRunnerContext.getNoteId(), reqRunnerContext.getParagraphId(), callBackEvent); } } catch (Exception e) { broken = true; logger.error("Can't get RemoteInterpreterEvent", e); waitQuietly(); } finally { interpreterProcess.releaseClient(interpreterServerMain, broken); } } private void sendResourcePoolResponseGetAll(ResourceSet resourceSet) { Client client = null; boolean broken = false; try { client = interpreterProcess.getClient(); List<String> resourceList = new LinkedList<>(); Gson gson = new Gson(); for (Resource r : resourceSet) { resourceList.add(gson.toJson(r)); } client.resourcePoolResponseGetAll(resourceList); } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { interpreterProcess.releaseClient(client, broken); } } } private ResourceSet getAllResourcePoolExcept() { ResourceSet resourceSet = new ResourceSet(); for (InterpreterGroup intpGroup : InterpreterGroup.getAll()) { if (intpGroup.getId().equals(interpreterGroup.getId())) { continue; } RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess(); if (remoteInterpreterProcess == null) { ResourcePool localPool = intpGroup.getResourcePool(); if (localPool != null) { resourceSet.addAll(localPool.getAll()); } } else if (interpreterProcess.isRunning()) { Client client = null; boolean broken = false; try { client = remoteInterpreterProcess.getClient(); List<String> resourceList = client.resourcePoolGetAll(); Gson gson = new Gson(); for (String res : resourceList) { resourceSet.add(gson.fromJson(res, Resource.class)); } } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken); } } } } return resourceSet; } private void sendResourceResponseGet(ResourceId resourceId, Object o) { Client client = null; boolean broken = false; try { client = interpreterProcess.getClient(); Gson gson = new Gson(); String rid = gson.toJson(resourceId); ByteBuffer obj; if (o == null) { obj = ByteBuffer.allocate(0); } else { obj = Resource.serializeObject(o); } client.resourceResponseGet(rid, obj); } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { interpreterProcess.releaseClient(client, broken); } } } private Object getResource(ResourceId resourceId) { InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId( resourceId.getResourcePoolId()); if (intpGroup == null) { return null; } RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess(); if (remoteInterpreterProcess == null) { ResourcePool localPool = intpGroup.getResourcePool(); if (localPool != null) { return localPool.get(resourceId.getName()); } } else if (interpreterProcess.isRunning()) { Client client = null; boolean broken = false; try { client = remoteInterpreterProcess.getClient(); ByteBuffer res = client.resourceGet( resourceId.getNoteId(), resourceId.getParagraphId(), resourceId.getName()); Object o = Resource.deserializeObject(res); return o; } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken); } } } return null; } public void sendInvokeMethodResult(InvokeResourceMethodEventMessage message, Object o) { Client client = null; boolean broken = false; try { client = interpreterProcess.getClient(); Gson gson = new Gson(); String invokeMessage = gson.toJson(message); ByteBuffer obj; if (o == null) { obj = ByteBuffer.allocate(0); } else { obj = Resource.serializeObject(o); } client.resourceResponseInvokeMethod(invokeMessage, obj); } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { interpreterProcess.releaseClient(client, broken); } } } private Object invokeResourceMethod(InvokeResourceMethodEventMessage message) { ResourceId resourceId = message.resourceId; InterpreterGroup intpGroup = InterpreterGroup.getByInterpreterGroupId( resourceId.getResourcePoolId()); if (intpGroup == null) { return null; } RemoteInterpreterProcess remoteInterpreterProcess = intpGroup.getRemoteInterpreterProcess(); if (remoteInterpreterProcess == null) { ResourcePool localPool = intpGroup.getResourcePool(); if (localPool != null) { Resource res = localPool.get(resourceId.getName()); if (res != null) { try { return res.invokeMethod( message.methodName, message.getParamTypes(), message.params, message.returnResourceName); } catch (Exception e) { logger.error(e.getMessage(), e); return null; } } else { // object is null. can't invoke any method logger.error("Can't invoke method {} on null object", message.methodName); return null; } } else { logger.error("no resource pool"); return null; } } else if (interpreterProcess.isRunning()) { Client client = null; boolean broken = false; try { client = remoteInterpreterProcess.getClient(); ByteBuffer res = client.resourceInvokeMethod( resourceId.getNoteId(), resourceId.getParagraphId(), resourceId.getName(), gson.toJson(message)); Object o = Resource.deserializeObject(res); return o; } catch (Exception e) { logger.error(e.getMessage(), e); broken = true; } finally { if (client != null) { intpGroup.getRemoteInterpreterProcess().releaseClient(client, broken); } } } return null; } private void waitQuietly() { try { synchronized (this) { wait(1000); } } catch (InterruptedException ignored) { logger.info("Error in RemoteInterpreterEventPoller while waitQuietly : ", ignored); } } public void shutdown() { shutdown = true; synchronized (this) { notify(); } } }