/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.zeppelin.notebook.repo; import java.io.IOException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.NoteInfo; import org.apache.zeppelin.notebook.Paragraph; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Notebook repository sync with remote storage */ public class NotebookRepoSync implements NotebookRepo { private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSync.class); private static final int maxRepoNum = 2; private static final String pushKey = "pushNoteIDs"; private static final String pullKey = "pullNoteIDs"; private static ZeppelinConfiguration config; private static final String defaultStorage = "org.apache.zeppelin.notebook.repo.VFSNotebookRepo"; private List<NotebookRepo> repos = new ArrayList<NotebookRepo>(); /** * @param noteIndex * @param (conf) * @throws - Exception */ @SuppressWarnings("static-access") public NotebookRepoSync(ZeppelinConfiguration conf) { config = conf; String allStorageClassNames = conf.getString(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE).trim(); if (allStorageClassNames.isEmpty()) { allStorageClassNames = defaultStorage; LOG.warn("Empty ZEPPELIN_NOTEBOOK_STORAGE conf parameter, using default {}", defaultStorage); } String[] storageClassNames = allStorageClassNames.split(","); if (storageClassNames.length > getMaxRepoNum()) { LOG.warn("Unsupported number {} of storage classes in ZEPPELIN_NOTEBOOK_STORAGE : {}\n" + "first {} will be used", storageClassNames.length, allStorageClassNames, getMaxRepoNum()); } for (int i = 0; i < Math.min(storageClassNames.length, getMaxRepoNum()); i++) { @SuppressWarnings("static-access") Class<?> notebookStorageClass; try { notebookStorageClass = getClass().forName(storageClassNames[i].trim()); Constructor<?> constructor = notebookStorageClass.getConstructor( ZeppelinConfiguration.class); repos.add((NotebookRepo) constructor.newInstance(conf)); } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { LOG.warn("Failed to initialize {} notebook storage class {}", storageClassNames[i], e); } } // couldn't initialize any storage, use default if (getRepoCount() == 0) { LOG.info("No storages could be initialized, using default {} storage", defaultStorage); initializeDefaultStorage(conf); } if (getRepoCount() > 1) { try { sync(0, 1); } catch (IOException e) { LOG.warn("Failed to sync with secondary storage on start {}", e); } } } @SuppressWarnings("static-access") private void initializeDefaultStorage(ZeppelinConfiguration conf) { Class<?> notebookStorageClass; try { notebookStorageClass = getClass().forName(defaultStorage); Constructor<?> constructor = notebookStorageClass.getConstructor( ZeppelinConfiguration.class); repos.add((NotebookRepo) constructor.newInstance(conf)); } catch (ClassNotFoundException | NoSuchMethodException | SecurityException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { LOG.warn("Failed to initialize {} notebook storage class {}", defaultStorage, e); } } /** * Lists Notebooks from the first repository */ @Override public List<NoteInfo> list() throws IOException { return getRepo(0).list(); } /* list from specific repo (for tests) */ List<NoteInfo> list(int repoIndex) throws IOException { return getRepo(repoIndex).list(); } /** * Returns from Notebook from the first repository */ @Override public Note get(String noteId) throws IOException { return getRepo(0).get(noteId); } /* get note from specific repo (for tests) */ Note get(int repoIndex, String noteId) throws IOException { return getRepo(repoIndex).get(noteId); } /** * Saves to all repositories */ @Override public void save(Note note) throws IOException { getRepo(0).save(note); if (getRepoCount() > 1) { try { getRepo(1).save(note); } catch (IOException e) { LOG.info(e.getMessage() + ": Failed to write to secondary storage"); } } } /* save note to specific repo (for tests) */ void save(int repoIndex, Note note) throws IOException { getRepo(repoIndex).save(note); } @Override public void remove(String noteId) throws IOException { for (NotebookRepo repo : repos) { repo.remove(noteId); } /* TODO(khalid): handle case when removing from secondary storage fails */ } /** * Copies new/updated notes from source to destination storage * * @throws IOException */ void sync(int sourceRepoIndex, int destRepoIndex) throws IOException { LOG.info("Sync started"); NotebookRepo srcRepo = getRepo(sourceRepoIndex); NotebookRepo dstRepo = getRepo(destRepoIndex); List <NoteInfo> srcNotes = srcRepo.list(); List <NoteInfo> dstNotes = dstRepo.list(); Map<String, List<String>> noteIDs = notesCheckDiff(srcNotes, srcRepo, dstNotes, dstRepo); List<String> pushNoteIDs = noteIDs.get(pushKey); List<String> pullNoteIDs = noteIDs.get(pullKey); if (!pushNoteIDs.isEmpty()) { LOG.info("Notes with the following IDs will be pushed"); for (String id : pushNoteIDs) { LOG.info("ID : " + id); } pushNotes(pushNoteIDs, srcRepo, dstRepo); } else { LOG.info("Nothing to push"); } if (!pullNoteIDs.isEmpty()) { LOG.info("Notes with the following IDs will be pulled"); for (String id : pullNoteIDs) { LOG.info("ID : " + id); } pushNotes(pullNoteIDs, dstRepo, srcRepo); } else { LOG.info("Nothing to pull"); } LOG.info("Sync ended"); } public void sync() throws IOException { sync(0, 1); } private void pushNotes(List<String> ids, NotebookRepo localRepo, NotebookRepo remoteRepo) throws IOException { for (String id : ids) { remoteRepo.save(localRepo.get(id)); } } public int getRepoCount() { return repos.size(); } int getMaxRepoNum() { return maxRepoNum; } NotebookRepo getRepo(int repoIndex) throws IOException { if (repoIndex < 0 || repoIndex >= getRepoCount()) { throw new IOException("Storage repo index is out of range"); } return repos.get(repoIndex); } private Map<String, List<String>> notesCheckDiff(List<NoteInfo> sourceNotes, NotebookRepo sourceRepo, List<NoteInfo> destNotes, NotebookRepo destRepo) throws IOException { List <String> pushIDs = new ArrayList<String>(); List <String> pullIDs = new ArrayList<String>(); NoteInfo dnote; Date sdate, ddate; for (NoteInfo snote : sourceNotes) { dnote = containsID(destNotes, snote.getId()); if (dnote != null) { /* note exists in source and destination storage systems */ sdate = lastModificationDate(sourceRepo.get(snote.getId())); ddate = lastModificationDate(destRepo.get(dnote.getId())); if (sdate.after(ddate)) { /* source contains more up to date note - push */ pushIDs.add(snote.getId()); LOG.info("Modified note is added to push list : " + sdate); } else if (sdate.compareTo(ddate) != 0) { /* destination contains more up to date note - pull */ LOG.info("Modified note is added to pull list : " + ddate); pullIDs.add(snote.getId()); } } else { /* note exists in source storage, and absent in destination * view source as up to date - push * (another scenario : note was deleted from destination - not considered)*/ pushIDs.add(snote.getId()); } } for (NoteInfo note : destNotes) { dnote = containsID(sourceNotes, note.getId()); if (dnote == null) { /* note exists in destination storage, and absent in source - pull*/ pullIDs.add(note.getId()); } } Map<String, List<String>> map = new HashMap<String, List<String>>(); map.put(pushKey, pushIDs); map.put(pullKey, pullIDs); return map; } private NoteInfo containsID(List <NoteInfo> notes, String id) { for (NoteInfo note : notes) { if (note.getId().equals(id)) { return note; } } return null; } /** * checks latest modification date based on Paragraph fields * @return -Date */ private Date lastModificationDate(Note note) { Date latest = new Date(0L); Date tempCreated, tempStarted, tempFinished; for (Paragraph paragraph : note.getParagraphs()) { tempCreated = paragraph.getDateCreated(); tempStarted = paragraph.getDateStarted(); tempFinished = paragraph.getDateFinished(); if (tempCreated != null && tempCreated.after(latest)) { latest = tempCreated; } if (tempStarted != null && tempStarted.after(latest)) { latest = tempStarted; } if (tempFinished != null && tempFinished.after(latest)) { latest = tempFinished; } } return latest; } @SuppressWarnings("unused") private void printParagraphs(Note note) { LOG.info("Note name : " + note.getName()); LOG.info("Note ID : " + note.id()); for (Paragraph p : note.getParagraphs()) { printParagraph(p); } } private void printParagraph(Paragraph paragraph) { LOG.info("Date created : " + paragraph.getDateCreated()); LOG.info("Date started : " + paragraph.getDateStarted()); LOG.info("Date finished : " + paragraph.getDateFinished()); LOG.info("Paragraph ID : " + paragraph.getId()); LOG.info("Paragraph title : " + paragraph.getTitle()); } @SuppressWarnings("unused") private void printNoteInfos(List <NoteInfo> notes) { LOG.info("The following is a list of note infos"); for (NoteInfo note : notes) { printNoteInfo(note); } } private void printNoteInfo(NoteInfo note) { LOG.info("Note info of notebook with name : " + note.getName()); LOG.info("ID : " + note.getId()); Map<String, Object> configs = note.getConfig(); for (Map.Entry<String, Object> entry : configs.entrySet()) { LOG.info("Config Key = " + entry.getKey() + " , Value = " + entry.getValue().toString() + "of class " + entry.getClass()); } } @Override public void close() { LOG.info("Closing all notebook storages"); for (NotebookRepo repo: repos) { repo.close(); } } }