NotebookRepoSyncTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.zeppelin.notebook.repo;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;

import java.io.File;
import java.io.IOException;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.zeppelin.conf.ZeppelinConfiguration;
import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars;
import org.apache.zeppelin.interpreter.InterpreterFactory;
import org.apache.zeppelin.interpreter.InterpreterOption;
import org.apache.zeppelin.interpreter.mock.MockInterpreter1;
import org.apache.zeppelin.interpreter.mock.MockInterpreter2;
import org.apache.zeppelin.notebook.JobListenerFactory;
import org.apache.zeppelin.notebook.Note;
import org.apache.zeppelin.notebook.Notebook;
import org.apache.zeppelin.notebook.Paragraph;
import org.apache.zeppelin.scheduler.Job;
import org.apache.zeppelin.scheduler.Job.Status;
import org.apache.zeppelin.scheduler.JobListener;
import org.apache.zeppelin.scheduler.SchedulerFactory;
import org.apache.zeppelin.search.SearchService;
import org.apache.zeppelin.search.LuceneSearch;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class NotebookRepoSyncTest implements JobListenerFactory {

  private File mainZepDir;
  private ZeppelinConfiguration conf;
  private SchedulerFactory schedulerFactory;
  private File mainNotebookDir;
  private File secNotebookDir;
  private Notebook notebookSync;
  private NotebookRepoSync notebookRepoSync;
  private InterpreterFactory factory;
  private static final Logger LOG = LoggerFactory.getLogger(NotebookRepoSyncTest.class);
  
  @Before
  public void setUp() throws Exception {
    String zpath = System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis();
    mainZepDir = new File(zpath);
    mainZepDir.mkdirs();
    new File(mainZepDir, "conf").mkdirs();
    String mainNotePath = zpath+"/notebook";
    String secNotePath = mainNotePath + "_secondary";
    mainNotebookDir = new File(mainNotePath);
    secNotebookDir = new File(secNotePath);
    mainNotebookDir.mkdirs();
    secNotebookDir.mkdirs();

    System.setProperty(ConfVars.ZEPPELIN_HOME.getVarName(), mainZepDir.getAbsolutePath());
    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_DIR.getVarName(), mainNotebookDir.getAbsolutePath());
    System.setProperty(ConfVars.ZEPPELIN_INTERPRETERS.getVarName(), "org.apache.zeppelin.interpreter.mock.MockInterpreter1,org.apache.zeppelin.interpreter.mock.MockInterpreter2");
    System.setProperty(ConfVars.ZEPPELIN_NOTEBOOK_STORAGE.getVarName(), "org.apache.zeppelin.notebook.repo.VFSNotebookRepo,org.apache.zeppelin.notebook.repo.mock.VFSNotebookRepoMock");
    LOG.info("main Note dir : " + mainNotePath);
    LOG.info("secondary note dir : " + secNotePath);
    conf = ZeppelinConfiguration.create();

    this.schedulerFactory = new SchedulerFactory();

    MockInterpreter1.register("mock1", "org.apache.zeppelin.interpreter.mock.MockInterpreter1");
    MockInterpreter2.register("mock2", "org.apache.zeppelin.interpreter.mock.MockInterpreter2");

    factory = new InterpreterFactory(conf, new InterpreterOption(false), null);
    
    SearchService search = mock(SearchService.class);
    notebookRepoSync = new NotebookRepoSync(conf);
    notebookSync = new Notebook(conf, notebookRepoSync, schedulerFactory, factory, this, search);
  }

  @After
  public void tearDown() throws Exception {
    delete(mainZepDir);
  }
  
  @Test
  public void testRepoCount() throws IOException {
    assertTrue(notebookRepoSync.getMaxRepoNum() >= notebookRepoSync.getRepoCount());
  }
  
  @Test
  public void testSyncOnCreate() throws IOException {
    /* check that both storage systems are empty */
    assertTrue(notebookRepoSync.getRepoCount() > 1);
    assertEquals(0, notebookRepoSync.list(0).size());
    assertEquals(0, notebookRepoSync.list(1).size());
    
    /* create note */
    Note note = notebookSync.createNote();

    // check that automatically saved on both storages
    assertEquals(1, notebookRepoSync.list(0).size());
    assertEquals(1, notebookRepoSync.list(1).size());
    assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
    
  }

  @Test
  public void testSyncOnDelete() throws IOException {
    /* create note */
    assertTrue(notebookRepoSync.getRepoCount() > 1);
    assertEquals(0, notebookRepoSync.list(0).size());
    assertEquals(0, notebookRepoSync.list(1).size());
    
    Note note = notebookSync.createNote();

    /* check that created in both storage systems */
    assertEquals(1, notebookRepoSync.list(0).size());
    assertEquals(1, notebookRepoSync.list(1).size());
    assertEquals(notebookRepoSync.list(0).get(0).getId(),notebookRepoSync.list(1).get(0).getId());
    
    /* remove Note */
    notebookSync.removeNote(notebookRepoSync.list(0).get(0).getId());
    
    /* check that deleted in both storages */
    assertEquals(0, notebookRepoSync.list(0).size());
    assertEquals(0, notebookRepoSync.list(1).size());
    
  }
  
  @Test
  public void testSyncUpdateMain() throws IOException {
    
    /* create note */
    Note note = notebookSync.createNote();
    Paragraph p1 = note.addParagraph();
    Map config = p1.getConfig();
    config.put("enabled", true);
    p1.setConfig(config);
    p1.setText("hello world");
    
    /* new paragraph exists in note instance */
    assertEquals(1, note.getParagraphs().size());
    
    /* new paragraph not yet saved into storages */
    assertEquals(0, notebookRepoSync.get(0,
        notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
    assertEquals(0, notebookRepoSync.get(1,
        notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
    
    /* save to storage under index 0 (first storage) */ 
    notebookRepoSync.save(0, note);
    
    /* check paragraph saved to first storage */
    assertEquals(1, notebookRepoSync.get(0,
        notebookRepoSync.list(0).get(0).getId()).getParagraphs().size());
    /* check paragraph isn't saved to second storage */
    assertEquals(0, notebookRepoSync.get(1,
        notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
    /* apply sync */
    notebookRepoSync.sync();
    /* check whether added to second storage */
    assertEquals(1, notebookRepoSync.get(1,
    notebookRepoSync.list(1).get(0).getId()).getParagraphs().size());
    /* check whether same paragraph id */
    assertEquals(p1.getId(), notebookRepoSync.get(0,
        notebookRepoSync.list(0).get(0).getId()).getLastParagraph().getId());
    assertEquals(p1.getId(), notebookRepoSync.get(1,
        notebookRepoSync.list(1).get(0).getId()).getLastParagraph().getId());
  }

  @Test
  public void testSyncOnReloadedList() throws IOException {
    /* check that both storage repos are empty */
    assertTrue(notebookRepoSync.getRepoCount() > 1);
    assertEquals(0, notebookRepoSync.list(0).size());
    assertEquals(0, notebookRepoSync.list(1).size());

    File srcDir = new File("src/test/resources/2A94M5J1Z");
    File destDir = new File(secNotebookDir + "/2A94M5J1Z");

    /* copy manually new notebook into secondary storage repo and check repos */
    try {
      FileUtils.copyDirectory(srcDir, destDir);
    } catch (IOException e) {
      e.printStackTrace();
    }
    assertEquals(0, notebookRepoSync.list(0).size());
    assertEquals(1, notebookRepoSync.list(1).size());

    // After reloading notebooks repos should be synchronized
    notebookSync.reloadAllNotes();
    assertEquals(1, notebookRepoSync.list(0).size());
    assertEquals(1, notebookRepoSync.list(1).size());
  }

  static void delete(File file){
    if(file.isFile()) file.delete();
      else if(file.isDirectory()){
        File [] files = file.listFiles();
        if(files!=null && files.length>0){
          for(File f : files){
            delete(f);
          }
        }
        file.delete();
      }
  }

  @Override
  public JobListener getParagraphJobListener(Note note) {
    return new JobListener(){

      @Override
      public void onProgressUpdate(Job job, int progress) {
      }

      @Override
      public void beforeStatusChange(Job job, Status before, Status after) {
      }

      @Override
      public void afterStatusChange(Job job, Status before, Status after) {
      }
    };
  }
	
}