TestFileSystemCheckpointPersistanceProvider.java

package com.linkedin.databus.client.pub;
/*
 *
 * Copyright 2013 LinkedIn Corp. All rights reserved
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
*/


import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertNotNull;
import static org.testng.AssertJUnit.assertNull;
import static org.testng.AssertJUnit.assertTrue;

import java.io.File;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Random;

import org.apache.log4j.BasicConfigurator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.testng.annotations.Test;

import com.linkedin.databus.client.pub.FileSystemCheckpointPersistenceProvider.CacheEntry;
import com.linkedin.databus.core.Checkpoint;
import com.linkedin.databus.core.DbusClientMode;
import com.linkedin.databus.core.data_model.DatabusSubscription;

public class TestFileSystemCheckpointPersistanceProvider
{
  public static final Logger LOG = Logger.getLogger(TestFileSystemCheckpointPersistanceProvider.class);

  static
  {
    BasicConfigurator.configure();
    Logger.getRootLogger().setLevel(Level.OFF);
    //Logger.getRootLogger().setLevel(Level.ERROR);
    //Logger.getRootLogger().setLevel(Level.INFO);
    //Logger.getRootLogger().setLevel(Level.DEBUG);
  }

  @Test
  public void testCacheEntryloadCurrentCheckpoint_new() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(false);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    List<String> sourceNames = Arrays.asList("source1", "source2");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);

    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2", streamId);

    File streamFile = new File(checkpointDir, streamId + ".current");
    if (streamFile.exists())
    {
      assertTrue(streamFile.delete());
    }

    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    Checkpoint checkpoint = cacheEntry.getCheckpoint();
    assertNull(checkpoint);
  }

  @Test
  public void testCacheEntry_store_load_noroll() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(false);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    List<String> sourceNames = Arrays.asList("source1", "source2");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);

    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2", streamId);

    File checkpointFile = new File(checkpointDir, streamId + ".current");
    if (checkpointFile.exists())
    {
      assertTrue(checkpointFile.delete());
    }

    //simple checkpoint
    Checkpoint cp1 = new Checkpoint();
    cp1.setFlexible();

    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    assertTrue(cacheEntry.setCheckpoint(cp1));
    assertTrue(checkpointFile.exists());

    CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    Checkpoint cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(DbusClientMode.ONLINE_CONSUMPTION, cp2.getConsumptionMode());
    assertTrue(cp2.getFlexible());

    //more complex checkpoint plus overwriting current state
    Checkpoint cp3 = new Checkpoint();
    cp3.setWindowScn(1234L);
    cp3.setWindowOffset(9876);
    cp3.setConsumptionMode(DbusClientMode.ONLINE_CONSUMPTION);

    assertTrue(cacheEntry.setCheckpoint(cp3));
    assertTrue(checkpointFile.exists());

    File cpBackupFile = new File(checkpointDir, streamId + ".oldcurrent");
    assertTrue(cpBackupFile.exists());

    cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(cp3.getConsumptionMode(), cp2.getConsumptionMode());
    assertEquals(cp3.getWindowScn(), cp2.getWindowScn());
    assertEquals(cp3.getPrevScn(), cp2.getPrevScn());
    assertEquals(cp3.getWindowOffset(), cp2.getWindowOffset());

    //make sure the backup still works
    assertTrue(checkpointFile.delete());
    assertTrue(!checkpointFile.exists());

    if (!cpBackupFile.renameTo(checkpointFile))
        LOG.error("file rename failed: " + cpBackupFile.getAbsolutePath() + " --> " +
                  checkpointFile.getAbsolutePath());
    assertTrue(checkpointFile.exists());

    cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
    cp2 = cacheEntry2.getCheckpoint();
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(DbusClientMode.ONLINE_CONSUMPTION, cp2.getConsumptionMode());
    assertTrue(cp2.getFlexible());

    //try to keep some stuff around and see if things go south
    Checkpoint cp4 = new Checkpoint();
    cp4.setWindowScn(1111L);
    cp4.setWindowOffset(2222);
    cp4.setConsumptionMode(DbusClientMode.BOOTSTRAP_CATCHUP);

    File newCpFile = new File(checkpointDir, streamId + ".newcheckpoint");
    PrintWriter tmpWriter = new PrintWriter(newCpFile);
    try{
      tmpWriter.println("Dummy");

      assertTrue(cacheEntry.setCheckpoint(cp4));
      assertTrue(checkpointFile.exists());

      cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
      cp2 = cacheEntry2.getCheckpoint();
      assertNotNull(cp2);
      //TODO need to use a Checkpoint.equals() method
      assertEquals(cp4.getConsumptionMode(), cp2.getConsumptionMode());
      assertEquals(cp4.getWindowScn(), cp2.getWindowScn());
      assertEquals(cp4.getPrevScn(), cp2.getPrevScn());
      assertEquals(cp4.getWindowOffset(), cp2.getWindowOffset());
    }
    finally
    {
      tmpWriter.close();
    }
  }

  @Test
  public void testCacheEntry_store_load_withroll() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(true);
    config.getRuntime().setHistorySize(10);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    List<String> sourceNames = Arrays.asList("source1", "source2", "source3");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);

    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    assertEquals("cp_source1-source2-source3", streamId);

    //clean up checkpoint files
    File checkpointFile = new File(checkpointDir, streamId + ".current");
    if (checkpointFile.exists())
    {
      assertTrue(checkpointFile.delete());
    }

    for (int i = 0; i < 15; ++i)
    {
      File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(
          checkpointDir, streamId + ".", i);
      if (f.exists())
      {
        assertTrue(f.delete());
      }
    }

    Checkpoint[] cp = new Checkpoint[15];

    //store checkpoints
    CacheEntry cacheEntry = checkpointProvider.new CacheEntry(streamId, null);
    for (int i = 0; i < 15; ++i)
    {
      cp[i] = new Checkpoint();
      cp[i].setWindowScn((long)i * i * i);
      cp[i].setWindowOffset(i * i);
      cp[i].setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);

      assertTrue(cacheEntry.setCheckpoint(cp[i]));
      assertTrue(checkpointFile.exists());

      CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
      Checkpoint cp2 = cacheEntry2.getCheckpoint();
      assertNotNull(cp2);
      //TODO need to use a Checkpoint.equals() method
      assertEquals(cp[i].getConsumptionMode(), cp2.getConsumptionMode());
      assertEquals(cp[i].getWindowScn(), cp2.getWindowScn());
      assertEquals(cp[i].getPrevScn(), cp2.getPrevScn());
      assertEquals(cp[i].getWindowOffset(), cp2.getWindowOffset());
    }

    //make sure we don't go over history size
    for (int i = 10; i < 15; ++i)
    {
      File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(
          checkpointDir, streamId, i);
      assertTrue(!f.exists());
    }

    //check correctness of history
    for (int i = 0; i < 10; ++i)
    {
      assertTrue(checkpointFile.delete());
      File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(
          checkpointDir, streamId + ".", i);
      assertTrue(f.renameTo(checkpointFile));

      CacheEntry cacheEntry2 = checkpointProvider.new CacheEntry(streamId, null);
      Checkpoint cp2 = cacheEntry2.getCheckpoint();
      assertNotNull(cp2);
      final int j = 13 -i;
      //TODO need to use a Checkpoint.equals() method
      assertEquals(cp[j].getConsumptionMode(), cp2.getConsumptionMode());
      assertEquals(cp[j].getWindowScn(), cp2.getWindowScn());
      assertEquals(cp[j].getPrevScn(), cp2.getPrevScn());
      assertEquals(cp[j].getWindowOffset(), cp2.getWindowOffset());
    }
  }

  @Test
  public void testCheckpointPersistance_singleSource() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(true);
    config.getRuntime().setHistorySize(10);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    List<String> sourceNames = Arrays.asList("source1", "source_2", "source-3");
    List<DatabusSubscription> subs = DatabusSubscription.createSubscriptionList(sourceNames);
    List<String> subsList = checkpointProvider.convertSubsToListOfStrings(subs);

    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(subsList);
    //assertEquals("correct streamid", "cp_source1-source_2-source_3", streamId);
    assertEquals("cp_source1-source_2-source_3", streamId);

    //clean up checkpoint files
    File checkpointFile = new File(checkpointDir, streamId + ".current");
    if (checkpointFile.exists())
    {
      assertTrue(checkpointFile.delete());
    }

    for (int i = 0; i < 15; ++i)
    {
      File f = FileSystemCheckpointPersistenceProvider.StaticConfig.generateCheckpointFile(
          checkpointDir, streamId + ".", i);
      if (f.exists())
      {
        assertTrue(f.delete());
      }
    }

    Checkpoint[] cp = new Checkpoint[15];

    Random rng = new Random();

    //store checkpoints
    for (int i = 0; i < 15; ++i)
    {
      cp[i] = new Checkpoint();
      cp[i].setWindowScn(rng.nextLong());
      cp[i].setWindowOffset(rng.nextInt());
      cp[i].setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);

      checkpointProvider.storeCheckpoint(sourceNames, cp[i]);
      assertTrue(checkpointFile.exists());

      Checkpoint cp2 = checkpointProvider.loadCheckpoint(sourceNames);
      assertNotNull(cp2);
      //TODO need to use a Checkpoint.equals() method
      assertEquals(cp[i].getConsumptionMode(), cp2.getConsumptionMode());
      assertEquals(cp[i].getWindowScn(), cp2.getWindowScn());
      assertEquals(cp[i].getPrevScn(), cp2.getPrevScn());
      assertEquals(cp[i].getWindowOffset(), cp2.getWindowOffset());
    }

    //create a new persister and read the last state
    FileSystemCheckpointPersistenceProvider checkpointProvider2 =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    Checkpoint cp2 = checkpointProvider2.loadCheckpoint(sourceNames);
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(cp[14].getConsumptionMode(), cp2.getConsumptionMode());
    assertEquals(cp[14].getWindowScn(), cp2.getWindowScn());
    assertEquals(cp[14].getPrevScn(), cp2.getPrevScn());
    assertEquals(cp[14].getWindowOffset(), cp2.getWindowOffset());
  }

  @Test
  public void testCheckpointPersistance_multiSource() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(true);
    config.getRuntime().setHistorySize(20);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    ArrayList<List<String>> sourceNames = new ArrayList<List<String>>();
    sourceNames.add(Arrays.asList("source1"));
    sourceNames.add(Arrays.asList("source2", "source1"));
    sourceNames.add(Arrays.asList("source3", "source2", "source1"));
    sourceNames.add(Arrays.asList("source4", "source3", "source2", "source1"));
    sourceNames.add(Arrays.asList("source5", "source4", "source3", "source2", "source1"));

    Checkpoint[] cp = new Checkpoint[15];

    Random rng = new Random();

    //store checkpoints
    for (int i = 0; i < 15; ++i)
    {
      cp[i] = new Checkpoint();
      cp[i].setWindowScn(rng.nextLong());
      cp[i].setWindowOffset(rng.nextInt());
      cp[i].setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);
    }

    for (int i = 0; i < 15; ++i)
    {
      for (int j = 0; j < 5; ++j)
      {
        final int cpN = (i + j) % 15;
        //List<DatabusSubscription> subs = DatabusSubscription.getSubscriptionList(sourceNames.get(j));
        checkpointProvider.storeCheckpoint(sourceNames.get(j), cp[cpN]);
        //checkpointProvider.storeCheckpointV3(subs, cp[cpN]);

        Checkpoint cp2 = checkpointProvider.loadCheckpoint(sourceNames.get(j));
        //Checkpoint cp2 = checkpointProvider.loadCheckpointV3(subs);
        assertNotNull(cp2);
        //TODO need to use a Checkpoint.equals() method
        assertEquals(cp[cpN].getConsumptionMode(), cp2.getConsumptionMode());
        assertEquals(cp[cpN].getWindowScn(), cp2.getWindowScn());
        assertEquals(cp[cpN].getPrevScn(), cp2.getPrevScn());
        assertEquals(cp[cpN].getWindowOffset(), cp2.getWindowOffset());
      }

      FileSystemCheckpointPersistenceProvider checkpointProvider2 =
        new FileSystemCheckpointPersistenceProvider(config, 2);

      for (int j = 0; j < 5; ++j)
      {
        final int cpN = (i + j) % 15;
        //List<DatabusSubscription> subs = DatabusSubscription.getSubscriptionList(sourceNames.get(j));
        Checkpoint cp2 = checkpointProvider2.loadCheckpoint(sourceNames.get(j));
        //Checkpoint cp2 = checkpointProvider2.loadCheckpointV3(subs);
        assertNotNull(cp2);
        //TODO need to use a Checkpoint.equals() method
        assertEquals(cp[cpN].getConsumptionMode(), cp2.getConsumptionMode());
        assertEquals(cp[cpN].getWindowScn(), cp2.getWindowScn());
        assertEquals(cp[cpN].getPrevScn(), cp2.getPrevScn());
        assertEquals(cp[cpN].getWindowOffset(), cp2.getWindowOffset());
      }
    }

  }

  @Test
  public void testCheckpointPersistance_manySources() throws Exception
  {
    File checkpointDir = new File("/tmp/databus2-checkpoints-test");

    FileSystemCheckpointPersistenceProvider.Config config =
      new FileSystemCheckpointPersistenceProvider.Config();

    config.setRootDirectory(checkpointDir.getAbsolutePath());
    config.getRuntime().setHistoryEnabled(true);
    config.getRuntime().setHistorySize(20);

    FileSystemCheckpointPersistenceProvider checkpointProvider =
      new FileSystemCheckpointPersistenceProvider(config, 2);

    //Test with com.linkedin.events. prefix
    List<String> sourceNames = new ArrayList<String>();
    int numberOfSources = 50;
    for (int i = 0; i < numberOfSources; ++i)
    {
      sourceNames.add("com.linkedin.events.VeryLongSource" + i);
    }

    String streamId = FileSystemCheckpointPersistenceProvider.calcStreamId(sourceNames);
    assertTrue(streamId.indexOf("com.linkedin.events") < 0);

    Random rng = new Random();

    //store checkpoint
    Checkpoint cp = new Checkpoint();
    cp.setWindowScn(rng.nextLong());
    cp.setWindowOffset(rng.nextInt());
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);

    checkpointProvider.storeCheckpoint(sourceNames, cp);
    Checkpoint cp2 = checkpointProvider.loadCheckpoint(sourceNames);
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(cp.getConsumptionMode(), cp2.getConsumptionMode());
    assertEquals(cp.getWindowScn(), cp2.getWindowScn());
    assertEquals(cp.getPrevScn(), cp2.getPrevScn());
    assertEquals(cp.getWindowOffset(), cp2.getWindowOffset());

    //Test without com.linkedin.events. prefix
    sourceNames.clear();
    for (int i = 0; i < numberOfSources; ++i)
    {
      sourceNames.add("com.linkedin.Some.Other.VeryLongSource" + i);
    }
    cp = new Checkpoint();
    cp.setWindowScn(rng.nextLong());
    cp.setWindowOffset(rng.nextInt());
    cp.setConsumptionMode(DbusClientMode.BOOTSTRAP_SNAPSHOT);

    checkpointProvider.storeCheckpoint(sourceNames, cp);
    cp2 = checkpointProvider.loadCheckpoint(sourceNames);
    assertNotNull(cp2);
    //TODO need to use a Checkpoint.equals() method
    assertEquals(cp.getConsumptionMode(), cp2.getConsumptionMode());
    assertEquals(cp.getWindowScn(), cp2.getWindowScn());
    assertEquals(cp.getPrevScn(), cp2.getPrevScn());
    assertEquals(cp.getWindowOffset(), cp2.getWindowOffset());
  }

}