// Copyright 2009 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.enterprise.connector.util.diffing; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Sets; import com.google.enterprise.connector.util.diffing.testing.TestDirectoryManager; import junit.framework.TestCase; import java.io.File; import java.io.IOException; import java.util.Set; public class SnapshotStoreTest extends TestCase { private File snapshotDir; private SnapshotStore store; @Override public void setUp() throws Exception { TestDirectoryManager testDirectoryManager = new TestDirectoryManager(this); File rootDir = testDirectoryManager.makeDirectory("rootDir"); snapshotDir = new File(rootDir, "snapshots"); store = new SnapshotStore(new File(rootDir, "snapshots"), new MockDocumentSnapshotFactory()); } /** * Make sure that if the store contains no snapshots, an initial empty * snapshot is returned. */ public void testEmptyDir() throws SnapshotStoreException { SnapshotReader in = store.openMostRecentSnapshot(); assertNull(in.read()); } /** * Make sure that if we write a snapshot and immediately read it, the contents * are correct. */ public void testWriteRead() throws Exception { SnapshotWriter out = store.openNewSnapshotWriter(); MockDocumentSnapshot before = new MockDocumentSnapshot("0", "extra.0"); out.write(before); SnapshotReader in = store.openMostRecentSnapshot(); DocumentSnapshot after = in.read(); assertEquals(before, after); assertNull(in.read()); store.close(in, out); } /** * Make sure that openMostRecentSnapshot always opens the most recent * snapshot. */ public void testSnapshotSorting() throws Exception { for (int k = 0; k < 10; ++k) { SnapshotWriter out = store.openNewSnapshotWriter(); String path = String.format("/foo/bar/%d", k); DocumentSnapshot before = new MockDocumentSnapshot(Integer.toString(k), "extra." + k); out.write(before); SnapshotReader in = store.openMostRecentSnapshot(); DocumentSnapshot after = in.read(); assertEquals(before, after); store.close(in, out); } } /** * Make sure that after a bunch of snapshots are created, only the last three * remain. */ public void testGarbageCollection() throws Exception { for (int k = 0; k < 10; ++k) { SnapshotWriter out = store.openNewSnapshotWriter(); long readSnapshotNum = Math.max(0, k - 1); MonitorCheckpoint cp = new MonitorCheckpoint("foo", readSnapshotNum, 2, 1); store.acceptGuarantee(cp); store.close(null, out); store.deleteOldSnapshots(); } assertSnapshotDirContains(ImmutableSet.of("snap.8", "snap.9", "snap.10")); } /** * Make sure that even if the only snapshots are old, we still keep * two of them. */ public void testMissingSnapshotFiles() throws Exception { for (int k = 0; k < 3; ++k) { SnapshotWriter out = store.openNewSnapshotWriter(); store.close(null, out); } assertSnapshotDirContains(ImmutableSet.of("snap.1", "snap.2", "snap.3")); MonitorCheckpoint cp = new MonitorCheckpoint("foo", 10, 2, 1); store.acceptGuarantee(cp); store.deleteOldSnapshots(); assertSnapshotDirContains(ImmutableSet.of("snap.2", "snap.3")); } private void assertSnapshotDirContains(Set<String> expected) { Set<String> filenames = Sets.newHashSet(); for (File f : snapshotDir.listFiles()) { filenames.add(f.getName()); } assertEquals(expected, filenames); } /** * Make sure that a new SnapshotStore recovers correctly from checkpoints. */ // TODO: add more recovery tests. public void testRecoveryBasics() throws IOException, SnapshotStoreException, InterruptedException { // Create the first snapshot with modification time 12345. SnapshotWriter ss1 = store.openNewSnapshotWriter(); writeRecords(ss1, "12345"); store.close(null, ss1); // Create a second snapshot with the same files, but modification time // 23456. SnapshotWriter ss2 = store.openNewSnapshotWriter(); writeRecords(ss2, "23456"); store.close(null, ss2); // Now pretend that the file-system monitor has scanned the first 7 records // from each snapshot and emitted changes for them. I.e., create a // checkpoint // as if the first 7 changes have been sent to the GSA. MonitorCheckpoint cp = new MonitorCheckpoint("foo", 1, 7, 7); SnapshotStore.stitch(snapshotDir, cp, new MockDocumentSnapshotFactory()); SnapshotStore after = new SnapshotStore(snapshotDir, new MockDocumentSnapshotFactory()); SnapshotReader reader = after.openMostRecentSnapshot(); assertEquals(3, reader.getSnapshotNumber()); // Snapshot should contain the first 7 records from snapshot 2 and the rest // from snapshot 1. for (int k = 0; k < 100; ++k) { DocumentSnapshot rec = reader.read(); assertNotNull(rec); String suffix = (k < 7) ? "23456" : "12345"; assertTrue(rec.getDocumentId().endsWith(suffix)); } store.close(reader, null); } public void testStitchWithInterrupt() throws Exception { // Create the first snapshot with modification time 12345. SnapshotWriter ss1 = store.openNewSnapshotWriter(); writeRecords(ss1, "12345"); store.close(null, ss1); // Create a second snapshot with the same files, but modification time // 23456. SnapshotWriter ss2 = store.openNewSnapshotWriter(); writeRecords(ss2, "23456"); store.close(null, ss2); // Now pretend that the file-system monitor has scanned the first 7 records // from each snapshot and emitted changes for them. I.e., create a // checkpoint // as if the first 7 changes have been sent to the GSA. MonitorCheckpoint cp = new MonitorCheckpoint("foo", 1, 7, 7); try { Thread.currentThread().interrupt(); SnapshotStore.stitch(snapshotDir, cp, new MockDocumentSnapshotFactory()); fail(); } catch (InterruptedException ie) { // Expected. } finally { assertFalse(Thread.interrupted()); } SnapshotStore after = new SnapshotStore(snapshotDir, new MockDocumentSnapshotFactory()); // Verify stitch did not create a new snapshot. SnapshotReader reader = after.openMostRecentSnapshot(); assertEquals(2, reader.getSnapshotNumber()); } /** * Write 100 records to {@code writer} with the specified {@code lastModified} * time. */ private void writeRecords(SnapshotWriter writer, String suffix) throws SnapshotWriterException { for (int k = 0; k < 100; ++k) { String path = String.format("/foo/bar/%d", k); DocumentSnapshot rec = new MockDocumentSnapshot(k + "." + suffix, "extra." + "k"); writer.write(rec); } } public void testTwoWriters() throws SnapshotStoreException { store.openNewSnapshotWriter(); try { store.openNewSnapshotWriter(); fail("opened second writer"); } catch (IllegalStateException expected) { assertEquals(expected.getMessage(), "There is already an active writer."); } } }