// Copyright 2009 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package com.google.enterprise.connector.util.diffing; import com.google.enterprise.connector.spi.RepositoryDocumentException; import com.google.enterprise.connector.spi.RepositoryException; import com.google.enterprise.connector.spi.TraversalSchedule; import com.google.enterprise.connector.util.ChecksumGenerator; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; /** * A {@link DocumentSnapshotRepositoryMonitorManager} implementation. There is * one instance of this class per {@link DiffingConnector} created by Spring. * That instance gets signals from {@link DiffingConnectorTraversalManager} * to start (go from "cold" to "warm") and does so from scratch or from recovery * state. It creates the {@link SnapshotStore} instances and invokes * their recovery method. It creates and manages the * {@link DocumentSnapshotRepositoryMonitor} instances and passes guaranteed * checkpoints to these monitors. * * @since 2.8 */ public class DocumentSnapshotRepositoryMonitorManagerImpl implements DocumentSnapshotRepositoryMonitorManager { /** Maximum time to wait for background threads to terminate (in ms). */ private static final long MAX_SHUTDOWN_MS = 5000; private static final DocumentSink DOCUMENT_SINK = new LoggingDocumentSink(); private static final Logger LOG = Logger.getLogger( DocumentSnapshotRepositoryMonitorManagerImpl.class.getName()); private volatile TraversalSchedule traversalSchedule; private String makeMonitorNameFromStartPath(String startPath) { String monitorName = checksumGenerator.getChecksum(startPath); return monitorName; } private final List<Thread> threads = Collections.synchronizedList(new ArrayList<Thread>()); private final Map<String, DocumentSnapshotRepositoryMonitor> fileSystemMonitorsByName = Collections.synchronizedMap(new HashMap<String, DocumentSnapshotRepositoryMonitor>()); private boolean isRunning = false; // Monitor threads start in off state. private final List<? extends SnapshotRepository<? extends DocumentSnapshot>> repositories; private final File snapshotDir; private final ChecksumGenerator checksumGenerator; private final CheckpointAndChangeQueue checkpointAndChangeQueue; private final ChangeQueue changeQueue; private final DocumentSnapshotFactory documentSnapshotFactory; /** * Constructs {@link DocumentSnapshotRepositoryMonitorManagerImpl} * for the {@link DiffingConnector}. * * @param repositories a {@code List} of {@link SnapshotRepository * SnapshotRepositorys} * @param documentSnapshotFactory a {@link DocumentSnapshotFactory} * @param snapshotDir directory to store {@link SnapshotRepository} * @param checksumGenerator a {@link ChecksumGenerator} used to * detect changes in a document's content * @param changeQueue a {@link ChangeQueue} * @param checkpointAndChangeQueue a * {@link CheckpointAndChangeQueue} */ public DocumentSnapshotRepositoryMonitorManagerImpl( List<? extends SnapshotRepository< ? extends DocumentSnapshot>> repositories, DocumentSnapshotFactory documentSnapshotFactory, File snapshotDir, ChecksumGenerator checksumGenerator, ChangeQueue changeQueue, CheckpointAndChangeQueue checkpointAndChangeQueue) { this.repositories = repositories; this.documentSnapshotFactory = documentSnapshotFactory; this.snapshotDir = snapshotDir; this.checksumGenerator = checksumGenerator; this.changeQueue = changeQueue; this.checkpointAndChangeQueue = checkpointAndChangeQueue; } private void flagAllMonitorsToStop() { for (SnapshotRepository<? extends DocumentSnapshot> repository : repositories) { String monitorName = makeMonitorNameFromStartPath(repository.getName()); DocumentSnapshotRepositoryMonitor monitor = fileSystemMonitorsByName.get(monitorName); if (null != monitor) { monitor.shutdown(); } else { LOG.fine("Unable to stop non existent monitor thread for " + monitorName); } } } @Override public synchronized void stop() { for (Thread thread : threads) { thread.interrupt(); } for (Thread thread : threads) { try { thread.join(MAX_SHUTDOWN_MS); if (thread.isAlive()) { LOG.warning("failed to stop background thread: " + thread.getName()); } } catch (InterruptedException e) { // Mark this thread as interrupted so it can be dealt with later. Thread.currentThread().interrupt(); } } threads.clear(); /* in case thread.interrupt doesn't stop monitors */ flagAllMonitorsToStop(); fileSystemMonitorsByName.clear(); changeQueue.clear(); this.isRunning = false; } /* For each start path gets its monitor recovery files in state were monitor * can be started. */ private Map<String, SnapshotStore> recoverSnapshotStores( String connectorManagerCheckpoint, Map<String, MonitorCheckpoint> monitorPoints) throws IOException, SnapshotStoreException, InterruptedException { Map<String, SnapshotStore> snapshotStores = new HashMap<String, SnapshotStore>(); for (SnapshotRepository<? extends DocumentSnapshot> repository : repositories) { String monitorName = makeMonitorNameFromStartPath(repository.getName()); File dir = new File(snapshotDir, monitorName); boolean startEmpty = (connectorManagerCheckpoint == null) || (!monitorPoints.containsKey(monitorName)); if (startEmpty) { LOG.info("Deleting " + repository.getName() + " global checkpoint=" + connectorManagerCheckpoint + " monitor checkpoint=" + monitorPoints.get(monitorName)); delete(dir); } else { SnapshotStore.stitch(dir, monitorPoints.get(monitorName), documentSnapshotFactory); } SnapshotStore snapshotStore = new SnapshotStore(dir, documentSnapshotFactory); snapshotStores.put(monitorName, snapshotStore); } return snapshotStores; } /** Go from "cold" to "warm" including CheckpointAndChangeQueue. */ public void start(String connectorManagerCheckpoint) throws RepositoryException { try { checkpointAndChangeQueue.start(connectorManagerCheckpoint); } catch (IOException e) { throw new RepositoryException("Failed starting CheckpointAndChangeQueue.", e); } Map<String, MonitorCheckpoint> monitorPoints = checkpointAndChangeQueue.getMonitorRestartPoints(); Map<String, SnapshotStore> snapshotStores = null; try { snapshotStores = recoverSnapshotStores(connectorManagerCheckpoint, monitorPoints); } catch (SnapshotStoreException e) { throw new RepositoryException("Snapshot recovery failed.", e); } catch (IOException e) { throw new RepositoryException("Snapshot recovery failed.", e); } catch (InterruptedException e) { throw new RepositoryException("Snapshot recovery interrupted.", e); } startMonitorThreads(snapshotStores, monitorPoints); isRunning = true; } @Override public synchronized void clean() { LOG.info("Cleaning snapshot directory: " + snapshotDir.getAbsolutePath()); if (!delete(snapshotDir)) { LOG.warning("failed to delete snapshot directory: " + snapshotDir.getAbsolutePath()); } checkpointAndChangeQueue.clean(); } @Override public int getThreadCount() { int result = 0; for (Thread t : threads) { if (t.isAlive()) { result++; } } return result; } @Override public synchronized CheckpointAndChangeQueue getCheckpointAndChangeQueue() { return checkpointAndChangeQueue; } /** * Delete a file or directory. * * @param file * @return true if the file is deleted. */ private boolean delete(File file) { if (file.isDirectory()) { for (File contents : file.listFiles()) { delete(contents); } } return file.delete(); } /** * Creates a {@link DocumentSnapshotRepositoryMonitor} thread for the provided * folder. * * @throws RepositoryDocumentException if {@code startPath} is not readable, * or if there is any problem reading or writing snapshots. */ private Thread newMonitorThread( SnapshotRepository<? extends DocumentSnapshot> repository, SnapshotStore snapshotStore, MonitorCheckpoint startCp) throws RepositoryDocumentException { String monitorName = makeMonitorNameFromStartPath(repository.getName()); DocumentSnapshotRepositoryMonitor monitor = new DocumentSnapshotRepositoryMonitor(monitorName, repository, snapshotStore, changeQueue.newCallback(), DOCUMENT_SINK, startCp, documentSnapshotFactory); monitor.setTraversalSchedule(traversalSchedule); LOG.fine("Adding a new monitor for " + monitorName + ": " + monitor); fileSystemMonitorsByName.put(monitorName, monitor); return new Thread(monitor); } /** * Creates a {@link DocumentSnapshotRepositoryMonitor} thread for each * startPath. * * @throws RepositoryDocumentException if any of the threads cannot be * started. */ private void startMonitorThreads(Map<String, SnapshotStore> snapshotStores, Map<String, MonitorCheckpoint> monitorPoints) throws RepositoryDocumentException { for (SnapshotRepository<? extends DocumentSnapshot> repository : repositories) { String monitorName = makeMonitorNameFromStartPath(repository.getName()); SnapshotStore snapshotStore = snapshotStores.get(monitorName); Thread monitorThread = newMonitorThread(repository, snapshotStore, monitorPoints.get(monitorName)); threads.add(monitorThread); LOG.info("starting monitor for <" + repository.getName() + ">"); monitorThread.setName(repository.getName()); monitorThread.setDaemon(true); monitorThread.start(); } } @Override public synchronized boolean isRunning() { return isRunning; } @Override public void acceptGuarantees(Map<String, MonitorCheckpoint> guarantees) { for (Map.Entry<String, MonitorCheckpoint> entry : guarantees.entrySet()) { String monitorName = entry.getKey(); MonitorCheckpoint checkpoint = entry.getValue(); DocumentSnapshotRepositoryMonitor monitor = fileSystemMonitorsByName.get(monitorName); if (monitor != null) { // Signal is asynch. Let monitor figure out how to use. monitor.acceptGuarantee(checkpoint); } } } @Override public synchronized void setTraversalSchedule(TraversalSchedule traversalSchedule) { this.traversalSchedule = traversalSchedule; changeQueue.setSleepInterval(traversalSchedule.getRetryDelay() * 1000); for (SnapshotRepository<? extends DocumentSnapshot> repository : repositories) { String monitorName = makeMonitorNameFromStartPath(repository.getName()); DocumentSnapshotRepositoryMonitor monitor = fileSystemMonitorsByName.get(monitorName); if (monitor != null) { monitor.setTraversalSchedule(traversalSchedule); } else { // During initial startup, this is called before all monitor threads are // actually invoked. LOG.info("Unable to set traversal schedule for: " + monitorName); } } } }