/*
* RapidMiner
*
* Copyright (C) 2001-2014 by RapidMiner and the contributors
*
* Complete list of developers available at our web site:
*
* http://rapidminer.com
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see http://www.gnu.org/licenses/.
*/
package com.rapidminer.repository;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import com.rapidminer.RapidMiner;
import com.rapidminer.io.process.XMLTools;
import com.rapidminer.operator.IOObject;
import com.rapidminer.operator.Operator;
import com.rapidminer.repository.db.DBRepository;
import com.rapidminer.repository.local.LocalRepository;
import com.rapidminer.repository.remote.RemoteRepository;
import com.rapidminer.repository.resource.ResourceRepository;
import com.rapidminer.tools.AbstractObservable;
import com.rapidminer.tools.FileSystemService;
import com.rapidminer.tools.I18N;
import com.rapidminer.tools.LogService;
import com.rapidminer.tools.Observer;
import com.rapidminer.tools.ProgressListener;
import com.rapidminer.tools.Tools;
import com.rapidminer.tools.XMLException;
/** Keeps static references to registered repositories and provides helper methods.
*
* Observers will be notified when repositories are added (with the repository passed as an argument to the
* {@link Observer#update(com.rapidminer.tools.Observable, Object)} method and when they are removed,
* in which case null is passed.
*
* @author Simon Fischer
*
*/
public class RepositoryManager extends AbstractObservable<Repository> {
public static final String SAMPLE_REPOSITORY_NAME = "Samples";
private static final Logger LOGGER = Logger.getLogger(RepositoryManager.class.getName());
private static RepositoryManager instance;
private static final Object INSTANCE_LOCK = new Object();
private static Repository sampleRepository;
private static final Map<RepositoryAccessor, RepositoryManager> CACHED_MANAGERS = new HashMap<RepositoryAccessor, RepositoryManager>();
private static final List<RepositoryFactory> FACTORIES = new LinkedList<RepositoryFactory>();
private final List<Repository> repositories = new LinkedList<Repository>();
public static RepositoryManager getInstance(RepositoryAccessor repositoryAccessor) {
synchronized (INSTANCE_LOCK) {
if (instance == null) {
init();
}
if (repositoryAccessor != null) {
RepositoryManager manager = CACHED_MANAGERS.get(repositoryAccessor);
if (manager == null) {
manager = new RepositoryManager(instance);
for (RepositoryFactory factory : FACTORIES) {
for (Repository repos : factory.createRepositoriesFor(repositoryAccessor)) {
manager.repositories.add(repos);
}
}
CACHED_MANAGERS.put(repositoryAccessor, manager);
}
return manager;
}
}
return instance;
}
private RepositoryManager(RepositoryManager cloned) {
this.repositories.addAll(cloned.repositories);
}
private RepositoryManager() {
if (sampleRepository == null) {
sampleRepository = new ResourceRepository(SAMPLE_REPOSITORY_NAME, "samples");
}
repositories.add(sampleRepository);
repositories.add(new DBRepository());
final String homeUrl = System.getProperty(RapidMiner.PROPERTY_HOME_REPOSITORY_URL);
if (homeUrl != null) {
try {
String username = System.getProperty(RapidMiner.PROPERTY_HOME_REPOSITORY_USER);
String password = System.getProperty(RapidMiner.PROPERTY_HOME_REPOSITORY_PASSWORD);
char[] passwordChars = null;
if (password != null) {
passwordChars = password.toCharArray();
}
RemoteRepository homeRepository = new RemoteRepository(new URL(homeUrl), "Home", username, passwordChars, true);
repositories.add(homeRepository);
//LogService.getRoot().config("Adding home repository " + homeUrl + ".");
LogService.getRoot().log(Level.CONFIG, "com.rapidminer.repository.RepositoryManager.adding_home_repository", homeUrl);
} catch (MalformedURLException e) {
//LogService.getRoot().log(Level.WARNING, "Illegal repository URL " + homeUrl + ": " + e, e);
LogService.getRoot().log(Level.WARNING,
I18N.getMessage(LogService.getRoot().getResourceBundle(),
"com.rapidminer.repository.RepositoryManager.illegal_repository_url",
homeUrl, e),
e);
}
}
load();
}
public static void init() {
synchronized (INSTANCE_LOCK) {
instance = new RepositoryManager();
instance.postInstall();
}
}
private void postInstall() {
for (Repository repository : getRepositories()) {
repository.postInstall();
}
}
public static void registerFactory(RepositoryFactory factory) {
synchronized (INSTANCE_LOCK) {
FACTORIES.add(factory);
}
}
/** Registers a repository.
*
* @see #removeRepository(Repository) */
public void addRepository(Repository repository) {
LOGGER.config("Adding repository " + repository.getName());
repositories.add(repository);
if (instance != null) {
// we cannot call post install during init(). The reason is that
// post install may access RepositoryManager.getInstance() which will be null and hence
// trigger further recursive, endless calls to init()
repository.postInstall();
save();
}
fireUpdate(repository);
}
/** Removes a registered repository.
*
* @see #addRepository(Repository) */
public void removeRepository(Repository repository) {
repository.preRemove();
repositories.remove(repository);
fireUpdate(null);
}
public List<Repository> getRepositories() {
return Collections.unmodifiableList(repositories);
}
/** Gets a registered ({@link #addRepository(Repository)} repository by {@link Repository#getName()}*/
public Repository getRepository(String name) throws RepositoryException {
for (Repository repos : repositories) {
if (repos.getName().equals(name)) {
return repos;
}
}
throw new RepositoryException("Requested repository " + name + " does not exist.");
}
/** Gets a list of all registered repositories inheriting from {@link RemoteRepository}. */
public List<RemoteRepository> getRemoteRepositories() {
List<RemoteRepository> result = new LinkedList<RemoteRepository>();
for (Repository repos : getRepositories()) {
if (repos instanceof RemoteRepository) {
result.add((RemoteRepository) repos);
}
}
return result;
}
private File getConfigFile() {
return FileSystemService.getUserConfigFile("repositories.xml");
}
/** Loads the XML configuration file.
*
* @see #save() */
public void load() {
if (!RapidMiner.getExecutionMode().canAccessFilesystem()) {
LOGGER.info("Cannot access file system in execution mode " + RapidMiner.getExecutionMode() + ". Not loading repositories.");
return;
}
File file = getConfigFile();
if (file.exists()) {
LOGGER.config("Loading repositories from " + file);
try {
Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(file);
if (!doc.getDocumentElement().getTagName().equals("repositories")) {
LOGGER.warning("Broken repositories file. Root element must be <reposities>.");
return;
}
NodeList list = doc.getDocumentElement().getChildNodes();
for (int i = 0; i < list.getLength(); i++) {
if (list.item(i) instanceof Element) {
Element element = (Element) list.item(i);
if ("localRepository".equals(element.getTagName())) {
addRepository(LocalRepository.fromXML(element));
} else if ("remoteRepository".equals(element.getTagName())) {
addRepository(RemoteRepository.fromXML(element));
} else {
LOGGER.warning("Unknown tag: " + element.getTagName());
}
}
}
} catch (Exception e) {
LOGGER.log(Level.WARNING, "Cannot read repository configuration file '" + file + "': " + e, e);
}
}
}
public void createRepositoryIfNoneIsDefined() {
boolean empty = true;
// check if we have at least one repository that is not pre-defined
for (Repository repository : repositories) {
if (!(repository instanceof ResourceRepository) && !(repository instanceof DBRepository)) {
empty = false;
break;
}
}
if (empty) {
// SwingTools.showMessageDialog("please_create_repository");
// NewRepositoryDialog.createNew();
try {
LocalRepository defaultRepo = new LocalRepository("Local Repository");
RepositoryManager.getInstance(null).addRepository(defaultRepo);
defaultRepo.createFolder("data");
defaultRepo.createFolder("processes");
} catch (RepositoryException e) {
LogService.getRoot().log(Level.WARNING, "com.rapidminer.repository.RepositoryManager.failed_to_create_default", e);
}
}
}
/** Stores the XML configuration file.
* @see #load()
*/
public void save() {
if (!RapidMiner.getExecutionMode().canAccessFilesystem()) {
LOGGER.config("Cannot access file system in execution mode " + RapidMiner.getExecutionMode() + ". Not saving repositories.");
return;
}
Document doc;
try {
doc = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument();
} catch (ParserConfigurationException e) {
LOGGER.log(Level.WARNING, "Cannot save repositories: " + e, e);
return;
}
Element root = doc.createElement("repositories");
doc.appendChild(root);
for (Repository repository : getRepositories()) {
if (repository.shouldSave()) {
Element repositoryElement = repository.createXML(doc);
if (repositoryElement != null) {
root.appendChild(repositoryElement);
}
}
}
try {
XMLTools.stream(doc, getConfigFile(), null);
} catch (XMLException e) {
LOGGER.log(Level.WARNING, "Cannot save repositories: " + e, e);
}
}
/** Stores an IOObject at the given location. Creates entries if they don't exist. */
public IOObject store(IOObject ioobject, RepositoryLocation location, Operator callingOperator) throws RepositoryException {
return store(ioobject, location, callingOperator, null);
}
/** Stores an IOObject at the given location. Creates entries if they don't exist. */
public IOObject store(IOObject ioobject, RepositoryLocation location, Operator callingOperator, ProgressListener progressListener) throws RepositoryException {
Entry entry = location.locateEntry();
if (entry == null) {
RepositoryLocation parentLocation = location.parent();
if (parentLocation != null) {
String childName = location.getName();
Entry parentEntry = parentLocation.locateEntry();
Folder parentFolder;
if (parentEntry != null) {
if (parentEntry instanceof Folder) {
parentFolder = (Folder) parentEntry;
} else {
throw new RepositoryException("Parent '" + parentLocation + "' of '" + location + "' is not a folder.");
}
} else {
parentFolder = parentLocation.createFoldersRecursively();
}
parentFolder.createIOObjectEntry(childName, ioobject, callingOperator, progressListener);
return ioobject;
} else {
throw new RepositoryException("Entry '" + location + "' does not exist.");
}
} else if (entry instanceof IOObjectEntry) {
((IOObjectEntry) entry).storeData(ioobject, callingOperator, null);
return ioobject;
} else {
throw new RepositoryException("Entry '" + location + "' is not a data entry, but " + entry.getType());
}
}
/** Gets the referenced blob entry. Creates a new one if it does not exist. */
public BlobEntry getOrCreateBlob(RepositoryLocation location) throws RepositoryException {
Entry entry = location.locateEntry();
if (entry == null) {
RepositoryLocation parentLocation = location.parent();
if (parentLocation != null) {
String childName = location.getName();
Entry parentEntry = parentLocation.locateEntry();
Folder parentFolder;
if (parentEntry != null) {
if (parentEntry instanceof Folder) {
parentFolder = (Folder) parentEntry;
} else {
throw new RepositoryException("Parent '" + parentLocation + "' of '" + location + "' is not a folder.");
}
} else {
parentFolder = parentLocation.createFoldersRecursively();
}
return parentFolder.createBlobEntry(childName);
} else {
throw new RepositoryException("Entry '" + location + "' does not exist.");
}
} else if (entry instanceof BlobEntry) {
return (BlobEntry) entry;
} else {
throw new RepositoryException("Entry '" + location + "' is not a blob entry, but a " + entry.getType());
}
}
/** Saves the configuration file. */
public static void shutdown() {
if (instance != null) {
instance.save();
}
}
/** Copies an entry to a given destination folder. */
public void copy(RepositoryLocation source, Folder destination, ProgressListener listener) throws RepositoryException {
copy(source, destination, null, listener);
}
/** Copies an entry to a given destination folder with the name newName. If newName is null the old name will be kept. */
public void copy(RepositoryLocation source, Folder destination, String newName, ProgressListener listener) throws RepositoryException {
if (listener != null) {
listener.setTotal(100000);
listener.setCompleted(0);
}
try {
copy(source, destination, newName, listener, 0, 100000);
} finally {
if (listener != null) {
listener.complete();
}
}
}
private void copy(RepositoryLocation source, Folder destination, String newName, ProgressListener listener, int minProgress, int maxProgress) throws RepositoryException {
Entry entry = source.locateEntry();
if (entry == null) {
throw new RepositoryException("No such entry: " + source);
}
copy(entry, destination, newName, listener, minProgress, maxProgress);
}
private void copy(Entry entry, Folder destination, String newName, ProgressListener listener, int minProgress, int maxProgress) throws RepositoryException {
if (listener != null) {
listener.setMessage(entry.getName());
}
if (newName == null) {
newName = entry.getName();
}
String originalName = newName;
if (destination.containsEntry(newName)) {
newName = "Copy of " + newName;
int i = 2;
while (destination.containsEntry(newName)) {
newName = "Copy " + (i++) + " of " + originalName;
}
}
if (entry instanceof ProcessEntry) {
ProcessEntry pe = (ProcessEntry) entry;
String xml = pe.retrieveXML();
if (listener != null) {
listener.setCompleted((minProgress + maxProgress) / 2);
}
destination.createProcessEntry(newName, xml);
if (listener != null) {
listener.setCompleted(maxProgress);
}
} else if (entry instanceof IOObjectEntry) {
IOObjectEntry iooe = (IOObjectEntry) entry;
IOObject original = iooe.retrieveData(null);
if (listener != null) {
listener.setCompleted((minProgress + maxProgress) / 2);
}
destination.createIOObjectEntry(newName, original, null, null);
if (listener != null) {
listener.setCompleted(maxProgress);
}
} else if (entry instanceof BlobEntry) {
BlobEntry blob = (BlobEntry) entry;
BlobEntry target = destination.createBlobEntry(newName);
try {
InputStream in = blob.openInputStream();
String mimeType = blob.getMimeType();
OutputStream out = target.openOutputStream(mimeType);
Tools.copyStreamSynchronously(in, out, true);
if (listener != null) {
listener.setCompleted(maxProgress);
}
} catch (IOException e) {
throw new RepositoryException(e);
}
} else if (entry instanceof Folder) {
String sourceAbsolutePath = entry.getLocation().getAbsoluteLocation();
String destinationAbsolutePath = destination.getLocation().getAbsoluteLocation();
// make sure same folder moves are forbidden
if (sourceAbsolutePath.equals(destinationAbsolutePath)) {
throw new RepositoryException(I18N.getMessage(I18N.getErrorBundle(), "repository.repository_copy_same_folder"));
}
// make sure moving parent folder into subfolder is forbidden
if (destinationAbsolutePath.contains(sourceAbsolutePath)) {
throw new RepositoryException(I18N.getMessage(I18N.getErrorBundle(), "repository.repository_copy_into_subfolder"));
}
Folder destinationFolder = destination.createFolder(newName);
List<Entry> allChildren = new LinkedList<Entry>();
allChildren.addAll(((Folder) entry).getSubfolders());
allChildren.addAll(((Folder) entry).getDataEntries());
final int count = allChildren.size();
int progressStart = minProgress;
int progressDiff = maxProgress - minProgress;
int i = 0;
for (Entry child : allChildren) {
copy(child, destinationFolder, null, listener, progressStart + i * progressDiff / count, progressStart + (i + 1) * progressDiff / count);
i++;
}
} else {
throw new RepositoryException("Cannot copy entry of type " + entry.getType());
}
}
/** Moves an entry to a given destination folder. */
public void move(RepositoryLocation source, Folder destination, ProgressListener listener) throws RepositoryException {
move(source, destination, null, listener);
}
/** Moves an entry to a given destination folder with the name newName. */
public void move(RepositoryLocation source, Folder destination, String newName, ProgressListener listener) throws RepositoryException {
Entry entry = source.locateEntry();
if (entry == null) {
throw new RepositoryException("No such entry: " + source);
} else {
String sourceAbsolutePath = source.getAbsoluteLocation();
String destinationAbsolutePath;
if (!(entry instanceof Folder)) {
destinationAbsolutePath = destination.getLocation().getAbsoluteLocation() + RepositoryLocation.SEPARATOR + source.getName();
} else {
destinationAbsolutePath = destination.getLocation().getAbsoluteLocation();
}
// make sure same folder moves are forbidden
if (sourceAbsolutePath.equals(destinationAbsolutePath)) {
throw new RepositoryException(I18N.getMessage(I18N.getErrorBundle(), "repository.repository_move_same_folder"));
}
// make sure moving parent folder into subfolder is forbidden
if (destinationAbsolutePath.contains(sourceAbsolutePath)) {
throw new RepositoryException(I18N.getMessage(I18N.getErrorBundle(), "repository.repository_move_into_subfolder"));
}
if (destination.getLocation().getRepository() != source.getRepository()) {
copy(source, destination, newName, listener);
entry.delete();
} else {
String effectiveNewName = newName != null ? newName : entry.getName();
Entry toDeleteEntry = null;
for (Folder folderEntry : destination.getSubfolders()) {
if (folderEntry.getName().equals(effectiveNewName)) {
throw new RepositoryException(I18N.getMessage(I18N.getErrorBundle(), "repository.repository_folder_already_exists", effectiveNewName));
}
}
if (destination.containsEntry(effectiveNewName)) {
for (DataEntry dataEntry : destination.getDataEntries()) {
if (dataEntry.getName().equals(effectiveNewName)) {
toDeleteEntry = dataEntry;
}
}
if (toDeleteEntry != null) {
toDeleteEntry.delete();
}
}
if (listener != null) {
listener.setTotal(100);
listener.setCompleted(10);
}
if (newName == null) {
entry.move(destination);
} else {
entry.move(destination, newName);
}
if (listener != null) {
listener.setCompleted(100);
listener.complete();
}
}
}
}
/** Looks up the entry with the given path in the given repository.
* This method will return null when it finds a folder that blocks (has not yet loaded
* all its data) AND failIfBlocks is true.
*
* This method can be used as a first approach to locate an entry and fall back
* to a more expensive solution when this fails.
*
*/
public Entry locate(Repository repository, String path, boolean failIfBlocks) throws RepositoryException {
if (path.startsWith("" + RepositoryLocation.SEPARATOR)) {
path = path.substring(1);
}
if (path.equals("")) {
return repository;
}
String[] splitted = path.split("" + RepositoryLocation.SEPARATOR);
Folder folder = repository;
int index = 0;
while (true) {
if (failIfBlocks && folder.willBlock()) {
return null;
}
if (index == splitted.length - 1) {
int retryCount = 0;
while (retryCount <= 1) {
List<Entry> all = new LinkedList<Entry>();
all.addAll(folder.getSubfolders());
all.addAll(folder.getDataEntries());
for (Entry child : all) {
if (child.getName().equals(splitted[index])) {
return child;
}
}
// missed entry -> refresh and try again
if (folder.canRefreshChild(splitted[index])) {
folder.refresh();
} else {
break;
}
retryCount++;
}
return null;
} else {
int retryCount = 0;
boolean found = false;
while (retryCount <= 1) {
for (Folder subfolder : folder.getSubfolders()) {
if (subfolder.getName().equals(splitted[index])) {
folder = subfolder;
found = true;
break;
}
}
if (found) {
// found in 1st round
break;
} else {
// missed entry -> refresh and try again
if (folder.canRefreshChild(splitted[index])) {
folder.refresh();
} else {
break;
}
retryCount++;
}
}
if (!found) {
return null;
}
}
index++;
}
}
/** Returns the repository containing the RapidMiner sample processes. */
public Repository getSampleRepository() {
return sampleRepository;
}
/** Visitor pattern for repositories. Callbacks to the visitor will be made
* only for matching types. (Recursion happens also if the type is not
* a Folder.
* @throws RepositoryException
* */
public <T extends Entry> void walk(Entry start, RepositoryVisitor<T> visitor, Class<T> visitedType) throws RepositoryException {
boolean continueChildren = true;
if (visitedType.isInstance(start)) {
continueChildren &= visitor.visit(visitedType.cast(start));
}
if (continueChildren && (start instanceof Folder)) {
Folder folder = (Folder) start;
for (Entry child : folder.getDataEntries()) {
walk(child, visitor, visitedType);
}
for (Folder childFolder : folder.getSubfolders()) {
walk(childFolder, visitor, visitedType);
}
}
}
}