/*
* ModeShape (http://www.modeshape.org)
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.modeshape.connector.filesystem;
import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE;
import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.WatchEvent;
import java.nio.file.WatchKey;
import java.nio.file.WatchService;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import java.util.concurrent.Callable;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.regex.Pattern;
import javax.jcr.NamespaceRegistry;
import javax.jcr.RepositoryException;
import org.modeshape.common.util.FileUtil;
import org.modeshape.common.util.IoUtil;
import org.modeshape.common.util.NamedThreadFactory;
import org.modeshape.common.util.SecureHash;
import org.modeshape.common.util.StringUtil;
import org.modeshape.jcr.JcrI18n;
import org.modeshape.jcr.JcrLexicon;
import org.modeshape.jcr.JcrNtLexicon;
import org.modeshape.jcr.RepositoryConfiguration;
import org.modeshape.jcr.api.Binary;
import org.modeshape.jcr.api.nodetype.NodeTypeManager;
import org.modeshape.jcr.api.value.DateTime;
import org.modeshape.jcr.cache.DocumentStoreException;
import org.modeshape.jcr.federation.NoExtraPropertiesStorage;
import org.modeshape.jcr.spi.federation.Connector;
import org.modeshape.jcr.spi.federation.ConnectorChangeSet;
import org.modeshape.jcr.spi.federation.ConnectorException;
import org.modeshape.jcr.spi.federation.DocumentChanges;
import org.modeshape.jcr.spi.federation.DocumentReader;
import org.modeshape.jcr.spi.federation.DocumentWriter;
import org.modeshape.jcr.spi.federation.PageKey;
import org.modeshape.jcr.spi.federation.Pageable;
import org.modeshape.jcr.spi.federation.WritableConnector;
import org.modeshape.jcr.value.BinaryValue;
import org.modeshape.jcr.value.Name;
import org.modeshape.jcr.value.Property;
import org.modeshape.jcr.value.binary.ExternalBinaryValue;
import org.modeshape.jcr.value.binary.UrlBinaryValue;
import org.modeshape.schematic.document.Document;
/**
* {@link Connector} implementation that exposes a single directory on the local file system. This connector has several
* properties that must be configured via the {@link RepositoryConfiguration}:
* <ul>
* <li><strong><code>directoryPath</code></strong> - The path to the file or folder that is to be accessed by this connector.</li>
* <li><strong><code>readOnly</code></strong> - A boolean flag that specifies whether this source can create/modify/remove files
* and directories on the file system to reflect changes in the JCR content. By default, sources are not read-only.</li>
* <li><strong><code>addMimeTypeMixin</code></strong> - A boolean flag that specifies whether this connector should add the
* 'mix:mimeType' mixin to the 'nt:resource' nodes to include the 'jcr:mimeType' property. If set to <code>true</code>, the MIME
* type is computed immediately when the 'nt:resource' node is accessed, which might be expensive for larger files. This is
* <code>false</code> by default.</li>
* <li><strong><code>extraPropertyStorage</code></strong> - An optional string flag that specifies how this source handles "extra"
* properties that are not stored via file system attributes. See {@link #extraPropertiesStorage} for details. By default, extra
* properties are stored in the same DB that the repository uses.</li>
* <li><strong><code>exclusionPattern</code></strong> - Optional property that specifies a regular expression that is used to
* determine which files and folders in the underlying file system are not exposed through this connector. Files and folders with
* a name that matches the provided regular expression will <i>not</i> be exposed by this source.</li>
* <li><strong><code>inclusionPattern</code></strong> - Optional property that specifies a regular expression that is used to
* determine which files and folders in the underlying file system are exposed through this connector. Files and folders with a
* name that matches the provided regular expression will be exposed by this source.</li>
* </ul>
* Inclusion and exclusion patterns can be used separately or in combination. For example, consider these cases:
* <table cellspacing="0" cellpadding="1" border="1">
* <tr>
* <th>Inclusion Pattern</th>
* <th>Exclusion Pattern</th>
* <th>Examples</th>
* </tr>
* <tr>
* <td>(.+)\\.txt$</td>
* <td></td>
* <td>Includes only files and directories whose names end in "<code>.txt</code>" (e.g., "<code>something.txt</code>" ), but does
* not include files and other folders such as "<code>something.jar</code>" or "<code>something.txt.zip</code>".</td>
* </tr>
* <tr>
* <td>(.+)\\.txt$</td>
* <td>my.txt</td>
* <td>Includes only files and directories whose names end in "<code>.txt</code>" (e.g., "<code>something.txt</code>" ) with the
* exception of "<code>my.txt</code>", and does not include files and other folders such as "<code>something.jar</code>" or "
* <code>something.txt.zip</code>".</td>
* </tr>
* <tr>
* <td>my.txt</td>
* <td>.+</td>
* <td>Excludes all files and directories except any named "<code>my.txt</code>".</td>
* </tr>
* </table>
*/
public class FileSystemConnector extends WritableConnector implements Pageable {
protected static final int DEFAULT_PAGE_SIZE = 20;
private static final String FILE_SEPARATOR = System.getProperty("file.separator");
private static final String DELIMITER = "/";
private static final String NT_FOLDER = "nt:folder";
private static final String NT_FILE = "nt:file";
private static final String NT_RESOURCE = "nt:resource";
private static final String MIX_MIME_TYPE = "mix:mimeType";
private static final String JCR_PRIMARY_TYPE = "jcr:primaryType";
private static final String JCR_DATA = "jcr:data";
private static final String JCR_MIME_TYPE = "jcr:mimeType";
private static final String JCR_CREATED = "jcr:created";
private static final String JCR_CREATED_BY = "jcr:createdBy";
private static final String JCR_LAST_MODIFIED = "jcr:lastModified";
private static final String JCR_LAST_MODIFIED_BY = "jcr:lastModifiedBy";
private static final String JCR_CONTENT = "jcr:content";
private static final String JCR_CONTENT_SUFFIX = DELIMITER + JCR_CONTENT;
private static final int JCR_CONTENT_SUFFIX_LENGTH = JCR_CONTENT_SUFFIX.length();
private static final String EXTRA_PROPERTIES_JSON = "json";
private static final String EXTRA_PROPERTIES_LEGACY = "legacy";
private static final String EXTRA_PROPERTIES_NONE = "none";
/**
* The string path for a {@link File} object that represents the top-level directory accessed by this connector. This is set
* via reflection and is required for this connector.
*/
private String directoryPath;
private File directory;
/**
* A string that is created in the {@link #initialize(NamespaceRegistry, NodeTypeManager)} method that represents the absolute
* path to the {@link #directory}. This path is removed from an absolute path of a file to obtain the ID of the node.
*/
private String directoryAbsolutePath;
private int directoryAbsolutePathLength;
/**
* A boolean flag that specifies whether this connector should add the 'mix:mimeType' mixin to the 'nt:resource' nodes to
* include the 'jcr:mimeType' property. If set to <code>true</code>, the MIME type is computed immediately when the
* 'nt:resource' node is accessed, which might be expensive for larger files. This is <code>false</code> by default.
*/
private boolean addMimeTypeMixin = false;
/**
* The regular expression that, if matched by a file or folder, indicates that the file or folder should be included.
*/
private String inclusionPattern;
/**
* The regular expression that, if matched by a file or folder, indicates that the file or folder should be ignored.
*/
private String exclusionPattern;
/**
* The maximum number of children a folder will expose at any given time.
*/
private int pageSize = DEFAULT_PAGE_SIZE;
/**
* The {@link FilenameFilter} implementation that is instantiated in the
* {@link #initialize(NamespaceRegistry, NodeTypeManager)} method.
*/
private InclusionExclusionFilenameFilter filenameFilter;
/**
* Flag which enables or disables external monitoring, meaning firing repository events when changes to the file system occur.
*/
private boolean enableEvents = false;
/**
* The thread which performs external monitoring
*/
private Future<Void> monitoringTask;
/**
* A string that specifies how the "extra" properties are to be stored, where an "extra" property is any JCR property that
* cannot be stored natively on the file system as file attributes. This field is set via reflection, and the value is
* expected to be one of these valid values:
* <ul>
* <li>"<code>store</code>" - Any extra properties are stored in the same DB where the content is stored. This
* is the default and is used if the actual value doesn't match any of the other accepted values.</li>
* <li>"<code>json</code>" - Any extra properties are stored in a JSON file next to the file or directory.</li>
* <li>"<code>legacy</code>" - Any extra properties are stored in a ModeShape 2.x-compatible file next to the file or
* directory. This is generally discouraged unless you were using ModeShape 2.x and have a directory structure that already
* contains these files.</li>
* <li>"<code>none</code>" - An extra properties that prevents the storage of extra properties by throwing an exception when
* such extra properties are defined.</li>
* </ul>
*/
private String extraPropertiesStorage;
/**
* A boolean which determines whether for external binary values (i.e. {@link UrlBinaryValue}) the SHA1 is computed based on
* the content of the file itself or whether it's computed based on the URL string. This is {@code true} by default, but if
* the connector needs to deal with very large values it might be worth turning off.
*/
private boolean contentBasedSha1 = true;
private NamespaceRegistry registry;
@Override
public void initialize( NamespaceRegistry registry,
NodeTypeManager nodeTypeManager ) throws RepositoryException, IOException {
super.initialize(registry, nodeTypeManager);
this.registry = registry;
// Initialize the directory path field that has been set via reflection when this method is called...
checkFieldNotNull(directoryPath, "directoryPath");
directory = new File(directoryPath);
if (!directory.exists() || !directory.isDirectory()) {
String msg = JcrI18n.fileConnectorTopLevelDirectoryMissingOrCannotBeRead.text(getSourceName(), "directoryPath");
throw new RepositoryException(msg);
}
if (!directory.canRead() && !directory.setReadable(true)) {
String msg = JcrI18n.fileConnectorTopLevelDirectoryMissingOrCannotBeRead.text(getSourceName(), "directoryPath");
throw new RepositoryException(msg);
}
directoryAbsolutePath = directory.getAbsolutePath();
if (!directoryAbsolutePath.endsWith(FILE_SEPARATOR)) directoryAbsolutePath = directoryAbsolutePath + FILE_SEPARATOR;
directoryAbsolutePathLength = directoryAbsolutePath.length() - FILE_SEPARATOR.length(); // does NOT include the separator
// Initialize the filename filter ...
filenameFilter = new InclusionExclusionFilenameFilter();
if (exclusionPattern != null) filenameFilter.setExclusionPattern(exclusionPattern);
if (inclusionPattern != null) filenameFilter.setInclusionPattern(inclusionPattern);
// Set up the extra properties storage ...
if (EXTRA_PROPERTIES_JSON.equalsIgnoreCase(extraPropertiesStorage)) {
JsonSidecarExtraPropertyStore store = new JsonSidecarExtraPropertyStore(this, translator());
setExtraPropertiesStore(store);
filenameFilter.setExtraPropertiesExclusionPattern(store.getExclusionPattern());
} else if (EXTRA_PROPERTIES_LEGACY.equalsIgnoreCase(extraPropertiesStorage)) {
LegacySidecarExtraPropertyStore store = new LegacySidecarExtraPropertyStore(this);
setExtraPropertiesStore(store);
filenameFilter.setExtraPropertiesExclusionPattern(store.getExclusionPattern());
} else if (EXTRA_PROPERTIES_NONE.equalsIgnoreCase(extraPropertiesStorage)) {
setExtraPropertiesStore(new NoExtraPropertiesStorage(this));
}
// otherwise use the default extra properties storage
if (enableEvents) {
try {
WatchService watchService = FileSystems.getDefault().newWatchService();
monitoringTask = Executors.newSingleThreadExecutor(new NamedThreadFactory("modeshape-fs-connector-monitor"))
.submit(new MonitoringTask(watchService, this, Paths.get(directoryAbsolutePath)));
} catch (UnsupportedOperationException e) {
log().warn("Unable to to turn on monitoring, because it is not supported on this OS");
}
}
}
@Override
public void shutdown() {
if (enableEvents && monitoringTask != null) {
try {
monitoringTask.cancel(true);
} finally {
monitoringTask = null;
}
}
}
/**
* Get the namespace registry.
*
* @return the namespace registry; never null
*/
NamespaceRegistry registry() {
return registry;
}
/**
* Utility method for determining if the supplied identifier is for the "jcr:content" child node of a file. * Subclasses may
* override this method to change the format of the identifiers, but in that case should also override the
* {@link #fileFor(String)}, {@link #isRoot(String)}, and {@link #idFor(File)} methods.
*
* @param id the identifier; may not be null
* @return true if the identifier signals the "jcr:content" child node of a file, or false otherwise
* @see #isRoot(String)
* @see #fileFor(String)
* @see #idFor(File)
*/
protected boolean isContentNode( String id ) {
return id.endsWith(JCR_CONTENT_SUFFIX);
}
/**
* Utility method for obtaining the {@link File} object that corresponds to the supplied identifier. Subclasses may override
* this method to change the format of the identifiers, but in that case should also override the {@link #isRoot(String)},
* {@link #isContentNode(String)}, and {@link #idFor(File)} methods.
*
* @param id the identifier; may not be null
* @return the File object for the given identifier
* @see #isRoot(String)
* @see #isContentNode(String)
* @see #idFor(File)
*/
protected File fileFor( String id ) {
assert id.startsWith(DELIMITER);
if (id.endsWith(DELIMITER)) {
id = id.substring(0, id.length() - DELIMITER.length());
}
if (isContentNode(id)) {
id = id.substring(0, id.length() - JCR_CONTENT_SUFFIX_LENGTH);
}
return new File(directory, id);
}
/**
* Utility method for determining if the node identifier is the identifier of the root node in this external source.
* Subclasses may override this method to change the format of the identifiers, but in that case should also override the
* {@link #fileFor(String)}, {@link #isContentNode(String)}, and {@link #idFor(File)} methods.
*
* @param id the identifier; may not be null
* @return true if the identifier is for the root of this source, or false otherwise
* @see #isContentNode(String)
* @see #fileFor(String)
* @see #idFor(File)
*/
protected boolean isRoot( String id ) {
return DELIMITER.equals(id);
}
/**
* Utility method for determining the node identifier for the supplied file. Subclasses may override this method to change the
* format of the identifiers, but in that case should also override the {@link #fileFor(String)},
* {@link #isContentNode(String)}, and {@link #isRoot(String)} methods.
*
* @param file the file; may not be null
* @return the node identifier; never null
* @see #isRoot(String)
* @see #isContentNode(String)
* @see #fileFor(String)
*/
protected String idFor( File file ) {
String path = file.getAbsolutePath();
if (!path.startsWith(directoryAbsolutePath)) {
if (directory.getAbsolutePath().equals(path)) {
// This is the root
return DELIMITER;
}
String msg = JcrI18n.fileConnectorNodeIdentifierIsNotWithinScopeOfConnector.text(getSourceName(), directoryPath, path);
throw new DocumentStoreException(path, msg);
}
String id = path.substring(directoryAbsolutePathLength);
id = id.replaceAll(Pattern.quote(FILE_SEPARATOR), DELIMITER);
assert id.startsWith(DELIMITER);
return id;
}
/**
* Utility method for creating a {@link BinaryValue} for the given {@link File} object. Subclasses should rarely override this
* method.
*
* @param file the file; may not be null
* @return the BinaryValue; never null
*/
protected ExternalBinaryValue binaryFor( File file ) {
try {
return createBinaryValue(file);
} catch (RuntimeException e) {
throw e;
} catch (Throwable e) {
throw new RuntimeException(e);
}
}
/**
* Utility method to create a {@link BinaryValue} object for the given file. Subclasses should rarely override this method,
* since the {@link UrlBinaryValue} will be applicable in most situations.
*
* @param file the file for which the {@link BinaryValue} is to be created; never null
* @return the binary value; never null
* @throws IOException if there is an error creating the value
*/
protected ExternalBinaryValue createBinaryValue( File file ) throws IOException {
URL content = createUrlForFile(file);
return new UrlBinaryValue(sha1(file), getSourceName(), content, file.length(), file.getName(), getMimeTypeDetector());
}
/**
* Computes the SHA1 for the given file. By default, this method will look at the
* {@link FileSystemConnector#contentBasedSha1()} flag and either take the URL of the file (using @see
* java.util.File#toURI().toURL() and return the SHA1 of the URL string or return the SHA1 of the entire file content.
*
* @param file a {@link File} instance; never null
* @return the SHA1 of the file.
*/
protected String sha1( File file ) {
try {
if (contentBasedSha1()) {
byte[] hash = SecureHash.getHash(SecureHash.Algorithm.SHA_1, file);
return StringUtil.getHexString(hash);
}
return SecureHash.sha1(createUrlForFile(file).toString());
} catch (Exception e) {
throw new ConnectorException(e);
}
}
/**
* Construct a {@link URL} object for the given file, to be used within the {@link Binary} value representing the "jcr:data"
* property of a 'nt:resource' node.
* <p>
* Subclasses can override this method to transform the URL into something different. For example, if the files are being
* served by a web server, the overridden method might transform the file-based URL into the corresponding HTTP-based URL.
* </p>
*
* @param file the file for which the URL is to be created; never null
* @return the URL for the file; never null
* @throws IOException if there is an error creating the URL
*/
protected URL createUrlForFile( File file ) throws IOException {
return file.toURI().toURL();
}
protected File createFileForUrl( URL url ) throws URISyntaxException {
return new File(url.toURI());
}
protected boolean contentBasedSha1() {
return contentBasedSha1;
}
/**
* Utility method to determine if the file is excluded by the inclusion/exclusion filter.
*
* @param file the file
* @return true if the file is excluded, or false if it is to be included
*/
protected boolean isExcluded( File file ) {
return !filenameFilter.accept(file.getParentFile(), file.getName());
}
/**
* Utility method to ensure that the file is writable by this connector.
*
* @param id the identifier of the node
* @param file the file
* @throws DocumentStoreException if the file is expected to be writable but is not or is excluded, or if the connector is
* readonly
*/
protected void checkFileNotExcluded( String id,
File file ) {
if (isExcluded(file)) {
String msg = JcrI18n.fileConnectorCannotStoreFileThatIsExcluded.text(getSourceName(), id, file.getAbsolutePath());
throw new DocumentStoreException(id, msg);
}
}
@Override
public boolean hasDocument( String id ) {
return fileFor(id).exists();
}
/**
* Hook for processing other node types (not resources or regular files)
* @param id
* @param file
* @param isRoot
* @param isResource
* @return DocumentWriter if node is recognized and processed; otherwise
* null (signaling subsequent fallback to default processing)
*/
protected DocumentWriter writerFor(String id, File file, boolean isRoot, boolean isResource) {
return null;
}
@Override
public Document getDocumentById( String id ) {
File file = fileFor(id);
boolean isRoot = isRoot(id);
// make sure the root folder is never excluded....
if (!isRoot && (isExcluded(file) || !acceptFile(file))) return null;
boolean isResource = isContentNode(id);
File parentFile = file.getParentFile();
DocumentWriter writer = writerFor(id, file, isRoot, isResource);
if (writer != null) {
// node configuration initialized by writerFor method
} else if (isResource) {
writer = newDocument(id);
BinaryValue binaryValue = binaryFor(file);
writer.setPrimaryType(NT_RESOURCE);
writer.addProperty(JCR_DATA, binaryValue);
if (addMimeTypeMixin) {
String mimeType = null;
try {
mimeType = binaryValue.getMimeType();
} catch (Throwable e) {
getLogger().error(e, JcrI18n.couldNotGetMimeType, getSourceName(), id, e.getMessage());
}
writer.addProperty(JCR_MIME_TYPE, mimeType);
}
writer.addProperty(JCR_LAST_MODIFIED, lastModifiedTimeFor(file));
writer.addProperty(JCR_LAST_MODIFIED_BY, ownerFor(file));
// // make these binary not queryable. If we really want to query them, we need to switch to external binaries
// writer.setNotQueryable();
if (!isQueryable()) writer.setNotQueryable();
parentFile = file;
} else if (file.isFile()) {
writer = newDocument(id);
writer.setPrimaryType(NT_FILE);
writer.addProperty(JCR_CREATED, createdTimeFor(file));
writer.addProperty(JCR_CREATED_BY, ownerFor(file));
String childId = contentChildId(id, isRoot);
writer.addChild(childId, JCR_CONTENT);
if (!isQueryable()) writer.setNotQueryable();
} else {
writer = newFolderWriter(id, file, 0);
}
if (!isRoot) {
// Set the reference to the parent ...
String parentId = idFor(parentFile);
writer.setParents(parentId);
}
// Add the extra properties (if there are any), overwriting any properties with the same names
// (e.g., jcr:primaryType, jcr:mixinTypes, jcr:mimeType, etc.) ...
writer.addProperties(extraPropertiesStore().getProperties(id));
// Add the 'mix:mixinType' mixin; if other mixins are stored in the extra properties, this will append ...
if (addMimeTypeMixin) {
writer.addMixinType(MIX_MIME_TYPE);
}
// Return the document ...
return writer.document();
}
private DateTime createdTimeFor( File file ) {
BasicFileAttributes basicFileAttributes = basicAttributesFor(file);
return basicFileAttributes != null ? factories().getDateFactory().create(basicFileAttributes.creationTime().toMillis()) : factories().getDateFactory()
.create(file.lastModified());
}
private DateTime lastModifiedTimeFor( File file ) {
BasicFileAttributes basicFileAttributes = basicAttributesFor(file);
return basicFileAttributes != null ? factories().getDateFactory().create(basicFileAttributes.lastModifiedTime()
.toMillis()) : factories().getDateFactory()
.create(file.lastModified());
}
private String ownerFor( File file ) {
Path filePath = Paths.get(file.toURI());
try {
return Files.getOwner(filePath).getName();
} catch (IOException e) {
log().debug(e, "Unable to read the owner of '{0}'", filePath);
return null;
}
}
private BasicFileAttributes basicAttributesFor( File file ) {
Path filePath = Paths.get(file.toURI());
try {
return Files.readAttributes(filePath, BasicFileAttributes.class);
} catch (IOException e) {
log().debug(e, "Unable to read attributes for '{0}'", filePath);
return null;
}
}
protected String contentChildId( String fileId,
boolean isRoot ) {
return isRoot ? JCR_CONTENT_SUFFIX : fileId + JCR_CONTENT_SUFFIX;
}
protected boolean acceptFile(File file) {
// By default, only include as a child if we can access and read the file. Permissions might prevent us from
// reading the file, and the file might not exist if it is a broken symlink (see MODE-1768 for details).
return file.exists() && file.canRead() && (file.isFile() || file.isDirectory());
}
private DocumentWriter newFolderWriter( String id,
File file,
int offset ) {
boolean root = isRoot(id);
DocumentWriter writer = newDocument(id);
writer.setPrimaryType(NT_FOLDER);
writer.addProperty(JCR_CREATED, createdTimeFor(file));
writer.addProperty(JCR_CREATED_BY, ownerFor(file));
if (!isQueryable()) writer.setNotQueryable();
File[] children = file.listFiles(filenameFilter);
long totalChildren = 0;
int nextOffset = 0;
for (int i = 0; i < children.length; i++) {
File child = children[i];
if (acceptFile(child)) {
// we need to count the total accessible children
totalChildren++;
// only add a child if it's in the current page
if (i >= offset && i < offset + pageSize) {
// We use identifiers that contain the file/directory name ...
String childName = child.getName();
String childId = root ? DELIMITER + childName : id + DELIMITER + childName;
writer.addChild(childId, childName);
nextOffset = i + 1;
}
}
}
// if there are still accessible children add the next page
if (nextOffset < totalChildren) {
writer.addPage(id, nextOffset, pageSize, totalChildren);
}
return writer;
}
@Override
public String getDocumentId( String path ) {
String id = path; // this connector treats the ID as the path
File file = fileFor(id);
return file.exists() ? id : null;
}
@Override
public Collection<String> getDocumentPathsById( String id ) {
// this connector treats the ID as the path
return Collections.singletonList(id);
}
@Override
public ExternalBinaryValue getBinaryValue( String id ) {
try {
File f = createFileForUrl(new URL(id));
return binaryFor(f);
} catch (IOException | URISyntaxException e) {
throw new DocumentStoreException(id, e);
}
}
@Override
public boolean removeDocument( String id ) {
File file = fileFor(id);
checkFileNotExcluded(id, file);
// Remove the extra properties at the old location ...
extraPropertiesStore().removeProperties(id);
// Now remove the file (if it is there) ...
if (!file.exists()) return false;
FileUtil.delete(file); // recursive delete
return true;
}
@Override
public void storeDocument( Document document ) {
// Create a new directory or file described by the document ...
DocumentReader reader = readDocument(document);
String id = reader.getDocumentId();
File file = fileFor(id);
checkFileNotExcluded(id, file);
File parent = file.getParentFile();
if (!parent.exists()) {
parent.mkdirs();
}
if (!parent.canWrite()) {
String parentPath = parent.getAbsolutePath();
String msg = JcrI18n.fileConnectorCannotWriteToDirectory.text(getSourceName(), getClass(), parentPath);
throw new DocumentStoreException(id, msg);
}
String primaryType = reader.getPrimaryTypeName();
Map<Name, Property> properties = reader.getProperties();
ExtraProperties extraProperties = extraPropertiesFor(id, false);
extraProperties.addAll(properties).except(JCR_PRIMARY_TYPE, JCR_CREATED, JCR_LAST_MODIFIED, JCR_DATA);
try {
if (NT_FILE.equals(primaryType)) {
file.createNewFile();
} else if (NT_FOLDER.equals(primaryType)) {
file.mkdirs();
} else if (isContentNode(id)) {
Property content = properties.get(JcrLexicon.DATA);
BinaryValue binary = factories().getBinaryFactory().create(content.getFirstValue());
OutputStream ostream = new BufferedOutputStream(new FileOutputStream(file));
IoUtil.write(binary.getStream(), ostream);
if (!NT_RESOURCE.equals(primaryType)) {
// This is the "jcr:content" child, but the primary type is non-standard so record it as an extra property
extraProperties.add(properties.get(JcrLexicon.PRIMARY_TYPE));
}
}
extraProperties.save();
} catch (RepositoryException | IOException e) {
throw new DocumentStoreException(id, e);
}
}
@Override
public String newDocumentId( String parentId,
Name newDocumentName,
Name newDocumentPrimaryType ) {
StringBuilder id = new StringBuilder(parentId);
if (!parentId.endsWith(DELIMITER)) {
id.append(DELIMITER);
}
// We're only using the name to check, which can be a bit dangerous if users don't follow the JCR conventions.
// However, it matches what "isContentNode(...)" does.
String childNameStr = getContext().getValueFactories().getStringFactory().create(newDocumentName);
if (JCR_CONTENT.equals(childNameStr)) {
// This is for the "jcr:content" node underneath a file node. Since this doesn't actually result in a file or folder
// on the file system (it's merged into the file for the parent 'nt:file' node), we'll keep the "jcr" namespace
// prefix in the ID so that 'isContentNode(...)' works properly ...
id.append(childNameStr);
} else {
// File systems don't universally deal well with ':' in the names, and when they do it can be a bit awkward. Since we
// don't often expect the node NAMES to contain namespaces (at leat with this connector), we'll just
// use the local part for the ID ...
id.append(newDocumentName.getLocalName());
if (!StringUtil.isBlank(newDocumentName.getNamespaceUri())) {
// the FS connector does not support namespaces in names
String ns = newDocumentName.getNamespaceUri();
getLogger().warn(JcrI18n.fileConnectorNamespaceIgnored, getSourceName(), ns, id, childNameStr, parentId);
}
}
return id.toString();
}
@Override
public void updateDocument( DocumentChanges documentChanges ) {
String id = documentChanges.getDocumentId();
Document document = documentChanges.getDocument();
DocumentReader reader = readDocument(document);
File file = fileFor(id);
String idOrig = id;
// if we're dealing with the root of the connector, we can't process any moves/removes because that would go "outside" the
// connector scope
if (!isRoot(id)) {
String newParentId = reader.getParentIds().get(0);
File parent = file.getParentFile();
String oldParentId = idFor(parent);
if (!oldParentId.equals(newParentId)) {
// The node has a new parent (via the 'update' method), meaning it was moved ...
File newParent = fileFor(newParentId);
File newFile = new File(newParent, file.getName());
if (!newParent.exists()) {
parent.mkdirs(); // in case they were removed since we created them ...
}
if (!newParent.canWrite()) {
String parentPath = newParent.getAbsolutePath();
String msg = JcrI18n.fileConnectorCannotWriteToDirectory.text(getSourceName(), getClass(), parentPath);
throw new DocumentStoreException(id, msg);
}
if (!file.renameTo(newFile)) {
getLogger().debug("Cannot move {0} to {1}", file.getAbsolutePath(), newFile.getAbsolutePath());
} else {
id = idFor(newFile);
// Make sure any existing extra properties are also kept up-to-date
// Note that if the children ar folders, we don't need to walk them recursively because the node id will
// reflect the new folder structure of the rename
moveExtraProperties(idOrig, id);
}
} else {
// It is the same parent as before ...
if (!parent.exists()) {
parent.mkdirs(); // in case they were removed since we created them ...
}
if (!parent.canWrite()) {
String parentPath = parent.getAbsolutePath();
String msg = JcrI18n.fileConnectorCannotWriteToDirectory.text(getSourceName(), getClass(), parentPath);
throw new DocumentStoreException(id, msg);
}
}
}
// children renames have to be processed in the parent
DocumentChanges.ChildrenChanges childrenChanges = documentChanges.getChildrenChanges();
Map<String, Name> renamedChildren = childrenChanges.getRenamed();
for (String renamedChildId : renamedChildren.keySet()) {
File child = fileFor(renamedChildId);
Name newName = renamedChildren.get(renamedChildId);
String newNameStr = getContext().getValueFactories().getStringFactory().create(newName);
File renamedChild = new File(file, newNameStr);
if (!child.renameTo(renamedChild)) {
getLogger().debug("Cannot rename {0} to {1}", child, renamedChild);
} else {
// Make sure any existing extra properties are also kept up-to-date
// Note that if the children ar folders, we don't need to walk them recursively because the node id will reflect
// the new folder structure of the rename
moveExtraProperties(idFor(child), idFor(renamedChild));
}
}
String primaryType = reader.getPrimaryTypeName();
Map<Name, Property> properties = reader.getProperties();
id = idOrig;
ExtraProperties extraProperties = extraPropertiesFor(id, true);
extraProperties.addAll(properties).except(JCR_PRIMARY_TYPE, JCR_CREATED, JCR_LAST_MODIFIED, JCR_DATA);
try {
if (NT_FILE.equals(primaryType)) {
file.createNewFile();
} else if (NT_FOLDER.equals(primaryType)) {
file.mkdir();
} else if (isContentNode(id)) {
Property content = reader.getProperty(JCR_DATA);
BinaryValue binary = factories().getBinaryFactory().create(content.getFirstValue());
OutputStream ostream = new BufferedOutputStream(new FileOutputStream(file));
IoUtil.write(binary.getStream(), ostream);
if (!NT_RESOURCE.equals(primaryType)) {
// This is the "jcr:content" child, but the primary type is non-standard so record it as an extra property
extraProperties.add(properties.get(JcrLexicon.PRIMARY_TYPE));
}
}
extraProperties.save();
} catch (RepositoryException | IOException e) {
throw new DocumentStoreException(id, e);
}
}
@Override
public Document getChildren( PageKey pageKey ) {
String parentId = pageKey.getParentId();
File folder = fileFor(parentId);
assert folder.isDirectory();
if (!folder.canRead()) {
getLogger().debug("Cannot read the {0} folder", folder.getAbsolutePath());
return null;
}
return newFolderWriter(parentId, folder, pageKey.getOffsetInt()).document();
}
private static class MonitoringTask implements Callable<Void> {
protected final static WatchEvent.Kind<?>[] EVENTS_TO_WATCH = new WatchEvent.Kind<?>[] {ENTRY_CREATE, ENTRY_MODIFY,
ENTRY_DELETE};
protected final static java.nio.file.WatchEvent.Modifier WATCH_MODIFIER;
private final WatchService watchService;
private final FileSystemConnector connector;
static {
java.nio.file.WatchEvent.Modifier modifier = null;
try {
@SuppressWarnings( "unchecked" )
Class<? extends Enum<?>> modifierClass = (Class<? extends Enum<?>>)Class.forName("com.sun.nio.file.SensitivityWatchEventModifier",
false,
FileSystemConnector.class.getClassLoader());
for (Enum<?> enumConstants : modifierClass.getEnumConstants()) {
if (enumConstants.name().equalsIgnoreCase("HIGH")) {
modifier = (WatchEvent.Modifier)enumConstants;
break;
}
}
} catch (ClassNotFoundException e) {
// modifier not available on this JDK
}
WATCH_MODIFIER = modifier;
}
protected MonitoringTask( WatchService watchService,
FileSystemConnector connector,
Path rootPath ) {
this.watchService = watchService;
this.connector = connector;
recursiveWatch(rootPath, watchService);
}
@Override
@SuppressWarnings( "unchecked" )
public Void call() throws Exception {
for (;;) {
try {
WatchKey watchKey = watchService.take();
@SuppressWarnings( "synthetic-access" )
ConnectorChangeSet connectorChangeSet = connector.newConnectorChangedSet();
for (WatchEvent<?> watchEvent : watchKey.pollEvents()) {
WatchEvent.Kind<?> kind = watchEvent.kind();
Path eventPath = ((WatchEvent<Path>)watchEvent).context();
Path resolvedPath = ((Path)watchKey.watchable()).resolve(eventPath);
File resolvedFile = resolvedPath.toFile();
if (connector.isExcluded(resolvedFile)) {
continue;
}
if (kind == ENTRY_CREATE) {
fireEntryCreated(connectorChangeSet, resolvedPath);
} else if (kind == ENTRY_DELETE) {
fireEntryDeleted(connectorChangeSet, resolvedPath);
} else if (kind == ENTRY_MODIFY) {
fireEntryModified(connectorChangeSet, resolvedPath);
}
}
watchKey.reset();
connectorChangeSet.publish(null);
} catch (InterruptedException e) {
Thread.interrupted();
watchService.close();
break;
}
}
return null;
}
@SuppressWarnings( "synthetic-access" )
private void fireEntryModified( ConnectorChangeSet connectorChangeSet,
Path resolvedPath ) {
connector.log().debug("Received ENTRY_MODIFY event on '{0}'", resolvedPath);
// this event is *very much dependent on the OS*, so we'll try to focus on the most general case
// we only handle file modifications and fire events for last_modified and content
boolean isFile = Files.isRegularFile(resolvedPath, LinkOption.NOFOLLOW_LINKS);
if (!isFile) {
connector.log().debug("The entry at {0} is not a regular file; ignoring modify event", resolvedPath);
return;
}
File file = resolvedPath.toFile();
String contentId = connector.contentChildId(connector.idFor(file), false);
boolean queryable = connector.isQueryable();
// fire PROPERTY_CHANGED for nt:file/jcr:content/jcr:data/jcr:lastModified
Property modifiedProperty = connector.propertyFactory().create(JcrLexicon.LAST_MODIFIED,
connector.lastModifiedTimeFor(file));
// there is no way to observe the previous value, so fire <null>
connectorChangeSet.propertyChanged(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId, null,
modifiedProperty);
// fire PROPERTY_CHANGED for nt:file/jcr:content/jcr:data/jcr:lastModifiedBy
String owner = connector.ownerFor(file);
if (owner != null) {
Property modifiedByProperty = connector.propertyFactory().create(JcrLexicon.LAST_MODIFIED_BY, owner);
// there is no way to observe the previous value, so fire <null>
connectorChangeSet.propertyChanged(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId,
null, modifiedByProperty);
}
try {
// fire PROPERTY_CHANGED for nt:file/jcr:content/jcr:data
BinaryValue binaryValue = connector.binaryFor(file);
Property binaryProperty = connector.propertyFactory().create(JcrLexicon.DATA, binaryValue);
connectorChangeSet.propertyChanged(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId,
null, binaryProperty);
if (connector.addMimeTypeMixin) {
try {
// fire PROPERTY_CHANGED for nt:file/jcr:content/jcr:mimetype
String mimeType = binaryValue.getMimeType();
Property mimeTypeProperty = connector.propertyFactory().create(JcrLexicon.MIMETYPE, mimeType);
connectorChangeSet.propertyChanged(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(),
contentId, null, mimeTypeProperty);
} catch (Throwable e) {
connector.getLogger().error(e, JcrI18n.couldNotGetMimeType, connector.getSourceName(), contentId,
e.getMessage());
}
}
} catch (Exception e) {
connector.log().warn("Cannot get binary value for '{0}'", resolvedPath);
}
}
private void fireEntryDeleted( ConnectorChangeSet connectorChangeSet,
Path resolvedPath ) {
connector.log().debug("Received ENTRY_DELETE event on '{0}'", resolvedPath);
boolean queryable = connector.isQueryable();
Name primaryType = primaryTypeFor(resolvedPath);
if (primaryType == null) {
// Atm when a deleted event is received, because the item is no longer accessible on the FS.
// it's neither a file nor a folder. So we do a "best effort" and use the base type
primaryType = JcrNtLexicon.HIERARCHY_NODE;
}
File deletedFile = resolvedPath.toFile();
String id = connector.idFor(deletedFile);
Path parentPath = resolvedPath.getParent();
Name parentPrimaryType = primaryTypeFor(parentPath);
if (parentPrimaryType == null) {
// Atm when a deleted event is received, because the item is no longer accessible on the FS.
// it's neither a file nor a folder. So we do a "best effort" and use the base type
parentPrimaryType = JcrNtLexicon.HIERARCHY_NODE;
}
connectorChangeSet.nodeRemoved(id, connector.idFor(parentPath.toFile()), id, primaryType,
Collections.<Name>emptySet(), parentPrimaryType,
Collections.<Name>emptySet());
}
@SuppressWarnings( "synthetic-access" )
private void fireEntryCreated( ConnectorChangeSet connectorChangeSet,
Path resolvedPath ) {
connector.log().debug("Received ENTRY_CREATE event on '{0}'", resolvedPath);
Name primaryType = primaryTypeFor(resolvedPath);
if (primaryType == null) {
return;
}
if (Files.isDirectory(resolvedPath, LinkOption.NOFOLLOW_LINKS)) {
// if a new directory has been created, watch it
recursiveWatch(resolvedPath, watchService);
}
File file = resolvedPath.toFile();
String docId = connector.idFor(file);
boolean queryable = connector.isQueryable();
// fire NODE_ADDED for nt:file
connectorChangeSet.nodeCreated(docId, connector.idFor(resolvedPath.getParent().toFile()), docId, primaryType,
Collections.<Name>emptySet(), Collections.<Name, Property>emptyMap());
// fire PROPERTY_ADDED for nt:file/jcr:created
Property createdProperty = connector.propertyFactory().create(JcrLexicon.CREATED, connector.createdTimeFor(file));
connectorChangeSet.propertyAdded(docId, primaryType, Collections.<Name>emptySet(), docId, createdProperty);
String owner = connector.ownerFor(file);
if (owner != null) {
// fire PROPERTY_ADDED for nt:file/jcr:createdBy
Property createdByProperty = connector.propertyFactory().create(JcrLexicon.CREATED_BY, owner);
connectorChangeSet.propertyAdded(docId, primaryType, Collections.<Name>emptySet(), docId, createdByProperty
);
}
if (Files.isRegularFile(resolvedPath, LinkOption.NOFOLLOW_LINKS)) {
// for files we need to fire extra events for their content
String contentId = connector.contentChildId(docId, false);
// fire NODE_ADDED for the nt:file/jcr:content
connectorChangeSet.nodeCreated(contentId, docId, contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(),
Collections.<Name, Property>emptyMap());
try {
// fire PROPERTY_ADDED for nt:file/jcr:content/jcr:data
BinaryValue binaryValue = connector.binaryFor(file);
Property dataProperty = connector.propertyFactory().create(JcrLexicon.DATA, binaryValue);
connectorChangeSet.propertyAdded(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId,
dataProperty);
if (connector.addMimeTypeMixin) {
try {
// fire PROPERTY_ADDED for nt:file/jcr:content/jcr:mimetype
String mimeType = binaryValue.getMimeType();
Property mimeTypeProperty = connector.propertyFactory().create(JcrLexicon.MIMETYPE, mimeType);
connectorChangeSet.propertyAdded(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(),
contentId, mimeTypeProperty);
} catch (Throwable e) {
connector.getLogger().error(e, JcrI18n.couldNotGetMimeType, connector.getSourceName(), contentId,
e.getMessage());
}
}
} catch (Exception e) {
connector.log().warn("Cannot get binary value for '{0}'", resolvedPath);
}
// fire PROPERTY_ADDED for nt:file/jcr:content/jcr:lastModified
Property lastModified = connector.propertyFactory().create(JcrLexicon.LAST_MODIFIED,
connector.lastModifiedTimeFor(file));
connectorChangeSet.propertyAdded(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId,
lastModified);
if (owner != null) {
// fire PROPERTY_ADDED for nt:file/jcr:content/jcr:lastModifiedBy
Property lastModifiedBy = connector.propertyFactory().create(JcrLexicon.LAST_MODIFIED_BY, owner);
connectorChangeSet.propertyAdded(contentId, JcrNtLexicon.RESOURCE, Collections.<Name>emptySet(), contentId,
lastModifiedBy);
}
}
}
private void recursiveWatch( Path path,
final WatchService watchService ) {
connector.log().debug("Recursively watching '{0}'", path);
try {
Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory( Path dir,
BasicFileAttributes attrs ) throws IOException {
if (WATCH_MODIFIER != null) {
dir.register(watchService, EVENTS_TO_WATCH, WATCH_MODIFIER);
} else {
dir.register(watchService, EVENTS_TO_WATCH);
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Name primaryTypeFor( Path resolvedPath ) {
boolean isFolder = Files.isDirectory(resolvedPath, LinkOption.NOFOLLOW_LINKS);
boolean isFile = Files.isRegularFile(resolvedPath, LinkOption.NOFOLLOW_LINKS);
if (!isFile && !isFolder) {
connector.log().debug("The entry at {0} is neither a file nor a folder", resolvedPath);
return null;
}
return isFolder ? JcrNtLexicon.FOLDER : JcrNtLexicon.FILE;
}
}
}