/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache license, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the license for the specific language governing permissions and
* limitations under the license.
*/
package org.apache.logging.log4j.core.appender;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.MappedByteBuffer;
import java.nio.channels.FileChannel;
import java.security.AccessController;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import org.apache.logging.log4j.core.Layout;
import org.apache.logging.log4j.core.util.Closer;
import org.apache.logging.log4j.core.util.FileUtils;
import org.apache.logging.log4j.core.util.NullOutputStream;
//Lines too long...
//CHECKSTYLE:OFF
/**
* Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
* memory and writes to this memory region.
* <p>
*
* @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
* http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
* @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
* @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
* @see <a
* href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
* http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
*
* @since 2.1
*/
//CHECKSTYLE:ON
public class MemoryMappedFileManager extends OutputStreamManager {
/**
* Default length of region to map.
*/
static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
private static final int MAX_REMAP_COUNT = 10;
private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
private final boolean immediateFlush;
private final int regionLength;
private final String advertiseURI;
private final RandomAccessFile randomAccessFile;
private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
private MappedByteBuffer mappedBuffer;
private long mappingOffset;
protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
final boolean immediateFlush, final long position, final int regionLength, final String advertiseURI,
final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0]));
this.immediateFlush = immediateFlush;
this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
this.regionLength = regionLength;
this.advertiseURI = advertiseURI;
this.isEndOfBatch.set(Boolean.FALSE);
this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
this.byteBuffer = mappedBuffer;
this.mappingOffset = position;
}
/**
* Returns the MemoryMappedFileManager.
*
* @param fileName The name of the file to manage.
* @param append true if the file should be appended to, false if it should be overwritten.
* @param immediateFlush true if the contents should be flushed to disk on every write
* @param regionLength The mapped region length.
* @param advertiseURI the URI to use when advertising the file
* @param layout The layout.
* @return A MemoryMappedFileManager for the File.
*/
public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
final boolean immediateFlush, final int regionLength, final String advertiseURI,
final Layout<? extends Serializable> layout) {
return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, immediateFlush, regionLength,
advertiseURI, layout), FACTORY);
}
public Boolean isEndOfBatch() {
return isEndOfBatch.get();
}
public void setEndOfBatch(final boolean endOfBatch) {
this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
}
@Override
protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
while (length > mappedBuffer.remaining()) {
final int chunk = mappedBuffer.remaining();
mappedBuffer.put(bytes, offset, chunk);
offset += chunk;
length -= chunk;
remap();
}
mappedBuffer.put(bytes, offset, length);
// no need to call flush() if force is true,
// already done in AbstractOutputStreamAppender.append
}
private synchronized void remap() {
final long offset = this.mappingOffset + mappedBuffer.position();
final int length = mappedBuffer.remaining() + regionLength;
try {
unsafeUnmap(mappedBuffer);
final long fileLength = randomAccessFile.length() + regionLength;
LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
regionLength, fileLength);
final long startNanos = System.nanoTime();
randomAccessFile.setLength(fileLength);
final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
millis);
mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
this.byteBuffer = mappedBuffer;
mappingOffset = offset;
} catch (final Exception ex) {
logError("Unable to remap", ex);
}
}
@Override
public synchronized void flush() {
mappedBuffer.force();
}
@Override
public synchronized boolean closeOutputStream() {
final long position = mappedBuffer.position();
final long length = mappingOffset + position;
try {
unsafeUnmap(mappedBuffer);
} catch (final Exception ex) {
logError("Unable to unmap MappedBuffer", ex);
}
try {
LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
length, mappingOffset, position);
randomAccessFile.setLength(length);
randomAccessFile.close();
return true;
} catch (final IOException ex) {
logError("Unable to close MemoryMappedFile", ex);
return false;
}
}
public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
final int size) throws IOException {
for (int i = 1;; i++) {
try {
LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
final long startNanos = System.nanoTime();
final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
map.order(ByteOrder.nativeOrder());
final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
return map;
} catch (final IOException e) {
if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
throw e;
}
LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
if (i < MAX_REMAP_COUNT) {
Thread.yield();
} else {
try {
Thread.sleep(1);
} catch (final InterruptedException ignored) {
Thread.currentThread().interrupt();
throw e;
}
}
}
}
}
private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
LOGGER.debug("MMapAppender unmapping old buffer...");
final long startNanos = System.nanoTime();
AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
@Override
public Object run() throws Exception {
final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
getCleanerMethod.setAccessible(true);
final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
final Method cleanMethod = cleaner.getClass().getMethod("clean");
cleanMethod.invoke(cleaner);
return null;
}
});
final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
}
/**
* Returns the name of the File being managed.
*
* @return The name of the File being managed.
*/
public String getFileName() {
return getName();
}
/**
* Returns the length of the memory mapped region.
*
* @return the length of the mapped region
*/
public int getRegionLength() {
return regionLength;
}
/**
* Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
* {@code false} otherwise.
*
* @return whether each write should be force-sync'ed
*/
public boolean isImmediateFlush() {
return immediateFlush;
}
/**
* Gets this FileManager's content format specified by:
* <p>
* Key: "fileURI" Value: provided "advertiseURI" param.
* </p>
*
* @return Map of content format keys supporting FileManager
*/
@Override
public Map<String, String> getContentFormat() {
final Map<String, String> result = new HashMap<>(super.getContentFormat());
result.put("fileURI", advertiseURI);
return result;
}
@Override
protected void flushBuffer(final ByteBuffer buffer) {
// do nothing (do not call drain() to avoid spurious remapping)
}
@Override
public ByteBuffer getByteBuffer() {
return mappedBuffer;
}
@Override
public ByteBuffer drain(final ByteBuffer buf) {
remap();
return mappedBuffer;
}
/**
* Factory Data.
*/
private static class FactoryData {
private final boolean append;
private final boolean immediateFlush;
private final int regionLength;
private final String advertiseURI;
private final Layout<? extends Serializable> layout;
/**
* Constructor.
*
* @param append Append to existing file or truncate.
* @param immediateFlush forces the memory content to be written to the storage device on every event
* @param regionLength length of the mapped region
* @param advertiseURI the URI to use when advertising the file
* @param layout The layout.
*/
public FactoryData(final boolean append, final boolean immediateFlush, final int regionLength,
final String advertiseURI, final Layout<? extends Serializable> layout) {
this.append = append;
this.immediateFlush = immediateFlush;
this.regionLength = regionLength;
this.advertiseURI = advertiseURI;
this.layout = layout;
}
}
/**
* Factory to create a MemoryMappedFileManager.
*/
private static class MemoryMappedFileManagerFactory
implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
/**
* Create a MemoryMappedFileManager.
*
* @param name The name of the File.
* @param data The FactoryData
* @return The MemoryMappedFileManager for the File.
*/
@SuppressWarnings("resource")
@Override
public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
final File file = new File(name);
if (!data.append) {
file.delete();
}
final boolean writeHeader = !data.append || !file.exists();
final OutputStream os = NullOutputStream.getInstance();
RandomAccessFile raf = null;
try {
FileUtils.makeParentDirs(file);
raf = new RandomAccessFile(name, "rw");
final long position = (data.append) ? raf.length() : 0;
raf.setLength(position + data.regionLength);
return new MemoryMappedFileManager(raf, name, os, data.immediateFlush, position, data.regionLength,
data.advertiseURI, data.layout, writeHeader);
} catch (final Exception ex) {
LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
Closer.closeSilently(raf);
}
return null;
}
}
}