001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache license, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the license for the specific language governing permissions and 015 * limitations under the license. 016 */ 017package org.apache.logging.log4j.core.appender; 018 019import java.io.File; 020import java.io.IOException; 021import java.io.OutputStream; 022import java.io.RandomAccessFile; 023import java.io.Serializable; 024import java.lang.reflect.Method; 025import java.nio.ByteOrder; 026import java.nio.MappedByteBuffer; 027import java.nio.channels.FileChannel; 028import java.security.AccessController; 029import java.security.PrivilegedActionException; 030import java.security.PrivilegedExceptionAction; 031import java.util.HashMap; 032import java.util.Map; 033import java.util.Objects; 034 035import org.apache.logging.log4j.core.Layout; 036import org.apache.logging.log4j.core.util.Closer; 037import org.apache.logging.log4j.core.util.NullOutputStream; 038 039/** 040 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into 041 * memory and writes to this memory region. 042 * <p> 043 * 044 * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java"> 045 * http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a> 046 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a> 047 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a> 048 * @see <a 049 * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation"> 050 * http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a> 051 * 052 * @since 2.1 053 */ 054public class MemoryMappedFileManager extends OutputStreamManager { 055 /** 056 * Default length of region to map. 057 */ 058 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024; 059 private static final int MAX_REMAP_COUNT = 10; 060 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory(); 061 private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0; 062 063 private final boolean isForce; 064 private final int regionLength; 065 private final String advertiseURI; 066 private final RandomAccessFile randomAccessFile; 067 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>(); 068 private MappedByteBuffer mappedBuffer; 069 private long mappingOffset; 070 071 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os, 072 final boolean force, final long position, final int regionLength, final String advertiseURI, 073 final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException { 074 super(os, fileName, layout, writeHeader); 075 this.isForce = force; 076 this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile"); 077 this.regionLength = regionLength; 078 this.advertiseURI = advertiseURI; 079 this.isEndOfBatch.set(Boolean.FALSE); 080 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength); 081 this.mappingOffset = position; 082 } 083 084 /** 085 * Returns the MemoryMappedFileManager. 086 * 087 * @param fileName The name of the file to manage. 088 * @param append true if the file should be appended to, false if it should be overwritten. 089 * @param isForce true if the contents should be flushed to disk on every write 090 * @param regionLength The mapped region length. 091 * @param advertiseURI the URI to use when advertising the file 092 * @param layout The layout. 093 * @return A MemoryMappedFileManager for the File. 094 */ 095 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append, 096 final boolean isForce, final int regionLength, final String advertiseURI, 097 final Layout<? extends Serializable> layout) { 098 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength, 099 advertiseURI, layout), FACTORY); 100 } 101 102 public Boolean isEndOfBatch() { 103 return isEndOfBatch.get(); 104 } 105 106 public void setEndOfBatch(final boolean endOfBatch) { 107 this.isEndOfBatch.set(Boolean.valueOf(endOfBatch)); 108 } 109 110 @Override 111 protected synchronized void write(final byte[] bytes, int offset, int length) { 112 super.write(bytes, offset, length); // writes to dummy output stream 113 114 while (length > mappedBuffer.remaining()) { 115 final int chunk = mappedBuffer.remaining(); 116 mappedBuffer.put(bytes, offset, chunk); 117 offset += chunk; 118 length -= chunk; 119 remap(); 120 } 121 mappedBuffer.put(bytes, offset, length); 122 123 // no need to call flush() if force is true, 124 // already done in AbstractOutputStreamAppender.append 125 } 126 127 private synchronized void remap() { 128 final long offset = this.mappingOffset + mappedBuffer.position(); 129 final int length = mappedBuffer.remaining() + regionLength; 130 try { 131 unsafeUnmap(mappedBuffer); 132 final long fileLength = randomAccessFile.length() + regionLength; 133 LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength); 134 135 final long startNanos = System.nanoTime(); 136 randomAccessFile.setLength(fileLength); 137 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 138 LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis); 139 140 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length); 141 mappingOffset = offset; 142 } catch (final Exception ex) { 143 LOGGER.error("Unable to remap " + getName() + ". " + ex); 144 } 145 } 146 147 @Override 148 public synchronized void flush() { 149 mappedBuffer.force(); 150 } 151 152 @Override 153 public synchronized void close() { 154 final long position = mappedBuffer.position(); 155 final long length = mappingOffset + position; 156 try { 157 unsafeUnmap(mappedBuffer); 158 } catch (final Exception ex) { 159 LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex); 160 } 161 try { 162 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(), 163 length, mappingOffset, position); 164 randomAccessFile.setLength(length); 165 randomAccessFile.close(); 166 } catch (final IOException ex) { 167 LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex); 168 } 169 } 170 171 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start, 172 final int size) throws IOException { 173 for (int i = 1;; i++) { 174 try { 175 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size); 176 177 final long startNanos = System.nanoTime(); 178 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size); 179 map.order(ByteOrder.nativeOrder()); 180 181 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 182 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis); 183 184 return map; 185 } catch (final IOException e) { 186 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) { 187 throw e; 188 } 189 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e); 190 if (i < MAX_REMAP_COUNT) { 191 Thread.yield(); 192 } else { 193 try { 194 Thread.sleep(1); 195 } catch (final InterruptedException ignored) { 196 Thread.currentThread().interrupt(); 197 throw e; 198 } 199 } 200 } 201 } 202 } 203 204 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException { 205 LOGGER.debug("MMapAppender unmapping old buffer..."); 206 final long startNanos = System.nanoTime(); 207 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() { 208 @Override 209 public Object run() throws Exception { 210 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner"); 211 getCleanerMethod.setAccessible(true); 212 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance 213 final Method cleanMethod = cleaner.getClass().getMethod("clean"); 214 cleanMethod.invoke(cleaner); 215 return null; 216 } 217 }); 218 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 219 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis); 220 } 221 222 /** 223 * Returns the name of the File being managed. 224 * 225 * @return The name of the File being managed. 226 */ 227 public String getFileName() { 228 return getName(); 229 } 230 231 /** 232 * Returns the length of the memory mapped region. 233 * 234 * @return the length of the mapped region 235 */ 236 public int getRegionLength() { 237 return regionLength; 238 } 239 240 /** 241 * Returns {@code true} if the content of the buffer should be forced to the storage device on every write, 242 * {@code false} otherwise. 243 * 244 * @return whether each write should be force-sync'ed 245 */ 246 public boolean isImmediateFlush() { 247 return isForce; 248 } 249 250 /** 251 * Gets this FileManager's content format specified by: 252 * <p> 253 * Key: "fileURI" Value: provided "advertiseURI" param. 254 * </p> 255 * 256 * @return Map of content format keys supporting FileManager 257 */ 258 @Override 259 public Map<String, String> getContentFormat() { 260 final Map<String, String> result = new HashMap<>(super.getContentFormat()); 261 result.put("fileURI", advertiseURI); 262 return result; 263 } 264 265 /** 266 * Factory Data. 267 */ 268 private static class FactoryData { 269 private final boolean append; 270 private final boolean force; 271 private final int regionLength; 272 private final String advertiseURI; 273 private final Layout<? extends Serializable> layout; 274 275 /** 276 * Constructor. 277 * 278 * @param append Append to existing file or truncate. 279 * @param force forces the memory content to be written to the storage device on every event 280 * @param regionLength length of the mapped region 281 */ 282 public FactoryData(final boolean append, final boolean force, final int regionLength, 283 final String advertiseURI, final Layout<? extends Serializable> layout) { 284 this.append = append; 285 this.force = force; 286 this.regionLength = regionLength; 287 this.advertiseURI = advertiseURI; 288 this.layout = layout; 289 } 290 } 291 292 /** 293 * Factory to create a MemoryMappedFileManager. 294 */ 295 private static class MemoryMappedFileManagerFactory 296 implements ManagerFactory<MemoryMappedFileManager, FactoryData> { 297 298 /** 299 * Create a MemoryMappedFileManager. 300 * 301 * @param name The name of the File. 302 * @param data The FactoryData 303 * @return The MemoryMappedFileManager for the File. 304 */ 305 @SuppressWarnings("resource") 306 @Override 307 public MemoryMappedFileManager createManager(final String name, final FactoryData data) { 308 final File file = new File(name); 309 final File parent = file.getParentFile(); 310 if (null != parent && !parent.exists()) { 311 parent.mkdirs(); 312 } 313 if (!data.append) { 314 file.delete(); 315 } 316 317 final boolean writeHeader = !data.append || !file.exists(); 318 final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM; 319 RandomAccessFile raf = null; 320 try { 321 raf = new RandomAccessFile(name, "rw"); 322 final long position = (data.append) ? raf.length() : 0; 323 raf.setLength(position + data.regionLength); 324 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength, 325 data.advertiseURI, data.layout, writeHeader); 326 } catch (final Exception ex) { 327 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex); 328 Closer.closeSilently(raf); 329 } 330 return null; 331 } 332 } 333}