001/* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache license, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the license for the specific language governing permissions and 015 * limitations under the license. 016 */ 017package org.apache.logging.log4j.core.appender; 018 019import java.io.File; 020import java.io.IOException; 021import java.io.OutputStream; 022import java.io.RandomAccessFile; 023import java.io.Serializable; 024import java.lang.reflect.Method; 025import java.nio.ByteBuffer; 026import java.nio.ByteOrder; 027import java.nio.MappedByteBuffer; 028import java.nio.channels.FileChannel; 029import java.security.AccessController; 030import java.security.PrivilegedActionException; 031import java.security.PrivilegedExceptionAction; 032import java.util.HashMap; 033import java.util.Map; 034import java.util.Objects; 035 036import org.apache.logging.log4j.core.Layout; 037import org.apache.logging.log4j.core.util.Closer; 038import org.apache.logging.log4j.core.util.NullOutputStream; 039 040//Lines too long... 041//CHECKSTYLE:OFF 042/** 043 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into 044 * memory and writes to this memory region. 045 * <p> 046 * 047 * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java"> 048 * http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a> 049 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a> 050 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a> 051 * @see <a 052 * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation"> 053 * http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a> 054 * 055 * @since 2.1 056 */ 057//CHECKSTYLE:ON 058public class MemoryMappedFileManager extends OutputStreamManager { 059 /** 060 * Default length of region to map. 061 */ 062 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024; 063 private static final int MAX_REMAP_COUNT = 10; 064 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory(); 065 private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0; 066 067 private final boolean isForce; 068 private final int regionLength; 069 private final String advertiseURI; 070 private final RandomAccessFile randomAccessFile; 071 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>(); 072 private MappedByteBuffer mappedBuffer; 073 private long mappingOffset; 074 075 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os, 076 final boolean force, final long position, final int regionLength, final String advertiseURI, 077 final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException { 078 super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0])); 079 this.isForce = force; 080 this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile"); 081 this.regionLength = regionLength; 082 this.advertiseURI = advertiseURI; 083 this.isEndOfBatch.set(Boolean.FALSE); 084 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength); 085 this.byteBuffer = mappedBuffer; 086 this.mappingOffset = position; 087 } 088 089 /** 090 * Returns the MemoryMappedFileManager. 091 * 092 * @param fileName The name of the file to manage. 093 * @param append true if the file should be appended to, false if it should be overwritten. 094 * @param isForce true if the contents should be flushed to disk on every write 095 * @param regionLength The mapped region length. 096 * @param advertiseURI the URI to use when advertising the file 097 * @param layout The layout. 098 * @return A MemoryMappedFileManager for the File. 099 */ 100 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append, 101 final boolean isForce, final int regionLength, final String advertiseURI, 102 final Layout<? extends Serializable> layout) { 103 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength, 104 advertiseURI, layout), FACTORY); 105 } 106 107 public Boolean isEndOfBatch() { 108 return isEndOfBatch.get(); 109 } 110 111 public void setEndOfBatch(final boolean endOfBatch) { 112 this.isEndOfBatch.set(Boolean.valueOf(endOfBatch)); 113 } 114 115 @Override 116 protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) { 117 while (length > mappedBuffer.remaining()) { 118 final int chunk = mappedBuffer.remaining(); 119 mappedBuffer.put(bytes, offset, chunk); 120 offset += chunk; 121 length -= chunk; 122 remap(); 123 } 124 mappedBuffer.put(bytes, offset, length); 125 126 // no need to call flush() if force is true, 127 // already done in AbstractOutputStreamAppender.append 128 } 129 130 private synchronized void remap() { 131 final long offset = this.mappingOffset + mappedBuffer.position(); 132 final int length = mappedBuffer.remaining() + regionLength; 133 try { 134 unsafeUnmap(mappedBuffer); 135 final long fileLength = randomAccessFile.length() + regionLength; 136 LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(), 137 regionLength, fileLength); 138 139 final long startNanos = System.nanoTime(); 140 randomAccessFile.setLength(fileLength); 141 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 142 LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(), 143 millis); 144 145 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length); 146 this.byteBuffer = mappedBuffer; 147 mappingOffset = offset; 148 } catch (final Exception ex) { 149 logError("unable to remap", ex); 150 } 151 } 152 153 @Override 154 public synchronized void flush() { 155 mappedBuffer.force(); 156 } 157 158 @Override 159 public synchronized void close() { 160 final long position = mappedBuffer.position(); 161 final long length = mappingOffset + position; 162 try { 163 unsafeUnmap(mappedBuffer); 164 } catch (final Exception ex) { 165 logError("unable to unmap MappedBuffer", ex); 166 } 167 try { 168 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(), 169 length, mappingOffset, position); 170 randomAccessFile.setLength(length); 171 randomAccessFile.close(); 172 } catch (final IOException ex) { 173 logError("unable to close MemoryMappedFile", ex); 174 } 175 } 176 177 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start, 178 final int size) throws IOException { 179 for (int i = 1;; i++) { 180 try { 181 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size); 182 183 final long startNanos = System.nanoTime(); 184 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size); 185 map.order(ByteOrder.nativeOrder()); 186 187 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 188 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis); 189 190 return map; 191 } catch (final IOException e) { 192 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) { 193 throw e; 194 } 195 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e); 196 if (i < MAX_REMAP_COUNT) { 197 Thread.yield(); 198 } else { 199 try { 200 Thread.sleep(1); 201 } catch (final InterruptedException ignored) { 202 Thread.currentThread().interrupt(); 203 throw e; 204 } 205 } 206 } 207 } 208 } 209 210 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException { 211 LOGGER.debug("MMapAppender unmapping old buffer..."); 212 final long startNanos = System.nanoTime(); 213 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() { 214 @Override 215 public Object run() throws Exception { 216 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner"); 217 getCleanerMethod.setAccessible(true); 218 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance 219 final Method cleanMethod = cleaner.getClass().getMethod("clean"); 220 cleanMethod.invoke(cleaner); 221 return null; 222 } 223 }); 224 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC); 225 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis); 226 } 227 228 /** 229 * Returns the name of the File being managed. 230 * 231 * @return The name of the File being managed. 232 */ 233 public String getFileName() { 234 return getName(); 235 } 236 237 /** 238 * Returns the length of the memory mapped region. 239 * 240 * @return the length of the mapped region 241 */ 242 public int getRegionLength() { 243 return regionLength; 244 } 245 246 /** 247 * Returns {@code true} if the content of the buffer should be forced to the storage device on every write, 248 * {@code false} otherwise. 249 * 250 * @return whether each write should be force-sync'ed 251 */ 252 public boolean isImmediateFlush() { 253 return isForce; 254 } 255 256 /** 257 * Gets this FileManager's content format specified by: 258 * <p> 259 * Key: "fileURI" Value: provided "advertiseURI" param. 260 * </p> 261 * 262 * @return Map of content format keys supporting FileManager 263 */ 264 @Override 265 public Map<String, String> getContentFormat() { 266 final Map<String, String> result = new HashMap<>(super.getContentFormat()); 267 result.put("fileURI", advertiseURI); 268 return result; 269 } 270 271 @Override 272 protected void flushBuffer(final ByteBuffer buffer) { 273 // do nothing (do not call drain() to avoid spurious remapping) 274 } 275 276 @Override 277 public ByteBuffer getByteBuffer() { 278 return mappedBuffer; 279 } 280 281 @Override 282 public ByteBuffer drain(final ByteBuffer buf) { 283 remap(); 284 return mappedBuffer; 285 } 286 287 /** 288 * Factory Data. 289 */ 290 private static class FactoryData { 291 private final boolean append; 292 private final boolean force; 293 private final int regionLength; 294 private final String advertiseURI; 295 private final Layout<? extends Serializable> layout; 296 297 /** 298 * Constructor. 299 * 300 * @param append Append to existing file or truncate. 301 * @param force forces the memory content to be written to the storage device on every event 302 * @param regionLength length of the mapped region 303 */ 304 public FactoryData(final boolean append, final boolean force, final int regionLength, 305 final String advertiseURI, final Layout<? extends Serializable> layout) { 306 this.append = append; 307 this.force = force; 308 this.regionLength = regionLength; 309 this.advertiseURI = advertiseURI; 310 this.layout = layout; 311 } 312 } 313 314 /** 315 * Factory to create a MemoryMappedFileManager. 316 */ 317 private static class MemoryMappedFileManagerFactory 318 implements ManagerFactory<MemoryMappedFileManager, FactoryData> { 319 320 /** 321 * Create a MemoryMappedFileManager. 322 * 323 * @param name The name of the File. 324 * @param data The FactoryData 325 * @return The MemoryMappedFileManager for the File. 326 */ 327 @SuppressWarnings("resource") 328 @Override 329 public MemoryMappedFileManager createManager(final String name, final FactoryData data) { 330 final File file = new File(name); 331 final File parent = file.getParentFile(); 332 if (null != parent && !parent.exists()) { 333 parent.mkdirs(); 334 } 335 if (!data.append) { 336 file.delete(); 337 } 338 339 final boolean writeHeader = !data.append || !file.exists(); 340 final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM; 341 RandomAccessFile raf = null; 342 try { 343 raf = new RandomAccessFile(name, "rw"); 344 final long position = (data.append) ? raf.length() : 0; 345 raf.setLength(position + data.regionLength); 346 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength, 347 data.advertiseURI, data.layout, writeHeader); 348 } catch (final Exception ex) { 349 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex); 350 Closer.closeSilently(raf); 351 } 352 return null; 353 } 354 } 355}