001 /* 002 * Licensed to the Apache Software Foundation (ASF) under one or more 003 * contributor license agreements. See the NOTICE file distributed with 004 * this work for additional information regarding copyright ownership. 005 * The ASF licenses this file to You under the Apache license, Version 2.0 006 * (the "License"); you may not use this file except in compliance with 007 * the License. You may obtain a copy of the License at 008 * 009 * http://www.apache.org/licenses/LICENSE-2.0 010 * 011 * Unless required by applicable law or agreed to in writing, software 012 * distributed under the License is distributed on an "AS IS" BASIS, 013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 014 * See the license for the specific language governing permissions and 015 * limitations under the license. 016 */ 017 package org.apache.logging.log4j.core.appender; 018 019 import java.io.File; 020 import java.io.IOException; 021 import java.io.OutputStream; 022 import java.io.RandomAccessFile; 023 import java.io.Serializable; 024 import java.lang.reflect.Method; 025 import java.nio.ByteOrder; 026 import java.nio.MappedByteBuffer; 027 import java.nio.channels.FileChannel; 028 import java.security.AccessController; 029 import java.security.PrivilegedActionException; 030 import java.security.PrivilegedExceptionAction; 031 import java.util.HashMap; 032 import java.util.Map; 033 034 import org.apache.logging.log4j.core.Layout; 035 import org.apache.logging.log4j.core.util.Assert; 036 import org.apache.logging.log4j.core.util.Closer; 037 038 /** 039 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into 040 * memory and writes to this memory region. 041 * <p> 042 * 043 * @see <a 044 * href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a> 045 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a> 046 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a> 047 * @see <a 048 * href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a> 049 * 050 * @since 2.1 051 */ 052 public class MemoryMappedFileManager extends OutputStreamManager { 053 static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024; 054 private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory(); 055 056 private final boolean isForce; 057 private final int regionLength; 058 private final String advertiseURI; 059 private final RandomAccessFile randomAccessFile; 060 private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>(); 061 private MappedByteBuffer mappedBuffer; 062 private long mappingOffset; 063 064 protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os, 065 final boolean force, final long position, final int regionLength, final String advertiseURI, 066 final Layout<? extends Serializable> layout) throws IOException { 067 super(os, fileName, layout); 068 this.isForce = force; 069 this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile"); 070 this.regionLength = regionLength; 071 this.advertiseURI = advertiseURI; 072 this.isEndOfBatch.set(Boolean.FALSE); 073 this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength); 074 this.mappingOffset = position; 075 } 076 077 /** 078 * Returns the MemoryMappedFileManager. 079 * 080 * @param fileName The name of the file to manage. 081 * @param append true if the file should be appended to, false if it should be overwritten. 082 * @param isForce true if the contents should be flushed to disk on every write 083 * @param regionLength The mapped region length. 084 * @param advertiseURI the URI to use when advertising the file 085 * @param layout The layout. 086 * @return A MemoryMappedFileManager for the File. 087 */ 088 public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append, 089 final boolean isForce, final int regionLength, final String advertiseURI, 090 final Layout<? extends Serializable> layout) { 091 return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength, 092 advertiseURI, layout), FACTORY); 093 } 094 095 public Boolean isEndOfBatch() { 096 return isEndOfBatch.get(); 097 } 098 099 public void setEndOfBatch(final boolean isEndOfBatch) { 100 this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch)); 101 } 102 103 @Override 104 protected synchronized void write(final byte[] bytes, int offset, int length) { 105 super.write(bytes, offset, length); // writes to dummy output stream 106 107 while (length > mappedBuffer.remaining()) { 108 final int chunk = mappedBuffer.remaining(); 109 mappedBuffer.put(bytes, offset, chunk); 110 offset += chunk; 111 length -= chunk; 112 remap(); 113 } 114 mappedBuffer.put(bytes, offset, length); 115 116 // no need to call flush() if force is true, 117 // already done in AbstractOutputStreamAppender.append 118 } 119 120 private synchronized void remap() { 121 final long offset = this.mappingOffset + mappedBuffer.position(); 122 final int length = mappedBuffer.remaining() + regionLength; 123 try { 124 unsafeUnmap(mappedBuffer); 125 final long fileLength = randomAccessFile.length() + regionLength; 126 LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength); 127 128 long startNanos = System.nanoTime(); 129 randomAccessFile.setLength(fileLength); 130 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0)); 131 LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis); 132 133 mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length); 134 mappingOffset = offset; 135 } catch (final Exception ex) { 136 LOGGER.error("Unable to remap " + getName() + ". " + ex); 137 } 138 } 139 140 @Override 141 public synchronized void flush() { 142 mappedBuffer.force(); 143 } 144 145 @Override 146 public synchronized void close() { 147 final long position = mappedBuffer.position(); 148 final long length = mappingOffset + position; 149 try { 150 unsafeUnmap(mappedBuffer); 151 } catch (final Exception ex) { 152 LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex); 153 } 154 try { 155 LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(), 156 length, mappingOffset, position); 157 randomAccessFile.setLength(length); 158 randomAccessFile.close(); 159 } catch (final IOException ex) { 160 LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex); 161 } 162 } 163 164 public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start, 165 final int size) throws IOException { 166 for (int i = 1;; i++) { 167 try { 168 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size); 169 170 final long startNanos = System.nanoTime(); 171 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size); 172 map.order(ByteOrder.nativeOrder()); 173 174 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0)); 175 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis); 176 177 return map; 178 } catch (final IOException e) { 179 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) { 180 throw e; 181 } 182 LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e); 183 if (i < 10) { 184 Thread.yield(); 185 } else { 186 try { 187 Thread.sleep(1); 188 } catch (final InterruptedException ignored) { 189 Thread.currentThread().interrupt(); 190 throw e; 191 } 192 } 193 } 194 } 195 } 196 197 private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException { 198 LOGGER.debug("MMapAppender unmapping old buffer..."); 199 final long startNanos = System.nanoTime(); 200 AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() { 201 @Override 202 public Object run() throws Exception { 203 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner"); 204 getCleanerMethod.setAccessible(true); 205 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance 206 final Method cleanMethod = cleaner.getClass().getMethod("clean"); 207 cleanMethod.invoke(cleaner); 208 return null; 209 } 210 }); 211 final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0)); 212 LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis); 213 } 214 215 /** 216 * Returns the name of the File being managed. 217 * 218 * @return The name of the File being managed. 219 */ 220 public String getFileName() { 221 return getName(); 222 } 223 224 /** 225 * Returns the length of the memory mapped region. 226 * 227 * @return the length of the mapped region 228 */ 229 public int getRegionLength() { 230 return regionLength; 231 } 232 233 /** 234 * Returns {@code true} if the content of the buffer should be forced to the storage device on every write, 235 * {@code false} otherwise. 236 * 237 * @return whether each write should be force-sync'ed 238 */ 239 public boolean isImmediateFlush() { 240 return isForce; 241 } 242 243 /** {@code OutputStream} subclass that does not write anything. */ 244 static class DummyOutputStream extends OutputStream { 245 @Override 246 public void write(final int b) throws IOException { 247 } 248 249 @Override 250 public void write(final byte[] b, final int off, final int len) throws IOException { 251 } 252 } 253 254 /** 255 * Gets this FileManager's content format specified by: 256 * <p> 257 * Key: "fileURI" Value: provided "advertiseURI" param. 258 * </p> 259 * 260 * @return Map of content format keys supporting FileManager 261 */ 262 @Override 263 public Map<String, String> getContentFormat() { 264 final Map<String, String> result = new HashMap<String, String>(super.getContentFormat()); 265 result.put("fileURI", advertiseURI); 266 return result; 267 } 268 269 /** 270 * Factory Data. 271 */ 272 private static class FactoryData { 273 private final boolean append; 274 private final boolean force; 275 private final int regionLength; 276 private final String advertiseURI; 277 private final Layout<? extends Serializable> layout; 278 279 /** 280 * Constructor. 281 * 282 * @param append Append to existing file or truncate. 283 * @param force forces the memory content to be written to the storage device on every event 284 * @param regionLength length of the mapped region 285 */ 286 public FactoryData(final boolean append, final boolean force, final int regionLength, 287 final String advertiseURI, final Layout<? extends Serializable> layout) { 288 this.append = append; 289 this.force = force; 290 this.regionLength = regionLength; 291 this.advertiseURI = advertiseURI; 292 this.layout = layout; 293 } 294 } 295 296 /** 297 * Factory to create a MemoryMappedFileManager. 298 */ 299 private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> { 300 301 /** 302 * Create a MemoryMappedFileManager. 303 * 304 * @param name The name of the File. 305 * @param data The FactoryData 306 * @return The MemoryMappedFileManager for the File. 307 */ 308 @SuppressWarnings("resource") 309 @Override 310 public MemoryMappedFileManager createManager(final String name, final FactoryData data) { 311 final File file = new File(name); 312 final File parent = file.getParentFile(); 313 if (null != parent && !parent.exists()) { 314 parent.mkdirs(); 315 } 316 if (!data.append) { 317 file.delete(); 318 } 319 320 final OutputStream os = new DummyOutputStream(); 321 RandomAccessFile raf = null; 322 try { 323 raf = new RandomAccessFile(name, "rw"); 324 final long position = (data.append) ? raf.length() : 0; 325 raf.setLength(position + data.regionLength); 326 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength, 327 data.advertiseURI, data.layout); 328 } catch (final Exception ex) { 329 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex); 330 Closer.closeSilently(raf); 331 } 332 return null; 333 } 334 } 335 }