View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache license, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License. You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the license for the specific language governing permissions and
15   * limitations under the license.
16   */
17  package org.apache.logging.log4j.core.appender;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.io.RandomAccessFile;
23  import java.io.Serializable;
24  import java.lang.reflect.Method;
25  import java.nio.ByteOrder;
26  import java.nio.MappedByteBuffer;
27  import java.nio.channels.FileChannel;
28  import java.security.AccessController;
29  import java.security.PrivilegedActionException;
30  import java.security.PrivilegedExceptionAction;
31  import java.util.HashMap;
32  import java.util.Map;
33  import java.util.Objects;
34  
35  import org.apache.logging.log4j.core.Layout;
36  import org.apache.logging.log4j.core.util.Closer;
37  import org.apache.logging.log4j.core.util.NullOutputStream;
38  
39  /**
40   * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
41   * memory and writes to this memory region.
42   * <p>
43   * 
44   * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
45   *      http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>;
46   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
47   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
48   * @see <a
49   *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
50   *      http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>;
51   * 
52   * @since 2.1
53   */
54  public class MemoryMappedFileManager extends OutputStreamManager {
55      /**
56       * Default length of region to map.
57       */
58      static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
59      private static final int MAX_REMAP_COUNT = 10;
60      private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
61      private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
62  
63      private final boolean isForce;
64      private final int regionLength;
65      private final String advertiseURI;
66      private final RandomAccessFile randomAccessFile;
67      private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
68      private MappedByteBuffer mappedBuffer;
69      private long mappingOffset;
70  
71      protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
72              final boolean force, final long position, final int regionLength, final String advertiseURI,
73              final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
74          super(os, fileName, layout, writeHeader);
75          this.isForce = force;
76          this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
77          this.regionLength = regionLength;
78          this.advertiseURI = advertiseURI;
79          this.isEndOfBatch.set(Boolean.FALSE);
80          this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
81          this.mappingOffset = position;
82      }
83  
84      /**
85       * Returns the MemoryMappedFileManager.
86       *
87       * @param fileName The name of the file to manage.
88       * @param append true if the file should be appended to, false if it should be overwritten.
89       * @param isForce true if the contents should be flushed to disk on every write
90       * @param regionLength The mapped region length.
91       * @param advertiseURI the URI to use when advertising the file
92       * @param layout The layout.
93       * @return A MemoryMappedFileManager for the File.
94       */
95      public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
96              final boolean isForce, final int regionLength, final String advertiseURI,
97              final Layout<? extends Serializable> layout) {
98          return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
99                  advertiseURI, layout), FACTORY);
100     }
101 
102     public Boolean isEndOfBatch() {
103         return isEndOfBatch.get();
104     }
105 
106     public void setEndOfBatch(final boolean endOfBatch) {
107         this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
108     }
109 
110     @Override
111     protected synchronized void write(final byte[] bytes, int offset, int length) {
112         super.write(bytes, offset, length); // writes to dummy output stream
113 
114         while (length > mappedBuffer.remaining()) {
115             final int chunk = mappedBuffer.remaining();
116             mappedBuffer.put(bytes, offset, chunk);
117             offset += chunk;
118             length -= chunk;
119             remap();
120         }
121         mappedBuffer.put(bytes, offset, length);
122 
123         // no need to call flush() if force is true,
124         // already done in AbstractOutputStreamAppender.append
125     }
126 
127     private synchronized void remap() {
128         final long offset = this.mappingOffset + mappedBuffer.position();
129         final int length = mappedBuffer.remaining() + regionLength;
130         try {
131             unsafeUnmap(mappedBuffer);
132             final long fileLength = randomAccessFile.length() + regionLength;
133             LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
134 
135             final long startNanos = System.nanoTime();
136             randomAccessFile.setLength(fileLength);
137             final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
138             LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
139 
140             mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
141             mappingOffset = offset;
142         } catch (final Exception ex) {
143             LOGGER.error("Unable to remap " + getName() + ". " + ex);
144         }
145     }
146 
147     @Override
148     public synchronized void flush() {
149         mappedBuffer.force();
150     }
151 
152     @Override
153     public synchronized void close() {
154         final long position = mappedBuffer.position();
155         final long length = mappingOffset + position;
156         try {
157             unsafeUnmap(mappedBuffer);
158         } catch (final Exception ex) {
159             LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
160         }
161         try {
162             LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
163                     length, mappingOffset, position);
164             randomAccessFile.setLength(length);
165             randomAccessFile.close();
166         } catch (final IOException ex) {
167             LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
168         }
169     }
170 
171     public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
172             final int size) throws IOException {
173         for (int i = 1;; i++) {
174             try {
175                 LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
176 
177                 final long startNanos = System.nanoTime();
178                 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
179                 map.order(ByteOrder.nativeOrder());
180 
181                 final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
182                 LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
183 
184                 return map;
185             } catch (final IOException e) {
186                 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
187                     throw e;
188                 }
189                 LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
190                 if (i < MAX_REMAP_COUNT) {
191                     Thread.yield();
192                 } else {
193                     try {
194                         Thread.sleep(1);
195                     } catch (final InterruptedException ignored) {
196                         Thread.currentThread().interrupt();
197                         throw e;
198                     }
199                 }
200             }
201         }
202     }
203 
204     private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
205         LOGGER.debug("MMapAppender unmapping old buffer...");
206         final long startNanos = System.nanoTime();
207         AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
208             @Override
209             public Object run() throws Exception {
210                 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
211                 getCleanerMethod.setAccessible(true);
212                 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
213                 final Method cleanMethod = cleaner.getClass().getMethod("clean");
214                 cleanMethod.invoke(cleaner);
215                 return null;
216             }
217         });
218         final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
219         LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
220     }
221 
222     /**
223      * Returns the name of the File being managed.
224      *
225      * @return The name of the File being managed.
226      */
227     public String getFileName() {
228         return getName();
229     }
230 
231     /**
232      * Returns the length of the memory mapped region.
233      * 
234      * @return the length of the mapped region
235      */
236     public int getRegionLength() {
237         return regionLength;
238     }
239 
240     /**
241      * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
242      * {@code false} otherwise.
243      * 
244      * @return whether each write should be force-sync'ed
245      */
246     public boolean isImmediateFlush() {
247         return isForce;
248     }
249 
250     /**
251      * Gets this FileManager's content format specified by:
252      * <p>
253      * Key: "fileURI" Value: provided "advertiseURI" param.
254      * </p>
255      * 
256      * @return Map of content format keys supporting FileManager
257      */
258     @Override
259     public Map<String, String> getContentFormat() {
260         final Map<String, String> result = new HashMap<>(super.getContentFormat());
261         result.put("fileURI", advertiseURI);
262         return result;
263     }
264 
265     /**
266      * Factory Data.
267      */
268     private static class FactoryData {
269         private final boolean append;
270         private final boolean force;
271         private final int regionLength;
272         private final String advertiseURI;
273         private final Layout<? extends Serializable> layout;
274 
275         /**
276          * Constructor.
277          *
278          * @param append Append to existing file or truncate.
279          * @param force forces the memory content to be written to the storage device on every event
280          * @param regionLength length of the mapped region
281          */
282         public FactoryData(final boolean append, final boolean force, final int regionLength,
283                 final String advertiseURI, final Layout<? extends Serializable> layout) {
284             this.append = append;
285             this.force = force;
286             this.regionLength = regionLength;
287             this.advertiseURI = advertiseURI;
288             this.layout = layout;
289         }
290     }
291 
292     /**
293      * Factory to create a MemoryMappedFileManager.
294      */
295     private static class MemoryMappedFileManagerFactory
296             implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
297 
298         /**
299          * Create a MemoryMappedFileManager.
300          *
301          * @param name The name of the File.
302          * @param data The FactoryData
303          * @return The MemoryMappedFileManager for the File.
304          */
305         @SuppressWarnings("resource")
306         @Override
307         public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
308             final File file = new File(name);
309             final File parent = file.getParentFile();
310             if (null != parent && !parent.exists()) {
311                 parent.mkdirs();
312             }
313             if (!data.append) {
314                 file.delete();
315             }
316 
317             final boolean writeHeader = !data.append || !file.exists();
318             final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
319             RandomAccessFile raf = null;
320             try {
321                 raf = new RandomAccessFile(name, "rw");
322                 final long position = (data.append) ? raf.length() : 0;
323                 raf.setLength(position + data.regionLength);
324                 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
325                         data.advertiseURI, data.layout, writeHeader);
326             } catch (final Exception ex) {
327                 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
328                 Closer.closeSilently(raf);
329             }
330             return null;
331         }
332     }
333 }