View Javadoc
1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with
4    * this work for additional information regarding copyright ownership.
5    * The ASF licenses this file to You under the Apache license, Version 2.0
6    * (the "License"); you may not use this file except in compliance with
7    * the License. You may obtain a copy of the License at
8    *
9    *      http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS,
13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14   * See the license for the specific language governing permissions and
15   * limitations under the license.
16   */
17  package org.apache.logging.log4j.core.appender;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.io.RandomAccessFile;
23  import java.io.Serializable;
24  import java.lang.reflect.Method;
25  import java.nio.ByteOrder;
26  import java.nio.MappedByteBuffer;
27  import java.nio.channels.FileChannel;
28  import java.security.AccessController;
29  import java.security.PrivilegedActionException;
30  import java.security.PrivilegedExceptionAction;
31  import java.util.HashMap;
32  import java.util.Map;
33  
34  import org.apache.logging.log4j.core.Layout;
35  import org.apache.logging.log4j.core.util.Assert;
36  import org.apache.logging.log4j.core.util.Closer;
37  
38  /**
39   * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
40   * memory and writes to this memory region.
41   * <p>
42   * 
43   * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
44   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
45   * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
46   * @see <a
47   *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
48   * 
49   * @since 2.1
50   */
51  public class MemoryMappedFileManager extends OutputStreamManager {
52      static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
53      private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
54  
55      private final boolean isForce;
56      private final int regionLength;
57      private final String advertiseURI;
58      private final RandomAccessFile randomAccessFile;
59      private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
60      private MappedByteBuffer mappedBuffer;
61      private long mappingOffset;
62  
63      protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
64              final boolean force, final long position, final int regionLength, final String advertiseURI,
65              final Layout<? extends Serializable> layout) throws IOException {
66          super(os, fileName, layout);
67          this.isForce = force;
68          this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
69          this.regionLength = regionLength;
70          this.advertiseURI = advertiseURI;
71          this.isEndOfBatch.set(Boolean.FALSE);
72          this.mappedBuffer = mmap(randomAccessFile.getChannel(), position, regionLength);
73          this.mappingOffset = position;
74      }
75  
76      /**
77       * Returns the MemoryMappedFileManager.
78       *
79       * @param fileName The name of the file to manage.
80       * @param append true if the file should be appended to, false if it should be overwritten.
81       * @param isForce true if the contents should be flushed to disk on every write
82       * @param regionLength The mapped region length.
83       * @param advertiseURI the URI to use when advertising the file
84       * @param layout The layout.
85       * @return A MemoryMappedFileManager for the File.
86       */
87      public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
88              final boolean isForce, final int regionLength, final String advertiseURI,
89              final Layout<? extends Serializable> layout) {
90          return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
91                  advertiseURI, layout), FACTORY);
92      }
93  
94      public Boolean isEndOfBatch() {
95          return isEndOfBatch.get();
96      }
97  
98      public void setEndOfBatch(final boolean isEndOfBatch) {
99          this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
100     }
101 
102     @Override
103     protected synchronized void write(final byte[] bytes, int offset, int length) {
104         super.write(bytes, offset, length); // writes to dummy output stream
105 
106         while (length > mappedBuffer.remaining()) {
107             final int chunk = mappedBuffer.remaining();
108             mappedBuffer.put(bytes, offset, chunk);
109             offset += chunk;
110             length -= chunk;
111             remap();
112         }
113         mappedBuffer.put(bytes, offset, length);
114 
115         // no need to call flush() if force is true,
116         // already done in AbstractOutputStreamAppender.append
117     }
118 
119     private synchronized void remap() {
120         final long offset = this.mappingOffset + mappedBuffer.position();
121         final int length = mappedBuffer.remaining() + regionLength;
122         try {
123             unsafeUnmap(mappedBuffer);
124             final long fileLength = randomAccessFile.length() + regionLength;
125             randomAccessFile.setLength(fileLength);
126             mappedBuffer = mmap(randomAccessFile.getChannel(), offset, length);
127             mappingOffset = offset;
128         } catch (final Exception ex) {
129             LOGGER.error("Unable to remap " + getName() + ". " + ex);
130         }
131     }
132 
133     @Override
134     public synchronized void flush() {
135         mappedBuffer.force();
136     }
137 
138     @Override
139     public synchronized void close() {
140         final long length = mappingOffset + mappedBuffer.position();
141         try {
142             unsafeUnmap(mappedBuffer);
143         } catch (final Exception ex) {
144             LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
145         }
146         try {
147             randomAccessFile.setLength(length);
148             randomAccessFile.close();
149         } catch (final IOException ex) {
150             LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
151         }
152     }
153 
154     public static MappedByteBuffer mmap(final FileChannel fileChannel, final long start, final int size) throws IOException {
155         for (int i = 1;; i++) {
156             try {
157                 final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
158                 map.order(ByteOrder.nativeOrder());
159                 return map;
160             } catch (final IOException e) {
161                 if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
162                     throw e;
163                 }
164                 if (i < 10) {
165                     Thread.yield();
166                 } else {
167                     try {
168                         Thread.sleep(1);
169                     } catch (final InterruptedException ignored) {
170                         Thread.currentThread().interrupt();
171                         throw e;
172                     }
173                 }
174             }
175         }
176     }
177 
178     private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
179         AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
180             @Override
181             public Object run() throws Exception {
182                 final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
183                 getCleanerMethod.setAccessible(true);
184                 final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
185                 final Method cleanMethod = cleaner.getClass().getMethod("clean");
186                 cleanMethod.invoke(cleaner);
187                 return null;
188             }
189         });
190     }
191 
192     /**
193      * Returns the name of the File being managed.
194      *
195      * @return The name of the File being managed.
196      */
197     public String getFileName() {
198         return getName();
199     }
200 
201     /**
202      * Returns the length of the memory mapped region.
203      * 
204      * @return the length of the mapped region
205      */
206     public int getRegionLength() {
207         return regionLength;
208     }
209     
210     /**
211      * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
212      * {@code false} otherwise.
213      * @return whether each write should be force-sync'ed
214      */
215     public boolean isImmediateFlush() {
216         return isForce;
217     }
218 
219     /** {@code OutputStream} subclass that does not write anything. */
220     static class DummyOutputStream extends OutputStream {
221         @Override
222         public void write(final int b) throws IOException {
223         }
224 
225         @Override
226         public void write(final byte[] b, final int off, final int len) throws IOException {
227         }
228     }
229 
230     /**
231      * Gets this FileManager's content format specified by:
232      * <p>
233      * Key: "fileURI" Value: provided "advertiseURI" param.
234      * </p>
235      * 
236      * @return Map of content format keys supporting FileManager
237      */
238     @Override
239     public Map<String, String> getContentFormat() {
240         final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
241         result.put("fileURI", advertiseURI);
242         return result;
243     }
244 
245     /**
246      * Factory Data.
247      */
248     private static class FactoryData {
249         private final boolean append;
250         private final boolean force;
251         private final int regionLength;
252         private final String advertiseURI;
253         private final Layout<? extends Serializable> layout;
254 
255         /**
256          * Constructor.
257          *
258          * @param append Append to existing file or truncate.
259          * @param force forces the memory content to be written to the storage device on every event
260          * @param regionLength length of the mapped region
261          */
262         public FactoryData(final boolean append, final boolean force, final int regionLength,
263                 final String advertiseURI, final Layout<? extends Serializable> layout) {
264             this.append = append;
265             this.force = force;
266             this.regionLength = regionLength;
267             this.advertiseURI = advertiseURI;
268             this.layout = layout;
269         }
270     }
271 
272     /**
273      * Factory to create a MemoryMappedFileManager.
274      */
275     private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
276 
277         /**
278          * Create a MemoryMappedFileManager.
279          *
280          * @param name The name of the File.
281          * @param data The FactoryData
282          * @return The MemoryMappedFileManager for the File.
283          */
284         @SuppressWarnings("resource")
285         @Override
286         public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
287             final File file = new File(name);
288             final File parent = file.getParentFile();
289             if (null != parent && !parent.exists()) {
290                 parent.mkdirs();
291             }
292             if (!data.append) {
293                 file.delete();
294             }
295 
296             final OutputStream os = new DummyOutputStream();
297             RandomAccessFile raf = null;
298             try {
299                 raf = new RandomAccessFile(name, "rw");
300                 final long position = (data.append) ? raf.length() : 0;
301                 raf.setLength(position + data.regionLength);
302                 return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
303                         data.advertiseURI, data.layout);
304             } catch (final Exception ex) {
305                 LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
306                 Closer.closeSilently(raf);
307             }
308             return null;
309         }
310     }
311 }