001    /*
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements. See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache license, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License. You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the license for the specific language governing permissions and
015     * limitations under the license.
016     */
017    package org.apache.logging.log4j.core.appender;
018    
019    import java.io.File;
020    import java.io.IOException;
021    import java.io.OutputStream;
022    import java.io.RandomAccessFile;
023    import java.io.Serializable;
024    import java.lang.reflect.Method;
025    import java.nio.ByteOrder;
026    import java.nio.MappedByteBuffer;
027    import java.nio.channels.FileChannel;
028    import java.security.AccessController;
029    import java.security.PrivilegedActionException;
030    import java.security.PrivilegedExceptionAction;
031    import java.util.HashMap;
032    import java.util.Map;
033    
034    import org.apache.logging.log4j.core.Layout;
035    import org.apache.logging.log4j.core.util.Assert;
036    import org.apache.logging.log4j.core.util.Closer;
037    
038    /**
039     * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
040     * memory and writes to this memory region.
041     * <p>
042     * 
043     * @see <a
044     *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
045     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
046     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
047     * @see <a
048     *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
049     * 
050     * @since 2.1
051     */
052    public class MemoryMappedFileManager extends OutputStreamManager {
053        static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
054        private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
055    
056        private final boolean isForce;
057        private final int regionLength;
058        private final String advertiseURI;
059        private final RandomAccessFile randomAccessFile;
060        private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
061        private MappedByteBuffer mappedBuffer;
062        private long mappingOffset;
063    
064        protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
065                final boolean force, final long position, final int regionLength, final String advertiseURI,
066                final Layout<? extends Serializable> layout) throws IOException {
067            super(os, fileName, layout);
068            this.isForce = force;
069            this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
070            this.regionLength = regionLength;
071            this.advertiseURI = advertiseURI;
072            this.isEndOfBatch.set(Boolean.FALSE);
073            this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
074            this.mappingOffset = position;
075        }
076    
077        /**
078         * Returns the MemoryMappedFileManager.
079         *
080         * @param fileName The name of the file to manage.
081         * @param append true if the file should be appended to, false if it should be overwritten.
082         * @param isForce true if the contents should be flushed to disk on every write
083         * @param regionLength The mapped region length.
084         * @param advertiseURI the URI to use when advertising the file
085         * @param layout The layout.
086         * @return A MemoryMappedFileManager for the File.
087         */
088        public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
089                final boolean isForce, final int regionLength, final String advertiseURI,
090                final Layout<? extends Serializable> layout) {
091            return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
092                    advertiseURI, layout), FACTORY);
093        }
094    
095        public Boolean isEndOfBatch() {
096            return isEndOfBatch.get();
097        }
098    
099        public void setEndOfBatch(final boolean isEndOfBatch) {
100            this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
101        }
102    
103        @Override
104        protected synchronized void write(final byte[] bytes, int offset, int length) {
105            super.write(bytes, offset, length); // writes to dummy output stream
106    
107            while (length > mappedBuffer.remaining()) {
108                final int chunk = mappedBuffer.remaining();
109                mappedBuffer.put(bytes, offset, chunk);
110                offset += chunk;
111                length -= chunk;
112                remap();
113            }
114            mappedBuffer.put(bytes, offset, length);
115    
116            // no need to call flush() if force is true,
117            // already done in AbstractOutputStreamAppender.append
118        }
119    
120        private synchronized void remap() {
121            final long offset = this.mappingOffset + mappedBuffer.position();
122            final int length = mappedBuffer.remaining() + regionLength;
123            try {
124                unsafeUnmap(mappedBuffer);
125                final long fileLength = randomAccessFile.length() + regionLength;
126                LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
127    
128                long startNanos = System.nanoTime();
129                randomAccessFile.setLength(fileLength);
130                final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
131                LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
132    
133                mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
134                mappingOffset = offset;
135            } catch (final Exception ex) {
136                LOGGER.error("Unable to remap " + getName() + ". " + ex);
137            }
138        }
139    
140        @Override
141        public synchronized void flush() {
142            mappedBuffer.force();
143        }
144    
145        @Override
146        public synchronized void close() {
147            final long position = mappedBuffer.position();
148            final long length = mappingOffset + position;
149            try {
150                unsafeUnmap(mappedBuffer);
151            } catch (final Exception ex) {
152                LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
153            }
154            try {
155                LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
156                        length, mappingOffset, position);
157                randomAccessFile.setLength(length);
158                randomAccessFile.close();
159            } catch (final IOException ex) {
160                LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
161            }
162        }
163    
164        public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
165                final int size) throws IOException {
166            for (int i = 1;; i++) {
167                try {
168                    LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
169    
170                    final long startNanos = System.nanoTime();
171                    final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
172                    map.order(ByteOrder.nativeOrder());
173    
174                    final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
175                    LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
176    
177                    return map;
178                } catch (final IOException e) {
179                    if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
180                        throw e;
181                    }
182                    LOGGER.debug("Remap attempt {}/10 failed. Retrying...", i, e);
183                    if (i < 10) {
184                        Thread.yield();
185                    } else {
186                        try {
187                            Thread.sleep(1);
188                        } catch (final InterruptedException ignored) {
189                            Thread.currentThread().interrupt();
190                            throw e;
191                        }
192                    }
193                }
194            }
195        }
196    
197        private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
198            LOGGER.debug("MMapAppender unmapping old buffer...");
199            final long startNanos = System.nanoTime();
200            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
201                @Override
202                public Object run() throws Exception {
203                    final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
204                    getCleanerMethod.setAccessible(true);
205                    final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
206                    final Method cleanMethod = cleaner.getClass().getMethod("clean");
207                    cleanMethod.invoke(cleaner);
208                    return null;
209                }
210            });
211            final float millis = (float) ((System.nanoTime() - startNanos) / (1000.0 * 1000.0));
212            LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
213        }
214    
215        /**
216         * Returns the name of the File being managed.
217         *
218         * @return The name of the File being managed.
219         */
220        public String getFileName() {
221            return getName();
222        }
223    
224        /**
225         * Returns the length of the memory mapped region.
226         * 
227         * @return the length of the mapped region
228         */
229        public int getRegionLength() {
230            return regionLength;
231        }
232    
233        /**
234         * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
235         * {@code false} otherwise.
236         * 
237         * @return whether each write should be force-sync'ed
238         */
239        public boolean isImmediateFlush() {
240            return isForce;
241        }
242    
243        /** {@code OutputStream} subclass that does not write anything. */
244        static class DummyOutputStream extends OutputStream {
245            @Override
246            public void write(final int b) throws IOException {
247            }
248    
249            @Override
250            public void write(final byte[] b, final int off, final int len) throws IOException {
251            }
252        }
253    
254        /**
255         * Gets this FileManager's content format specified by:
256         * <p>
257         * Key: "fileURI" Value: provided "advertiseURI" param.
258         * </p>
259         * 
260         * @return Map of content format keys supporting FileManager
261         */
262        @Override
263        public Map<String, String> getContentFormat() {
264            final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
265            result.put("fileURI", advertiseURI);
266            return result;
267        }
268    
269        /**
270         * Factory Data.
271         */
272        private static class FactoryData {
273            private final boolean append;
274            private final boolean force;
275            private final int regionLength;
276            private final String advertiseURI;
277            private final Layout<? extends Serializable> layout;
278    
279            /**
280             * Constructor.
281             *
282             * @param append Append to existing file or truncate.
283             * @param force forces the memory content to be written to the storage device on every event
284             * @param regionLength length of the mapped region
285             */
286            public FactoryData(final boolean append, final boolean force, final int regionLength,
287                    final String advertiseURI, final Layout<? extends Serializable> layout) {
288                this.append = append;
289                this.force = force;
290                this.regionLength = regionLength;
291                this.advertiseURI = advertiseURI;
292                this.layout = layout;
293            }
294        }
295    
296        /**
297         * Factory to create a MemoryMappedFileManager.
298         */
299        private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
300    
301            /**
302             * Create a MemoryMappedFileManager.
303             *
304             * @param name The name of the File.
305             * @param data The FactoryData
306             * @return The MemoryMappedFileManager for the File.
307             */
308            @SuppressWarnings("resource")
309            @Override
310            public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
311                final File file = new File(name);
312                final File parent = file.getParentFile();
313                if (null != parent && !parent.exists()) {
314                    parent.mkdirs();
315                }
316                if (!data.append) {
317                    file.delete();
318                }
319    
320                final OutputStream os = new DummyOutputStream();
321                RandomAccessFile raf = null;
322                try {
323                    raf = new RandomAccessFile(name, "rw");
324                    final long position = (data.append) ? raf.length() : 0;
325                    raf.setLength(position + data.regionLength);
326                    return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
327                            data.advertiseURI, data.layout);
328                } catch (final Exception ex) {
329                    LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
330                    Closer.closeSilently(raf);
331                }
332                return null;
333            }
334        }
335    }