001    /*
002     * Licensed to the Apache Software Foundation (ASF) under one or more
003     * contributor license agreements. See the NOTICE file distributed with
004     * this work for additional information regarding copyright ownership.
005     * The ASF licenses this file to You under the Apache license, Version 2.0
006     * (the "License"); you may not use this file except in compliance with
007     * the License. You may obtain a copy of the License at
008     *
009     *      http://www.apache.org/licenses/LICENSE-2.0
010     *
011     * Unless required by applicable law or agreed to in writing, software
012     * distributed under the License is distributed on an "AS IS" BASIS,
013     * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014     * See the license for the specific language governing permissions and
015     * limitations under the license.
016     */
017    package org.apache.logging.log4j.core.appender;
018    
019    import java.io.File;
020    import java.io.IOException;
021    import java.io.OutputStream;
022    import java.io.RandomAccessFile;
023    import java.io.Serializable;
024    import java.lang.reflect.Method;
025    import java.nio.ByteOrder;
026    import java.nio.MappedByteBuffer;
027    import java.nio.channels.FileChannel;
028    import java.security.AccessController;
029    import java.security.PrivilegedActionException;
030    import java.security.PrivilegedExceptionAction;
031    import java.util.HashMap;
032    import java.util.Map;
033    
034    import org.apache.logging.log4j.core.Layout;
035    import org.apache.logging.log4j.core.util.Assert;
036    import org.apache.logging.log4j.core.util.Closer;
037    
038    /**
039     * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
040     * memory and writes to this memory region.
041     * <p>
042     * 
043     * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
044     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
045     * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
046     * @see <a
047     *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
048     * 
049     * @since 2.1
050     */
051    public class MemoryMappedFileManager extends OutputStreamManager {
052        static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
053        private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
054    
055        private final boolean isForce;
056        private final int regionLength;
057        private final String advertiseURI;
058        private final RandomAccessFile randomAccessFile;
059        private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<Boolean>();
060        private MappedByteBuffer mappedBuffer;
061        private long mappingOffset;
062    
063        protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
064                final boolean force, final long position, final int regionLength, final String advertiseURI,
065                final Layout<? extends Serializable> layout) throws IOException {
066            super(os, fileName, layout);
067            this.isForce = force;
068            this.randomAccessFile = Assert.requireNonNull(file, "RandomAccessFile");
069            this.regionLength = regionLength;
070            this.advertiseURI = advertiseURI;
071            this.isEndOfBatch.set(Boolean.FALSE);
072            this.mappedBuffer = mmap(randomAccessFile.getChannel(), position, regionLength);
073            this.mappingOffset = position;
074        }
075    
076        /**
077         * Returns the MemoryMappedFileManager.
078         *
079         * @param fileName The name of the file to manage.
080         * @param append true if the file should be appended to, false if it should be overwritten.
081         * @param isForce true if the contents should be flushed to disk on every write
082         * @param regionLength The mapped region length.
083         * @param advertiseURI the URI to use when advertising the file
084         * @param layout The layout.
085         * @return A MemoryMappedFileManager for the File.
086         */
087        public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
088                final boolean isForce, final int regionLength, final String advertiseURI,
089                final Layout<? extends Serializable> layout) {
090            return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
091                    advertiseURI, layout), FACTORY);
092        }
093    
094        public Boolean isEndOfBatch() {
095            return isEndOfBatch.get();
096        }
097    
098        public void setEndOfBatch(final boolean isEndOfBatch) {
099            this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
100        }
101    
102        @Override
103        protected synchronized void write(final byte[] bytes, int offset, int length) {
104            super.write(bytes, offset, length); // writes to dummy output stream
105    
106            while (length > mappedBuffer.remaining()) {
107                final int chunk = mappedBuffer.remaining();
108                mappedBuffer.put(bytes, offset, chunk);
109                offset += chunk;
110                length -= chunk;
111                remap();
112            }
113            mappedBuffer.put(bytes, offset, length);
114    
115            // no need to call flush() if force is true,
116            // already done in AbstractOutputStreamAppender.append
117        }
118    
119        private synchronized void remap() {
120            final long offset = this.mappingOffset + mappedBuffer.position();
121            final int length = mappedBuffer.remaining() + regionLength;
122            try {
123                unsafeUnmap(mappedBuffer);
124                final long fileLength = randomAccessFile.length() + regionLength;
125                randomAccessFile.setLength(fileLength);
126                mappedBuffer = mmap(randomAccessFile.getChannel(), offset, length);
127                mappingOffset = offset;
128            } catch (final Exception ex) {
129                LOGGER.error("Unable to remap " + getName() + ". " + ex);
130            }
131        }
132    
133        @Override
134        public synchronized void flush() {
135            mappedBuffer.force();
136        }
137    
138        @Override
139        public synchronized void close() {
140            final long length = mappingOffset + mappedBuffer.position();
141            try {
142                unsafeUnmap(mappedBuffer);
143            } catch (final Exception ex) {
144                LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
145            }
146            try {
147                randomAccessFile.setLength(length);
148                randomAccessFile.close();
149            } catch (final IOException ex) {
150                LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
151            }
152        }
153    
154        public static MappedByteBuffer mmap(final FileChannel fileChannel, final long start, final int size) throws IOException {
155            for (int i = 1;; i++) {
156                try {
157                    final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
158                    map.order(ByteOrder.nativeOrder());
159                    return map;
160                } catch (final IOException e) {
161                    if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
162                        throw e;
163                    }
164                    if (i < 10) {
165                        Thread.yield();
166                    } else {
167                        try {
168                            Thread.sleep(1);
169                        } catch (final InterruptedException ignored) {
170                            Thread.currentThread().interrupt();
171                            throw e;
172                        }
173                    }
174                }
175            }
176        }
177    
178        private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
179            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
180                @Override
181                public Object run() throws Exception {
182                    final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
183                    getCleanerMethod.setAccessible(true);
184                    final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
185                    final Method cleanMethod = cleaner.getClass().getMethod("clean");
186                    cleanMethod.invoke(cleaner);
187                    return null;
188                }
189            });
190        }
191    
192        /**
193         * Returns the name of the File being managed.
194         *
195         * @return The name of the File being managed.
196         */
197        public String getFileName() {
198            return getName();
199        }
200    
201        /**
202         * Returns the length of the memory mapped region.
203         * 
204         * @return the length of the mapped region
205         */
206        public int getRegionLength() {
207            return regionLength;
208        }
209        
210        /**
211         * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
212         * {@code false} otherwise.
213         * @return whether each write should be force-sync'ed
214         */
215        public boolean isImmediateFlush() {
216            return isForce;
217        }
218    
219        /** {@code OutputStream} subclass that does not write anything. */
220        static class DummyOutputStream extends OutputStream {
221            @Override
222            public void write(final int b) throws IOException {
223            }
224    
225            @Override
226            public void write(final byte[] b, final int off, final int len) throws IOException {
227            }
228        }
229    
230        /**
231         * Gets this FileManager's content format specified by:
232         * <p>
233         * Key: "fileURI" Value: provided "advertiseURI" param.
234         * </p>
235         * 
236         * @return Map of content format keys supporting FileManager
237         */
238        @Override
239        public Map<String, String> getContentFormat() {
240            final Map<String, String> result = new HashMap<String, String>(super.getContentFormat());
241            result.put("fileURI", advertiseURI);
242            return result;
243        }
244    
245        /**
246         * Factory Data.
247         */
248        private static class FactoryData {
249            private final boolean append;
250            private final boolean force;
251            private final int regionLength;
252            private final String advertiseURI;
253            private final Layout<? extends Serializable> layout;
254    
255            /**
256             * Constructor.
257             *
258             * @param append Append to existing file or truncate.
259             * @param force forces the memory content to be written to the storage device on every event
260             * @param regionLength length of the mapped region
261             */
262            public FactoryData(final boolean append, final boolean force, final int regionLength,
263                    final String advertiseURI, final Layout<? extends Serializable> layout) {
264                this.append = append;
265                this.force = force;
266                this.regionLength = regionLength;
267                this.advertiseURI = advertiseURI;
268                this.layout = layout;
269            }
270        }
271    
272        /**
273         * Factory to create a MemoryMappedFileManager.
274         */
275        private static class MemoryMappedFileManagerFactory implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
276    
277            /**
278             * Create a MemoryMappedFileManager.
279             *
280             * @param name The name of the File.
281             * @param data The FactoryData
282             * @return The MemoryMappedFileManager for the File.
283             */
284            @SuppressWarnings("resource")
285            @Override
286            public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
287                final File file = new File(name);
288                final File parent = file.getParentFile();
289                if (null != parent && !parent.exists()) {
290                    parent.mkdirs();
291                }
292                if (!data.append) {
293                    file.delete();
294                }
295    
296                final OutputStream os = new DummyOutputStream();
297                RandomAccessFile raf = null;
298                try {
299                    raf = new RandomAccessFile(name, "rw");
300                    final long position = (data.append) ? raf.length() : 0;
301                    raf.setLength(position + data.regionLength);
302                    return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
303                            data.advertiseURI, data.layout);
304                } catch (final Exception ex) {
305                    LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
306                    Closer.closeSilently(raf);
307                }
308                return null;
309            }
310        }
311    }