001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.appender;
018
019import java.io.File;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.RandomAccessFile;
023import java.io.Serializable;
024import java.lang.reflect.Method;
025import java.nio.ByteOrder;
026import java.nio.MappedByteBuffer;
027import java.nio.channels.FileChannel;
028import java.security.AccessController;
029import java.security.PrivilegedActionException;
030import java.security.PrivilegedExceptionAction;
031import java.util.HashMap;
032import java.util.Map;
033import java.util.Objects;
034
035import org.apache.logging.log4j.core.Layout;
036import org.apache.logging.log4j.core.util.Closer;
037import org.apache.logging.log4j.core.util.NullOutputStream;
038
039/**
040 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
041 * memory and writes to this memory region.
042 * <p>
043 * 
044 * @see <a
045 *      href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
046 *        http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
047 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
048 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
049 * @see <a
050 *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
051 *        http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
052 * 
053 * @since 2.1
054 */
055public class MemoryMappedFileManager extends OutputStreamManager {
056    /**
057     * 
058     */
059    private static final int MAX_REMAP_COUNT = 10;
060    static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
061    private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
062    private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
063
064    private final boolean isForce;
065    private final int regionLength;
066    private final String advertiseURI;
067    private final RandomAccessFile randomAccessFile;
068    private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
069    private MappedByteBuffer mappedBuffer;
070    private long mappingOffset;
071
072    protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
073            final boolean force, final long position, final int regionLength, final String advertiseURI,
074            final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
075        super(os, fileName, layout, writeHeader);
076        this.isForce = force;
077        this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
078        this.regionLength = regionLength;
079        this.advertiseURI = advertiseURI;
080        this.isEndOfBatch.set(Boolean.FALSE);
081        this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
082        this.mappingOffset = position;
083    }
084
085    /**
086     * Returns the MemoryMappedFileManager.
087     *
088     * @param fileName The name of the file to manage.
089     * @param append true if the file should be appended to, false if it should be overwritten.
090     * @param isForce true if the contents should be flushed to disk on every write
091     * @param regionLength The mapped region length.
092     * @param advertiseURI the URI to use when advertising the file
093     * @param layout The layout.
094     * @return A MemoryMappedFileManager for the File.
095     */
096    public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
097            final boolean isForce, final int regionLength, final String advertiseURI,
098            final Layout<? extends Serializable> layout) {
099        return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
100                advertiseURI, layout), FACTORY);
101    }
102
103    public Boolean isEndOfBatch() {
104        return isEndOfBatch.get();
105    }
106
107    public void setEndOfBatch(final boolean isEndOfBatch) {
108        this.isEndOfBatch.set(Boolean.valueOf(isEndOfBatch));
109    }
110
111    @Override
112    protected synchronized void write(final byte[] bytes, int offset, int length) {
113        super.write(bytes, offset, length); // writes to dummy output stream
114
115        while (length > mappedBuffer.remaining()) {
116            final int chunk = mappedBuffer.remaining();
117            mappedBuffer.put(bytes, offset, chunk);
118            offset += chunk;
119            length -= chunk;
120            remap();
121        }
122        mappedBuffer.put(bytes, offset, length);
123
124        // no need to call flush() if force is true,
125        // already done in AbstractOutputStreamAppender.append
126    }
127
128    private synchronized void remap() {
129        final long offset = this.mappingOffset + mappedBuffer.position();
130        final int length = mappedBuffer.remaining() + regionLength;
131        try {
132            unsafeUnmap(mappedBuffer);
133            final long fileLength = randomAccessFile.length() + regionLength;
134            LOGGER.debug("MMapAppender extending {} by {} bytes to {}", getFileName(), regionLength, fileLength);
135
136            final long startNanos = System.nanoTime();
137            randomAccessFile.setLength(fileLength);
138            final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
139            LOGGER.debug("MMapAppender extended {} OK in {} millis", getFileName(), millis);
140
141            mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
142            mappingOffset = offset;
143        } catch (final Exception ex) {
144            LOGGER.error("Unable to remap " + getName() + ". " + ex);
145        }
146    }
147
148    @Override
149    public synchronized void flush() {
150        mappedBuffer.force();
151    }
152
153    @Override
154    public synchronized void close() {
155        final long position = mappedBuffer.position();
156        final long length = mappingOffset + position;
157        try {
158            unsafeUnmap(mappedBuffer);
159        } catch (final Exception ex) {
160            LOGGER.error("Unable to unmap MappedBuffer " + getName() + ". " + ex);
161        }
162        try {
163            LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
164                    length, mappingOffset, position);
165            randomAccessFile.setLength(length);
166            randomAccessFile.close();
167        } catch (final IOException ex) {
168            LOGGER.error("Unable to close MemoryMappedFile " + getName() + ". " + ex);
169        }
170    }
171
172    public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
173            final int size) throws IOException {
174        for (int i = 1;; i++) {
175            try {
176                LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
177
178                final long startNanos = System.nanoTime();
179                final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
180                map.order(ByteOrder.nativeOrder());
181
182                final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
183                LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
184
185                return map;
186            } catch (final IOException e) {
187                if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
188                    throw e;
189                }
190                LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
191                if (i < MAX_REMAP_COUNT) {
192                    Thread.yield();
193                } else {
194                    try {
195                        Thread.sleep(1);
196                    } catch (final InterruptedException ignored) {
197                        Thread.currentThread().interrupt();
198                        throw e;
199                    }
200                }
201            }
202        }
203    }
204
205    private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
206        LOGGER.debug("MMapAppender unmapping old buffer...");
207        final long startNanos = System.nanoTime();
208        AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
209            @Override
210            public Object run() throws Exception {
211                final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
212                getCleanerMethod.setAccessible(true);
213                final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
214                final Method cleanMethod = cleaner.getClass().getMethod("clean");
215                cleanMethod.invoke(cleaner);
216                return null;
217            }
218        });
219        final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
220        LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
221    }
222
223    /**
224     * Returns the name of the File being managed.
225     *
226     * @return The name of the File being managed.
227     */
228    public String getFileName() {
229        return getName();
230    }
231
232    /**
233     * Returns the length of the memory mapped region.
234     * 
235     * @return the length of the mapped region
236     */
237    public int getRegionLength() {
238        return regionLength;
239    }
240
241    /**
242     * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
243     * {@code false} otherwise.
244     * 
245     * @return whether each write should be force-sync'ed
246     */
247    public boolean isImmediateFlush() {
248        return isForce;
249    }
250
251    /**
252     * Gets this FileManager's content format specified by:
253     * <p>
254     * Key: "fileURI" Value: provided "advertiseURI" param.
255     * </p>
256     * 
257     * @return Map of content format keys supporting FileManager
258     */
259    @Override
260    public Map<String, String> getContentFormat() {
261        final Map<String, String> result = new HashMap<>(super.getContentFormat());
262        result.put("fileURI", advertiseURI);
263        return result;
264    }
265
266    /**
267     * Factory Data.
268     */
269    private static class FactoryData {
270        private final boolean append;
271        private final boolean force;
272        private final int regionLength;
273        private final String advertiseURI;
274        private final Layout<? extends Serializable> layout;
275
276        /**
277         * Constructor.
278         *
279         * @param append Append to existing file or truncate.
280         * @param force forces the memory content to be written to the storage device on every event
281         * @param regionLength length of the mapped region
282         */
283        public FactoryData(final boolean append, final boolean force, final int regionLength,
284                final String advertiseURI, final Layout<? extends Serializable> layout) {
285            this.append = append;
286            this.force = force;
287            this.regionLength = regionLength;
288            this.advertiseURI = advertiseURI;
289            this.layout = layout;
290        }
291    }
292
293    /**
294     * Factory to create a MemoryMappedFileManager.
295     */
296    private static class MemoryMappedFileManagerFactory 
297            implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
298
299        /**
300         * Create a MemoryMappedFileManager.
301         *
302         * @param name The name of the File.
303         * @param data The FactoryData
304         * @return The MemoryMappedFileManager for the File.
305         */
306        @SuppressWarnings("resource")
307        @Override
308        public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
309            final File file = new File(name);
310            final File parent = file.getParentFile();
311            if (null != parent && !parent.exists()) {
312                parent.mkdirs();
313            }
314            if (!data.append) {
315                file.delete();
316            }
317
318            final boolean writeHeader = !data.append || !file.exists();
319            final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
320            RandomAccessFile raf = null;
321            try {
322                raf = new RandomAccessFile(name, "rw");
323                final long position = (data.append) ? raf.length() : 0;
324                raf.setLength(position + data.regionLength);
325                return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
326                        data.advertiseURI, data.layout, writeHeader);
327            } catch (final Exception ex) {
328                LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex);
329                Closer.closeSilently(raf);
330            }
331            return null;
332        }
333    }
334}