001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements. See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache license, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License. You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the license for the specific language governing permissions and
015 * limitations under the license.
016 */
017package org.apache.logging.log4j.core.appender;
018
019import java.io.File;
020import java.io.IOException;
021import java.io.OutputStream;
022import java.io.RandomAccessFile;
023import java.io.Serializable;
024import java.lang.reflect.Method;
025import java.nio.ByteBuffer;
026import java.nio.ByteOrder;
027import java.nio.MappedByteBuffer;
028import java.nio.channels.FileChannel;
029import java.security.AccessController;
030import java.security.PrivilegedActionException;
031import java.security.PrivilegedExceptionAction;
032import java.util.HashMap;
033import java.util.Map;
034import java.util.Objects;
035
036import org.apache.logging.log4j.core.Layout;
037import org.apache.logging.log4j.core.util.Closer;
038import org.apache.logging.log4j.core.util.NullOutputStream;
039
040//Lines too long...
041//CHECKSTYLE:OFF
042/**
043 * Extends OutputStreamManager but instead of using a buffered output stream, this class maps a region of a file into
044 * memory and writes to this memory region.
045 * <p>
046 *
047 * @see <a href="http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java">
048 *      http://www.codeproject.com/Tips/683614/Things-to-Know-about-Memory-Mapped-File-in-Java</a>
049 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=6893654">http://bugs.java.com/view_bug.do?bug_id=6893654</a>
050 * @see <a href="http://bugs.java.com/view_bug.do?bug_id=4724038">http://bugs.java.com/view_bug.do?bug_id=4724038</a>
051 * @see <a
052 *      href="http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation">
053 *      http://stackoverflow.com/questions/9261316/memory-mapped-mappedbytebuffer-or-direct-bytebuffer-for-db-implementation</a>
054 *
055 * @since 2.1
056 */
057//CHECKSTYLE:ON
058public class MemoryMappedFileManager extends OutputStreamManager {
059    /**
060     * Default length of region to map.
061     */
062    static final int DEFAULT_REGION_LENGTH = 32 * 1024 * 1024;
063    private static final int MAX_REMAP_COUNT = 10;
064    private static final MemoryMappedFileManagerFactory FACTORY = new MemoryMappedFileManagerFactory();
065    private static final double NANOS_PER_MILLISEC = 1000.0 * 1000.0;
066
067    private final boolean isForce;
068    private final int regionLength;
069    private final String advertiseURI;
070    private final RandomAccessFile randomAccessFile;
071    private final ThreadLocal<Boolean> isEndOfBatch = new ThreadLocal<>();
072    private MappedByteBuffer mappedBuffer;
073    private long mappingOffset;
074
075    protected MemoryMappedFileManager(final RandomAccessFile file, final String fileName, final OutputStream os,
076            final boolean force, final long position, final int regionLength, final String advertiseURI,
077            final Layout<? extends Serializable> layout, final boolean writeHeader) throws IOException {
078        super(os, fileName, layout, writeHeader, ByteBuffer.wrap(new byte[0]));
079        this.isForce = force;
080        this.randomAccessFile = Objects.requireNonNull(file, "RandomAccessFile");
081        this.regionLength = regionLength;
082        this.advertiseURI = advertiseURI;
083        this.isEndOfBatch.set(Boolean.FALSE);
084        this.mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), position, regionLength);
085        this.byteBuffer = mappedBuffer;
086        this.mappingOffset = position;
087    }
088
089    /**
090     * Returns the MemoryMappedFileManager.
091     *
092     * @param fileName The name of the file to manage.
093     * @param append true if the file should be appended to, false if it should be overwritten.
094     * @param isForce true if the contents should be flushed to disk on every write
095     * @param regionLength The mapped region length.
096     * @param advertiseURI the URI to use when advertising the file
097     * @param layout The layout.
098     * @return A MemoryMappedFileManager for the File.
099     */
100    public static MemoryMappedFileManager getFileManager(final String fileName, final boolean append,
101            final boolean isForce, final int regionLength, final String advertiseURI,
102            final Layout<? extends Serializable> layout) {
103        return (MemoryMappedFileManager) getManager(fileName, new FactoryData(append, isForce, regionLength,
104                advertiseURI, layout), FACTORY);
105    }
106
107    public Boolean isEndOfBatch() {
108        return isEndOfBatch.get();
109    }
110
111    public void setEndOfBatch(final boolean endOfBatch) {
112        this.isEndOfBatch.set(Boolean.valueOf(endOfBatch));
113    }
114
115    @Override
116    protected synchronized void write(final byte[] bytes, int offset, int length, final boolean immediateFlush) {
117        while (length > mappedBuffer.remaining()) {
118            final int chunk = mappedBuffer.remaining();
119            mappedBuffer.put(bytes, offset, chunk);
120            offset += chunk;
121            length -= chunk;
122            remap();
123        }
124        mappedBuffer.put(bytes, offset, length);
125
126        // no need to call flush() if force is true,
127        // already done in AbstractOutputStreamAppender.append
128    }
129
130    private synchronized void remap() {
131        final long offset = this.mappingOffset + mappedBuffer.position();
132        final int length = mappedBuffer.remaining() + regionLength;
133        try {
134            unsafeUnmap(mappedBuffer);
135            final long fileLength = randomAccessFile.length() + regionLength;
136            LOGGER.debug("{} {} extending {} by {} bytes to {}", getClass().getSimpleName(), getName(), getFileName(),
137                    regionLength, fileLength);
138
139            final long startNanos = System.nanoTime();
140            randomAccessFile.setLength(fileLength);
141            final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
142            LOGGER.debug("{} {} extended {} OK in {} millis", getClass().getSimpleName(), getName(), getFileName(),
143                    millis);
144
145            mappedBuffer = mmap(randomAccessFile.getChannel(), getFileName(), offset, length);
146            this.byteBuffer = mappedBuffer;
147            mappingOffset = offset;
148        } catch (final Exception ex) {
149            logError("unable to remap", ex);
150        }
151    }
152
153    @Override
154    public synchronized void flush() {
155        mappedBuffer.force();
156    }
157
158    @Override
159    public synchronized void close() {
160        final long position = mappedBuffer.position();
161        final long length = mappingOffset + position;
162        try {
163            unsafeUnmap(mappedBuffer);
164        } catch (final Exception ex) {
165            logError("unable to unmap MappedBuffer", ex);
166        }
167        try {
168            LOGGER.debug("MMapAppender closing. Setting {} length to {} (offset {} + position {})", getFileName(),
169                    length, mappingOffset, position);
170            randomAccessFile.setLength(length);
171            randomAccessFile.close();
172        } catch (final IOException ex) {
173            logError("unable to close MemoryMappedFile", ex);
174        }
175    }
176
177    public static MappedByteBuffer mmap(final FileChannel fileChannel, final String fileName, final long start,
178            final int size) throws IOException {
179        for (int i = 1;; i++) {
180            try {
181                LOGGER.debug("MMapAppender remapping {} start={}, size={}", fileName, start, size);
182
183                final long startNanos = System.nanoTime();
184                final MappedByteBuffer map = fileChannel.map(FileChannel.MapMode.READ_WRITE, start, size);
185                map.order(ByteOrder.nativeOrder());
186
187                final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
188                LOGGER.debug("MMapAppender remapped {} OK in {} millis", fileName, millis);
189
190                return map;
191            } catch (final IOException e) {
192                if (e.getMessage() == null || !e.getMessage().endsWith("user-mapped section open")) {
193                    throw e;
194                }
195                LOGGER.debug("Remap attempt {}/{} failed. Retrying...", i, MAX_REMAP_COUNT, e);
196                if (i < MAX_REMAP_COUNT) {
197                    Thread.yield();
198                } else {
199                    try {
200                        Thread.sleep(1);
201                    } catch (final InterruptedException ignored) {
202                        Thread.currentThread().interrupt();
203                        throw e;
204                    }
205                }
206            }
207        }
208    }
209
210    private static void unsafeUnmap(final MappedByteBuffer mbb) throws PrivilegedActionException {
211        LOGGER.debug("MMapAppender unmapping old buffer...");
212        final long startNanos = System.nanoTime();
213        AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {
214            @Override
215            public Object run() throws Exception {
216                final Method getCleanerMethod = mbb.getClass().getMethod("cleaner");
217                getCleanerMethod.setAccessible(true);
218                final Object cleaner = getCleanerMethod.invoke(mbb); // sun.misc.Cleaner instance
219                final Method cleanMethod = cleaner.getClass().getMethod("clean");
220                cleanMethod.invoke(cleaner);
221                return null;
222            }
223        });
224        final float millis = (float) ((System.nanoTime() - startNanos) / NANOS_PER_MILLISEC);
225        LOGGER.debug("MMapAppender unmapped buffer OK in {} millis", millis);
226    }
227
228    /**
229     * Returns the name of the File being managed.
230     *
231     * @return The name of the File being managed.
232     */
233    public String getFileName() {
234        return getName();
235    }
236
237    /**
238     * Returns the length of the memory mapped region.
239     *
240     * @return the length of the mapped region
241     */
242    public int getRegionLength() {
243        return regionLength;
244    }
245
246    /**
247     * Returns {@code true} if the content of the buffer should be forced to the storage device on every write,
248     * {@code false} otherwise.
249     *
250     * @return whether each write should be force-sync'ed
251     */
252    public boolean isImmediateFlush() {
253        return isForce;
254    }
255
256    /**
257     * Gets this FileManager's content format specified by:
258     * <p>
259     * Key: "fileURI" Value: provided "advertiseURI" param.
260     * </p>
261     *
262     * @return Map of content format keys supporting FileManager
263     */
264    @Override
265    public Map<String, String> getContentFormat() {
266        final Map<String, String> result = new HashMap<>(super.getContentFormat());
267        result.put("fileURI", advertiseURI);
268        return result;
269    }
270
271    @Override
272    protected void flushBuffer(final ByteBuffer buffer) {
273        // do nothing (do not call drain() to avoid spurious remapping)
274    }
275
276    @Override
277    public ByteBuffer getByteBuffer() {
278        return mappedBuffer;
279    }
280
281    @Override
282    public ByteBuffer drain(final ByteBuffer buf) {
283        remap();
284        return mappedBuffer;
285    }
286
287    /**
288     * Factory Data.
289     */
290    private static class FactoryData {
291        private final boolean append;
292        private final boolean force;
293        private final int regionLength;
294        private final String advertiseURI;
295        private final Layout<? extends Serializable> layout;
296
297        /**
298         * Constructor.
299         *
300         * @param append Append to existing file or truncate.
301         * @param force forces the memory content to be written to the storage device on every event
302         * @param regionLength length of the mapped region
303         */
304        public FactoryData(final boolean append, final boolean force, final int regionLength,
305                final String advertiseURI, final Layout<? extends Serializable> layout) {
306            this.append = append;
307            this.force = force;
308            this.regionLength = regionLength;
309            this.advertiseURI = advertiseURI;
310            this.layout = layout;
311        }
312    }
313
314    /**
315     * Factory to create a MemoryMappedFileManager.
316     */
317    private static class MemoryMappedFileManagerFactory
318            implements ManagerFactory<MemoryMappedFileManager, FactoryData> {
319
320        /**
321         * Create a MemoryMappedFileManager.
322         *
323         * @param name The name of the File.
324         * @param data The FactoryData
325         * @return The MemoryMappedFileManager for the File.
326         */
327        @SuppressWarnings("resource")
328        @Override
329        public MemoryMappedFileManager createManager(final String name, final FactoryData data) {
330            final File file = new File(name);
331            final File parent = file.getParentFile();
332            if (null != parent && !parent.exists()) {
333                parent.mkdirs();
334            }
335            if (!data.append) {
336                file.delete();
337            }
338
339            final boolean writeHeader = !data.append || !file.exists();
340            final OutputStream os = NullOutputStream.NULL_OUTPUT_STREAM;
341            RandomAccessFile raf = null;
342            try {
343                raf = new RandomAccessFile(name, "rw");
344                final long position = (data.append) ? raf.length() : 0;
345                raf.setLength(position + data.regionLength);
346                return new MemoryMappedFileManager(raf, name, os, data.force, position, data.regionLength,
347                        data.advertiseURI, data.layout, writeHeader);
348            } catch (final Exception ex) {
349                LOGGER.error("MemoryMappedFileManager (" + name + ") " + ex, ex);
350                Closer.closeSilently(raf);
351            }
352            return null;
353        }
354    }
355}