View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  
21  package org.apache.hadoop.hbase.regionserver.wal;
22  
23  import java.io.IOException;
24  import java.util.Arrays;
25  import java.io.InterruptedIOException;
26  import java.util.List;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.classification.InterfaceAudience;
31  import org.apache.hadoop.conf.Configuration;
32  import org.apache.hadoop.fs.FSDataInputStream;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HConstants;
36  import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
37  import org.apache.hadoop.hbase.regionserver.wal.HLog.Writer;
38  import org.apache.hadoop.hbase.util.CancelableProgressable;
39  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
40  import org.apache.hadoop.hbase.util.LeaseNotRecoveredException;
41  
42  @InterfaceAudience.Private
43  public class HLogFactory {
44      private static final Log LOG = LogFactory.getLog(HLogFactory.class);
45  
46      public static HLog createHLog(final FileSystem fs, final Path root, final String logName,
47          final Configuration conf) throws IOException {
48        return new FSHLog(fs, root, logName, conf);
49      }
50  
51      public static HLog createHLog(final FileSystem fs, final Path root, final String logName,
52          final String oldLogName, final Configuration conf) throws IOException {
53        return new FSHLog(fs, root, logName, oldLogName, conf);
54  }
55  
56      public static HLog createHLog(final FileSystem fs, final Path root, final String logName,
57          final Configuration conf, final List<WALActionsListener> listeners,
58          final String prefix) throws IOException {
59        return new FSHLog(fs, root, logName, conf, listeners, prefix);
60      }
61  
62      public static HLog createMetaHLog(final FileSystem fs, final Path root, final String logName,
63          final Configuration conf, final List<WALActionsListener> listeners,
64          final String prefix) throws IOException {
65        return new FSHLog(fs, root, logName, HConstants.HREGION_OLDLOGDIR_NAME,
66              conf, listeners, false, prefix, true);
67      }
68  
69      /*
70       * WAL Reader
71       */
72      private static Class<? extends Reader> logReaderClass;
73  
74      static void resetLogReaderClass() {
75        logReaderClass = null;
76      }
77  
78      public static HLog.Reader createReader(final FileSystem fs,
79          final Path path, Configuration conf) throws IOException {
80        return createReader(fs, path, conf, null);
81      }
82  
83      /**
84       * Create a reader for the WAL. If you are reading from a file that's being written to
85       * and need to reopen it multiple times, use {@link HLog.Reader#reset()} instead of this method
86       * then just seek back to the last known good position.
87       * @return A WAL reader.  Close when done with it.
88       * @throws IOException
89       */
90      public static HLog.Reader createReader(final FileSystem fs, final Path path,
91          Configuration conf, CancelableProgressable reporter) throws IOException {
92        return createReader(fs, path, conf, reporter, true);
93      }
94  
95      public static HLog.Reader createReader(final FileSystem fs, final Path path,
96        Configuration conf, CancelableProgressable reporter, boolean allowCustom)
97          throws IOException {
98        if (allowCustom && (logReaderClass == null)) {
99          logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
100           ProtobufLogReader.class, Reader.class);
101       }
102       Class<? extends Reader> lrClass = allowCustom ? logReaderClass : ProtobufLogReader.class;
103 
104       try {
105         // A hlog file could be under recovery, so it may take several
106         // tries to get it open. Instead of claiming it is corrupted, retry
107         // to open it up to 5 minutes by default.
108         long startWaiting = EnvironmentEdgeManager.currentTimeMillis();
109         long openTimeout = conf.getInt("hbase.hlog.open.timeout", 300000) + startWaiting;
110         int nbAttempt = 0;
111         FSDataInputStream stream = null;
112         while (true) {
113           try {
114             if (lrClass != ProtobufLogReader.class) {
115               // User is overriding the WAL reader, let them.
116               HLog.Reader reader = lrClass.newInstance();
117               reader.init(fs, path, conf, null);
118               return reader;
119             } else {
120               stream = fs.open(path);
121               // Note that zero-length file will fail to read PB magic, and attempt to create
122               // a non-PB reader and fail the same way existing code expects it to. If we get
123               // rid of the old reader entirely, we need to handle 0-size files differently from
124               // merely non-PB files.
125               byte[] magic = new byte[ProtobufLogReader.PB_WAL_MAGIC.length];
126               boolean isPbWal = (stream.read(magic) == magic.length)
127                   && Arrays.equals(magic, ProtobufLogReader.PB_WAL_MAGIC);
128               HLog.Reader reader =
129                   isPbWal ? new ProtobufLogReader() : new SequenceFileLogReader();
130               reader.init(fs, path, conf, stream);
131               return reader;
132             }
133           } catch (IOException e) {
134             try {
135               if (stream != null) {
136                 stream.close();
137               }
138             } catch (IOException exception) {
139               LOG.warn("Could not close FSDataInputStream" + exception.getMessage());
140               LOG.debug("exception details", exception);
141             }
142             String msg = e.getMessage();
143             if (msg != null && (msg.contains("Cannot obtain block length")
144                 || msg.contains("Could not obtain the last block")
145                 || msg.matches("Blocklist for [^ ]* has changed.*"))) {
146               if (++nbAttempt == 1) {
147                 LOG.warn("Lease should have recovered. This is not expected. Will retry", e);
148               }
149               if (reporter != null && !reporter.progress()) {
150                 throw new InterruptedIOException("Operation is cancelled");
151               }
152               if (nbAttempt > 2 && openTimeout < EnvironmentEdgeManager.currentTimeMillis()) {
153                 LOG.error("Can't open after " + nbAttempt + " attempts and "
154                   + (EnvironmentEdgeManager.currentTimeMillis() - startWaiting)
155                   + "ms " + " for " + path);
156               } else {
157                 try {
158                   Thread.sleep(nbAttempt < 3 ? 500 : 1000);
159                   continue; // retry
160                 } catch (InterruptedException ie) {
161                   InterruptedIOException iioe = new InterruptedIOException();
162                   iioe.initCause(ie);
163                   throw iioe;
164                 }
165               }
166               throw new LeaseNotRecoveredException(e);
167             } else {
168               throw e;
169             }
170           }
171         }
172       } catch (IOException ie) {
173         throw ie;
174       } catch (Exception e) {
175         throw new IOException("Cannot get log reader", e);
176       }
177     }
178 
179     /*
180      * WAL writer
181      */
182     private static Class<? extends Writer> logWriterClass;
183 
184     static void resetLogWriterClass() {
185       logWriterClass = null;
186     }
187 
188     /**
189      * Create a writer for the WAL.
190      * @return A WAL writer.  Close when done with it.
191      * @throws IOException
192      */
193     public static HLog.Writer createWALWriter(final FileSystem fs,
194         final Path path, Configuration conf) throws IOException {
195       return createWriter(fs, path, conf, false);
196     }
197 
198     public static HLog.Writer createRecoveredEditsWriter(final FileSystem fs,
199         final Path path, Configuration conf) throws IOException {
200       return createWriter(fs, path, conf, true);
201     }
202 
203     private static HLog.Writer createWriter(final FileSystem fs,
204         final Path path, Configuration conf, boolean overwritable)
205     throws IOException {
206       try {
207         if (logWriterClass == null) {
208           logWriterClass = conf.getClass("hbase.regionserver.hlog.writer.impl",
209               ProtobufLogWriter.class, Writer.class);
210         }
211         HLog.Writer writer = (HLog.Writer)logWriterClass.newInstance();
212         writer.init(fs, path, conf, overwritable);
213         return writer;
214       } catch (Exception e) {
215         throw new IOException("cannot get log writer", e);
216       }
217     }
218 }