View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  
21  package org.apache.hadoop.hbase.wal;
22  
23  import java.io.IOException;
24  import java.util.Arrays;
25  import java.io.InterruptedIOException;
26  import java.util.Collections;
27  import java.util.List;
28  import java.util.concurrent.atomic.AtomicReference;
29  
30  import com.google.common.annotations.VisibleForTesting;
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.hbase.classification.InterfaceAudience;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.fs.FSDataInputStream;
36  import org.apache.hadoop.fs.FileSystem;
37  import org.apache.hadoop.fs.Path;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.wal.WAL.Reader;
40  import org.apache.hadoop.hbase.wal.WALProvider.Writer;
41  import org.apache.hadoop.hbase.util.CancelableProgressable;
42  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
43  
44  // imports for things that haven't moved from regionserver.wal yet.
45  import org.apache.hadoop.hbase.regionserver.wal.MetricsWAL;
46  import org.apache.hadoop.hbase.regionserver.wal.ProtobufLogReader;
47  import org.apache.hadoop.hbase.regionserver.wal.SequenceFileLogReader;
48  import org.apache.hadoop.hbase.regionserver.wal.WALActionsListener;
49  
50  /**
51   * Entry point for users of the Write Ahead Log.
52   * Acts as the shim between internal use and the particular WALProvider we use to handle wal
53   * requests.
54   *
55   * Configure which provider gets used with the configuration setting "hbase.wal.provider". Available
56   * implementations:
57   * <ul>
58   *   <li><em>defaultProvider</em> : whatever provider is standard for the hbase version. Currently
59   *                                  "filesystem"</li>
60   *   <li><em>filesystem</em> : a provider that will run on top of an implementation of the Hadoop
61   *                             FileSystem interface, normally HDFS.</li>
62   *   <li><em>multiwal</em> : a provider that will use multiple "filesystem" wal instances per region
63   *                           server.</li>
64   * </ul>
65   *
66   * Alternatively, you may provide a custome implementation of {@link WALProvider} by class name.
67   */
68  @InterfaceAudience.Private
69  public class WALFactory {
70  
71    private static final Log LOG = LogFactory.getLog(WALFactory.class);
72  
73    /**
74     * Maps between configuration names for providers and implementation classes.
75     */
76    static enum Providers {
77      defaultProvider(DefaultWALProvider.class),
78      filesystem(DefaultWALProvider.class),
79      multiwal(BoundedRegionGroupingProvider.class);
80  
81      Class<? extends WALProvider> clazz;
82      Providers(Class<? extends WALProvider> clazz) {
83        this.clazz = clazz;
84      }
85    }
86  
87    static final String WAL_PROVIDER = "hbase.wal.provider";
88    static final String DEFAULT_WAL_PROVIDER = Providers.defaultProvider.name();
89  
90    static final String META_WAL_PROVIDER = "hbase.wal.meta_provider";
91    static final String DEFAULT_META_WAL_PROVIDER = Providers.defaultProvider.name();
92  
93    final String factoryId;
94    final WALProvider provider;
95    // The meta updates are written to a different wal. If this
96    // regionserver holds meta regions, then this ref will be non-null.
97    // lazily intialized; most RegionServers don't deal with META
98    final AtomicReference<WALProvider> metaProvider = new AtomicReference<WALProvider>();
99  
100   /**
101    * Configuration-specified WAL Reader used when a custom reader is requested
102    */
103   private final Class<? extends DefaultWALProvider.Reader> logReaderClass;
104 
105   /**
106    * How long to attempt opening in-recovery wals
107    */
108   private final int timeoutMillis;
109 
110   private final Configuration conf;
111 
112   // Used for the singleton WALFactory, see below.
113   private WALFactory(Configuration conf) {
114     // this code is duplicated here so we can keep our members final.
115     // until we've moved reader/writer construction down into providers, this initialization must
116     // happen prior to provider initialization, in case they need to instantiate a reader/writer.
117     timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
118     /* TODO Both of these are probably specific to the fs wal provider */
119     logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
120         DefaultWALProvider.Reader.class);
121     this.conf = conf;
122     // end required early initialization
123 
124     // this instance can't create wals, just reader/writers.
125     provider = null;
126     factoryId = SINGLETON_ID;
127   }
128 
129   /**
130    * instantiate a provider from a config property.
131    * requires conf to have already been set (as well as anything the provider might need to read).
132    */
133   WALProvider getProvider(final String key, final String defaultValue,
134       final List<WALActionsListener> listeners, final String providerId) throws IOException {
135     Class<? extends WALProvider> clazz;
136     try {
137       clazz = Providers.valueOf(conf.get(key, defaultValue)).clazz;
138     } catch (IllegalArgumentException exception) {
139       // Fall back to them specifying a class name
140       // Note that the passed default class shouldn't actually be used, since the above only fails
141       // when there is a config value present.
142       clazz = conf.getClass(key, DefaultWALProvider.class, WALProvider.class);
143     }
144     LOG.info("Instantiating WALProvider of type " + clazz);
145     try {
146       final WALProvider result = clazz.newInstance();
147       result.init(this, conf, listeners, providerId);
148       return result;
149     } catch (InstantiationException exception) {
150       LOG.error("couldn't set up WALProvider, check config key " + key);
151       LOG.debug("Exception details for failure to load WALProvider.", exception);
152       throw new IOException("couldn't set up WALProvider", exception);
153     } catch (IllegalAccessException exception) {
154       LOG.error("couldn't set up WALProvider, check config key " + key);
155       LOG.debug("Exception details for failure to load WALProvider.", exception);
156       throw new IOException("couldn't set up WALProvider", exception);
157     }
158   }
159 
160   /**
161    * @param conf must not be null, will keep a reference to read params in later reader/writer
162    *     instances.
163    * @param listeners may be null. will be given to all created wals (and not meta-wals)
164    * @param factoryId a unique identifier for this factory. used i.e. by filesystem implementations
165    *     to make a directory
166    */
167   public WALFactory(final Configuration conf, final List<WALActionsListener> listeners,
168       final String factoryId) throws IOException {
169     // until we've moved reader/writer construction down into providers, this initialization must
170     // happen prior to provider initialization, in case they need to instantiate a reader/writer.
171     timeoutMillis = conf.getInt("hbase.hlog.open.timeout", 300000);
172     /* TODO Both of these are probably specific to the fs wal provider */
173     logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl", ProtobufLogReader.class,
174         DefaultWALProvider.Reader.class);
175     this.conf = conf;
176     this.factoryId = factoryId;
177     // end required early initialization
178     if (conf.getBoolean("hbase.regionserver.hlog.enabled", true)) {
179       provider = getProvider(WAL_PROVIDER, DEFAULT_WAL_PROVIDER, listeners, null);
180     } else {
181       // special handling of existing configuration behavior.
182       LOG.warn("Running with WAL disabled.");
183       provider = new DisabledWALProvider();
184       provider.init(this, conf, null, factoryId);
185     }
186   }
187 
188   /**
189    * Shutdown all WALs and clean up any underlying storage.
190    * Use only when you will not need to replay and edits that have gone to any wals from this
191    * factory.
192    */
193   public void close() throws IOException {
194     final WALProvider metaProvider = this.metaProvider.get();
195     if (null != metaProvider) {
196       metaProvider.close();
197     }
198     // close is called on a WALFactory with null provider in the case of contention handling
199     // within the getInstance method.
200     if (null != provider) {
201       provider.close();
202     }
203   }
204 
205   /**
206    * Tell the underlying WAL providers to shut down, but do not clean up underlying storage.
207    * If you are not ending cleanly and will need to replay edits from this factory's wals,
208    * use this method if you can as it will try to leave things as tidy as possible.
209    */
210   public void shutdown() throws IOException {
211     IOException exception = null;
212     final WALProvider metaProvider = this.metaProvider.get();
213     if (null != metaProvider) {
214       try {
215         metaProvider.shutdown();
216       } catch(IOException ioe) {
217         exception = ioe;
218       }
219     }
220     provider.shutdown();
221     if (null != exception) {
222       throw exception;
223     }
224   }
225 
226   /**
227    * @param identifier may not be null, contents will not be altered
228    */
229   public WAL getWAL(final byte[] identifier) throws IOException {
230     return provider.getWAL(identifier);
231   }
232 
233   /**
234    * @param identifier may not be null, contents will not be altered
235    */
236   public WAL getMetaWAL(final byte[] identifier) throws IOException {
237     WALProvider metaProvider = this.metaProvider.get();
238     if (null == metaProvider) {
239       final WALProvider temp = getProvider(META_WAL_PROVIDER, DEFAULT_META_WAL_PROVIDER,
240           Collections.<WALActionsListener>singletonList(new MetricsWAL()),
241           DefaultWALProvider.META_WAL_PROVIDER_ID);
242       if (this.metaProvider.compareAndSet(null, temp)) {
243         metaProvider = temp;
244       } else {
245         // reference must now be to a provider created in another thread.
246         temp.close();
247         metaProvider = this.metaProvider.get();
248       }
249     }
250     return metaProvider.getWAL(identifier);
251   }
252 
253   public Reader createReader(final FileSystem fs, final Path path) throws IOException {
254     return createReader(fs, path, (CancelableProgressable)null);
255   }
256 
257   /**
258    * Create a reader for the WAL. If you are reading from a file that's being written to and need
259    * to reopen it multiple times, use {@link WAL.Reader#reset()} instead of this method
260    * then just seek back to the last known good position.
261    * @return A WAL reader.  Close when done with it.
262    * @throws IOException
263    */
264   public Reader createReader(final FileSystem fs, final Path path,
265       CancelableProgressable reporter) throws IOException {
266     return createReader(fs, path, reporter, true);
267   }
268 
269   public Reader createReader(final FileSystem fs, final Path path,
270       CancelableProgressable reporter, boolean allowCustom)
271       throws IOException {
272     Class<? extends DefaultWALProvider.Reader> lrClass =
273         allowCustom ? logReaderClass : ProtobufLogReader.class;
274 
275     try {
276       // A wal file could be under recovery, so it may take several
277       // tries to get it open. Instead of claiming it is corrupted, retry
278       // to open it up to 5 minutes by default.
279       long startWaiting = EnvironmentEdgeManager.currentTime();
280       long openTimeout = timeoutMillis + startWaiting;
281       int nbAttempt = 0;
282       while (true) {
283         try {
284           if (lrClass != ProtobufLogReader.class) {
285             // User is overriding the WAL reader, let them.
286             DefaultWALProvider.Reader reader = lrClass.newInstance();
287             reader.init(fs, path, conf, null);
288             return reader;
289           } else {
290             FSDataInputStream stream = fs.open(path);
291             // Note that zero-length file will fail to read PB magic, and attempt to create
292             // a non-PB reader and fail the same way existing code expects it to. If we get
293             // rid of the old reader entirely, we need to handle 0-size files differently from
294             // merely non-PB files.
295             byte[] magic = new byte[ProtobufLogReader.PB_WAL_MAGIC.length];
296             boolean isPbWal = (stream.read(magic) == magic.length)
297                 && Arrays.equals(magic, ProtobufLogReader.PB_WAL_MAGIC);
298             DefaultWALProvider.Reader reader =
299                 isPbWal ? new ProtobufLogReader() : new SequenceFileLogReader();
300             reader.init(fs, path, conf, stream);
301             return reader;
302           }
303         } catch (IOException e) {
304           String msg = e.getMessage();
305           if (msg != null && (msg.contains("Cannot obtain block length")
306               || msg.contains("Could not obtain the last block")
307               || msg.matches("Blocklist for [^ ]* has changed.*"))) {
308             if (++nbAttempt == 1) {
309               LOG.warn("Lease should have recovered. This is not expected. Will retry", e);
310             }
311             if (reporter != null && !reporter.progress()) {
312               throw new InterruptedIOException("Operation is cancelled");
313             }
314             if (nbAttempt > 2 && openTimeout < EnvironmentEdgeManager.currentTime()) {
315               LOG.error("Can't open after " + nbAttempt + " attempts and "
316                 + (EnvironmentEdgeManager.currentTime() - startWaiting)
317                 + "ms " + " for " + path);
318             } else {
319               try {
320                 Thread.sleep(nbAttempt < 3 ? 500 : 1000);
321                 continue; // retry
322               } catch (InterruptedException ie) {
323                 InterruptedIOException iioe = new InterruptedIOException();
324                 iioe.initCause(ie);
325                 throw iioe;
326               }
327             }
328           }
329           throw e;
330         }
331       }
332     } catch (IOException ie) {
333       throw ie;
334     } catch (Exception e) {
335       throw new IOException("Cannot get log reader", e);
336     }
337   }
338 
339   /**
340    * Create a writer for the WAL.
341    * should be package-private. public only for tests and
342    * {@link org.apache.hadoop.hbase.regionserver.wal.Compressor}
343    * @return A WAL writer.  Close when done with it.
344    * @throws IOException
345    */
346   public Writer createWALWriter(final FileSystem fs, final Path path) throws IOException {
347     return DefaultWALProvider.createWriter(conf, fs, path, false);
348   }
349 
350   /**
351    * should be package-private, visible for recovery testing.
352    * @return an overwritable writer for recovered edits. caller should close.
353    */
354   @VisibleForTesting
355   public Writer createRecoveredEditsWriter(final FileSystem fs, final Path path)
356       throws IOException {
357     return DefaultWALProvider.createWriter(conf, fs, path, true);
358   }
359 
360   // These static methods are currently used where it's impractical to
361   // untangle the reliance on state in the filesystem. They rely on singleton
362   // WALFactory that just provides Reader / Writers.
363   // For now, first Configuration object wins. Practically this just impacts the reader/writer class
364   private static final AtomicReference<WALFactory> singleton = new AtomicReference<WALFactory>();
365   private static final String SINGLETON_ID = WALFactory.class.getName();
366   
367   // public only for FSHLog and UpgradeTo96
368   public static WALFactory getInstance(Configuration configuration) {
369     WALFactory factory = singleton.get();
370     if (null == factory) {
371       WALFactory temp = new WALFactory(configuration);
372       if (singleton.compareAndSet(null, temp)) {
373         factory = temp;
374       } else {
375         // someone else beat us to initializing
376         try {
377           temp.close();
378         } catch (IOException exception) {
379           LOG.debug("failed to close temporary singleton. ignoring.", exception);
380         }
381         factory = singleton.get();
382       }
383     }
384     return factory;
385   }
386 
387   /**
388    * Create a reader for the given path, accept custom reader classes from conf.
389    * If you already have a WALFactory, you should favor the instance method.
390    * @return a WAL Reader, caller must close.
391    */
392   public static Reader createReader(final FileSystem fs, final Path path,
393       final Configuration configuration) throws IOException {
394     return getInstance(configuration).createReader(fs, path);
395   }
396 
397   /**
398    * Create a reader for the given path, accept custom reader classes from conf.
399    * If you already have a WALFactory, you should favor the instance method.
400    * @return a WAL Reader, caller must close.
401    */
402   static Reader createReader(final FileSystem fs, final Path path,
403       final Configuration configuration, final CancelableProgressable reporter) throws IOException {
404     return getInstance(configuration).createReader(fs, path, reporter);
405   }
406 
407   /**
408    * Create a reader for the given path, ignore custom reader classes from conf.
409    * If you already have a WALFactory, you should favor the instance method.
410    * only public pending move of {@link org.apache.hadoop.hbase.regionserver.wal.Compressor}
411    * @return a WAL Reader, caller must close.
412    */
413   public static Reader createReaderIgnoreCustomClass(final FileSystem fs, final Path path,
414       final Configuration configuration) throws IOException {
415     return getInstance(configuration).createReader(fs, path, null, false);
416   }
417 
418   /**
419    * If you already have a WALFactory, you should favor the instance method.
420    * @return a Writer that will overwrite files. Caller must close.
421    */
422   static Writer createRecoveredEditsWriter(final FileSystem fs, final Path path,
423       final Configuration configuration)
424       throws IOException {
425     return DefaultWALProvider.createWriter(configuration, fs, path, true);
426   }
427 
428   /**
429    * If you already have a WALFactory, you should favor the instance method.
430    * @return a writer that won't overwrite files. Caller must close.
431    */
432   @VisibleForTesting
433   public static Writer createWALWriter(final FileSystem fs, final Path path,
434       final Configuration configuration)
435       throws IOException {
436     return DefaultWALProvider.createWriter(configuration, fs, path, false);
437   }
438 }