View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.DataInput;
23  import java.io.DataOutput;
24  import java.io.IOException;
25  import java.util.List;
26  import java.util.UUID;
27  import java.util.regex.Pattern;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.classification.InterfaceAudience;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FSDataInputStream;
34  import org.apache.hadoop.fs.FileSystem;
35  import org.apache.hadoop.fs.Path;
36  import org.apache.hadoop.hbase.HRegionInfo;
37  import org.apache.hadoop.hbase.HTableDescriptor;
38  import org.apache.hadoop.hbase.TableName;
39  import org.apache.hadoop.hbase.protobuf.generated.WALProtos.WALTrailer;
40  import org.apache.hadoop.io.Writable;
41  
42  
43  @InterfaceAudience.Private
44  // TODO: Rename interface to WAL
45  public interface HLog {
46    Log LOG = LogFactory.getLog(HLog.class);
47  
48    /** File Extension used while splitting an HLog into regions (HBASE-2312) */
49    // TODO: this seems like an implementation detail that does not belong here.
50    String SPLITTING_EXT = "-splitting";
51    boolean SPLIT_SKIP_ERRORS_DEFAULT = false;
52    /** The hbase:meta region's HLog filename extension */
53    String META_HLOG_FILE_EXTN = ".meta";
54  
55    /**
56     * Configuration name of HLog Trailer's warning size. If a waltrailer's size is greater than the
57     * configured size, a warning is logged. This is used with Protobuf reader/writer.
58     */
59    // TODO: Implementation detail.  Why in here?
60    String WAL_TRAILER_WARN_SIZE = "hbase.regionserver.waltrailer.warn.size";
61    int DEFAULT_WAL_TRAILER_WARN_SIZE = 1024 * 1024; // 1MB
62  
63    // TODO: Implemenation detail.  Why in here?
64    Pattern EDITFILES_NAME_PATTERN = Pattern.compile("-?[0-9]+");
65    String RECOVERED_LOG_TMPFILE_SUFFIX = ".temp";
66  
67    interface Reader {
68  
69      /**
70       * @param fs File system.
71       * @param path Path.
72       * @param c Configuration.
73       * @param s Input stream that may have been pre-opened by the caller; may be null.
74       */
75      void init(FileSystem fs, Path path, Configuration c, FSDataInputStream s) throws IOException;
76  
77      void close() throws IOException;
78  
79      Entry next() throws IOException;
80  
81      Entry next(Entry reuse) throws IOException;
82  
83      void seek(long pos) throws IOException;
84  
85      long getPosition() throws IOException;
86      void reset() throws IOException;
87  
88      /**
89       * @return the WALTrailer of the current HLog. It may be null in case of legacy or corrupt WAL
90       *         files.
91       */
92      // TODO: What we need a trailer on WAL for?
93      WALTrailer getWALTrailer();
94    }
95  
96    interface Writer {
97      void init(FileSystem fs, Path path, Configuration c, boolean overwritable) throws IOException;
98  
99      void close() throws IOException;
100 
101     void sync() throws IOException;
102 
103     void append(Entry entry) throws IOException;
104 
105     long getLength() throws IOException;
106 
107     /**
108      * Sets HLog's WALTrailer. This trailer is appended at the end of WAL on closing.
109      * @param walTrailer trailer to append to WAL.
110      */
111     void setWALTrailer(WALTrailer walTrailer);
112   }
113 
114   /**
115    * Utility class that lets us keep track of the edit with it's key.
116    * Only used when splitting logs.
117    */
118   // TODO: Remove this Writable.
119   class Entry implements Writable {
120     private WALEdit edit;
121     private HLogKey key;
122 
123     public Entry() {
124       edit = new WALEdit();
125       key = new HLogKey();
126     }
127 
128     /**
129      * Constructor for both params
130      *
131      * @param edit log's edit
132      * @param key log's key
133      */
134     public Entry(HLogKey key, WALEdit edit) {
135       super();
136       this.key = key;
137       this.edit = edit;
138     }
139 
140     /**
141      * Gets the edit
142      *
143      * @return edit
144      */
145     public WALEdit getEdit() {
146       return edit;
147     }
148 
149     /**
150      * Gets the key
151      *
152      * @return key
153      */
154     public HLogKey getKey() {
155       return key;
156     }
157 
158     /**
159      * Set compression context for this entry.
160      *
161      * @param compressionContext
162      *          Compression context
163      */
164     public void setCompressionContext(CompressionContext compressionContext) {
165       edit.setCompressionContext(compressionContext);
166       key.setCompressionContext(compressionContext);
167     }
168 
169     @Override
170     public String toString() {
171       return this.key + "=" + this.edit;
172     }
173 
174     @Override
175     @SuppressWarnings("deprecation")
176     public void write(DataOutput dataOutput) throws IOException {
177       this.key.write(dataOutput);
178       this.edit.write(dataOutput);
179     }
180 
181     @Override
182     public void readFields(DataInput dataInput) throws IOException {
183       this.key.readFields(dataInput);
184       this.edit.readFields(dataInput);
185     }
186   }
187 
188   /**
189    * registers WALActionsListener
190    *
191    * @param listener
192    */
193   void registerWALActionsListener(final WALActionsListener listener);
194 
195   /**
196    * unregisters WALActionsListener
197    *
198    * @param listener
199    */
200   boolean unregisterWALActionsListener(final WALActionsListener listener);
201 
202   /**
203    * @return Current state of the monotonically increasing file id.
204    */
205   // TODO: Remove.  Implementation detail.
206   long getFilenum();
207 
208   /**
209    * Called to ensure that log sequence numbers are always greater
210    *
211    * @param newvalue We'll set log edit/sequence number to this value if it is greater
212    * than the current value.
213    */
214   void setSequenceNumber(final long newvalue);
215 
216   /**
217    * @return log sequence number
218    */
219   long getSequenceNumber();
220 
221   /**
222    * @return the number of HLog files
223    */
224   int getNumLogFiles();
225   
226   /**
227    * @return the size of HLog files
228    */
229   long getLogFileSize();
230   
231   // TODO: Log rolling should not be in this interface.
232   /**
233    * Roll the log writer. That is, start writing log messages to a new file.
234    *
235    * <p>
236    * The implementation is synchronized in order to make sure there's one rollWriter
237    * running at any given time.
238    *
239    * @return If lots of logs, flush the returned regions so next time through we
240    *         can clean logs. Returns null if nothing to flush. Names are actual
241    *         region names as returned by {@link HRegionInfo#getEncodedName()}
242    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
243    * @throws IOException
244    */
245   byte[][] rollWriter() throws FailedLogCloseException, IOException;
246 
247   /**
248    * Roll the log writer. That is, start writing log messages to a new file.
249    *
250    * <p>
251    * The implementation is synchronized in order to make sure there's one rollWriter
252    * running at any given time.
253    *
254    * @param force
255    *          If true, force creation of a new writer even if no entries have
256    *          been written to the current writer
257    * @return If lots of logs, flush the returned regions so next time through we
258    *         can clean logs. Returns null if nothing to flush. Names are actual
259    *         region names as returned by {@link HRegionInfo#getEncodedName()}
260    * @throws org.apache.hadoop.hbase.regionserver.wal.FailedLogCloseException
261    * @throws IOException
262    */
263   byte[][] rollWriter(boolean force) throws FailedLogCloseException,
264       IOException;
265 
266   /**
267    * Shut down the log.
268    *
269    * @throws IOException
270    */
271   void close() throws IOException;
272 
273   /**
274    * Shut down the log and delete the log directory
275    *
276    * @throws IOException
277    */
278   void closeAndDelete() throws IOException;
279 
280   /**
281    * Same as appendNoSync(HRegionInfo, TableName, WALEdit, List, long, HTableDescriptor),
282    * except it causes a sync on the log
283    */
284   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
285       final long now, HTableDescriptor htd) throws IOException;
286 
287   /**
288    * Append a set of edits to the log. Log edits are keyed by (encoded)
289    * regionName, row name, and log-sequence-id. The HLog is flushed after this
290    * transaction is written to the log.
291    * @param info
292    * @param tableName
293    * @param edits
294    * @param now
295    * @param htd
296    * @param isInMemstore Whether the record is in memstore. False for system records.
297    */
298   public void append(HRegionInfo info, TableName tableName, WALEdit edits,
299       final long now, HTableDescriptor htd, boolean isInMemstore) throws IOException;
300 
301   /**
302    * For notification post append to the writer.
303    * @param entries
304    */
305   void postAppend(final List<Entry> entries);
306 
307   /**
308    * For notification post writer sync.
309    */
310   void postSync();
311 
312   /**
313    * Append a set of edits to the log. Log edits are keyed by (encoded) regionName, rowname, and
314    * log-sequence-id. The HLog is not flushed after this transaction is written to the log.
315    * @param info
316    * @param tableName
317    * @param edits
318    * @param clusterIds The clusters that have consumed the change (for replication)
319    * @param now
320    * @param htd
321    * @return txid of this transaction
322    * @throws IOException
323    */
324   public long appendNoSync(HRegionInfo info, TableName tableName, WALEdit edits,
325       List<UUID> clusterIds, final long now, HTableDescriptor htd) throws IOException;
326 
327   // TODO: Do we need all these versions of sync?
328   void hsync() throws IOException;
329 
330   void hflush() throws IOException;
331 
332   void sync() throws IOException;
333 
334   void sync(long txid) throws IOException;
335 
336   /**
337    * Obtain a log sequence number.
338    */
339   // TODO: Name better to differentiate from getSequenceNumber.
340   long obtainSeqNum();
341 
342   /**
343    * WAL keeps track of the sequence numbers that were not yet flushed from memstores
344    * in order to be able to do cleanup. This method tells WAL that some region is about
345    * to flush memstore.
346    *
347    * We stash the oldest seqNum for the region, and let the the next edit inserted in this
348    * region be recorded in {@link #append(HRegionInfo, TableName, WALEdit, long, HTableDescriptor)}
349    * as new oldest seqnum. In case of flush being aborted, we put the stashed value back;
350    * in case of flush succeeding, the seqNum of that first edit after start becomes the
351    * valid oldest seqNum for this region.
352    *
353    * @return current seqNum, to pass on to flushers (who will put it into the metadata of
354    *         the resulting file as an upper-bound seqNum for that file), or NULL if flush
355    *         should not be started.
356    */
357   Long startCacheFlush(final byte[] encodedRegionName);
358 
359   /**
360    * Complete the cache flush.
361    * @param encodedRegionName Encoded region name.
362    */
363   void completeCacheFlush(final byte[] encodedRegionName);
364 
365   /**
366    * Abort a cache flush. Call if the flush fails. Note that the only recovery
367    * for an aborted flush currently is a restart of the regionserver so the
368    * snapshot content dropped by the failure gets restored to the memstore.v
369    * @param encodedRegionName Encoded region name.
370    */
371   void abortCacheFlush(byte[] encodedRegionName);
372 
373   /**
374    * @return Coprocessor host.
375    */
376   WALCoprocessorHost getCoprocessorHost();
377 
378   /**
379    * Get LowReplication-Roller status
380    *
381    * @return lowReplicationRollEnabled
382    */
383   // TODO: This is implementation detail?
384   boolean isLowReplicationRollEnabled();
385 
386   /** Gets the earliest sequence number in the memstore for this particular region.
387    * This can serve as best-effort "recent" WAL number for this region.
388    * @param encodedRegionName The region to get the number for.
389    * @return The number if present, HConstants.NO_SEQNUM if absent.
390    */
391   long getEarliestMemstoreSeqNum(byte[] encodedRegionName);
392 }