View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  
20  package org.apache.hadoop.hbase.regionserver.wal;
21  
22  import java.io.IOException;
23  import java.util.NavigableSet;
24  import java.util.TreeSet;
25  import java.util.regex.Matcher;
26  import java.util.regex.Pattern;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.conf.Configuration;
31  import org.apache.hadoop.fs.FileSystem;
32  import org.apache.hadoop.fs.FileStatus;
33  import org.apache.hadoop.fs.Path;
34  import org.apache.hadoop.fs.PathFilter;
35  import org.apache.hadoop.hbase.ServerName;
36  import org.apache.hadoop.hbase.util.FSUtils;
37  
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.util.Bytes;
40  
41  public class HLogUtil {
42    static final Log LOG = LogFactory.getLog(HLogUtil.class);
43  
44    /**
45     * @param family
46     * @return true if the column is a meta column
47     */
48    public static boolean isMetaFamily(byte[] family) {
49      return Bytes.equals(HLog.METAFAMILY, family);
50    }
51  
52    @SuppressWarnings("unchecked")
53    public static Class<? extends HLogKey> getKeyClass(Configuration conf) {
54      return (Class<? extends HLogKey>) conf.getClass(
55          "hbase.regionserver.hlog.keyclass", HLogKey.class);
56    }
57  
58    public static HLogKey newKey(Configuration conf) throws IOException {
59      Class<? extends HLogKey> keyClass = getKeyClass(conf);
60      try {
61        return keyClass.newInstance();
62      } catch (InstantiationException e) {
63        throw new IOException("cannot create hlog key");
64      } catch (IllegalAccessException e) {
65        throw new IOException("cannot create hlog key");
66      }
67    }
68  
69    /**
70     * Pattern used to validate a HLog file name
71     */
72    private static final Pattern pattern = 
73        Pattern.compile(".*\\.\\d*("+HLog.META_HLOG_FILE_EXTN+")*");
74  
75    /**
76     * @param filename
77     *          name of the file to validate
78     * @return <tt>true</tt> if the filename matches an HLog, <tt>false</tt>
79     *         otherwise
80     */
81    public static boolean validateHLogFilename(String filename) {
82      return pattern.matcher(filename).matches();
83    }
84  
85    /*
86     * Get a reader for the WAL.
87     * 
88     * @param fs
89     * 
90     * @param path
91     * 
92     * @param conf
93     * 
94     * @return A WAL reader. Close when done with it.
95     * 
96     * @throws IOException
97     * 
98     * public static HLog.Reader getReader(final FileSystem fs, final Path path,
99     * Configuration conf) throws IOException { try {
100    * 
101    * if (logReaderClass == null) {
102    * 
103    * logReaderClass = conf.getClass("hbase.regionserver.hlog.reader.impl",
104    * SequenceFileLogReader.class, Reader.class); }
105    * 
106    * 
107    * HLog.Reader reader = logReaderClass.newInstance(); reader.init(fs, path,
108    * conf); return reader; } catch (IOException e) { throw e; } catch (Exception
109    * e) { throw new IOException("Cannot get log reader", e); } }
110    * 
111    * * Get a writer for the WAL.
112    * 
113    * @param path
114    * 
115    * @param conf
116    * 
117    * @return A WAL writer. Close when done with it.
118    * 
119    * @throws IOException
120    * 
121    * public static HLog.Writer createWriter(final FileSystem fs, final Path
122    * path, Configuration conf) throws IOException { try { if (logWriterClass ==
123    * null) { logWriterClass =
124    * conf.getClass("hbase.regionserver.hlog.writer.impl",
125    * SequenceFileLogWriter.class, Writer.class); } FSHLog.Writer writer =
126    * (FSHLog.Writer) logWriterClass.newInstance(); writer.init(fs, path, conf);
127    * return writer; } catch (Exception e) { throw new
128    * IOException("cannot get log writer", e); } }
129    */
130 
131   /**
132    * Construct the HLog directory name
133    * 
134    * @param serverName
135    *          Server name formatted as described in {@link ServerName}
136    * @return the relative HLog directory name, e.g.
137    *         <code>.logs/1.example.org,60030,12345</code> if
138    *         <code>serverName</code> passed is
139    *         <code>1.example.org,60030,12345</code>
140    */
141   public static String getHLogDirectoryName(final String serverName) {
142     StringBuilder dirName = new StringBuilder(HConstants.HREGION_LOGDIR_NAME);
143     dirName.append("/");
144     dirName.append(serverName);
145     return dirName.toString();
146   }
147 
148   /**
149    * @param regiondir
150    *          This regions directory in the filesystem.
151    * @return The directory that holds recovered edits files for the region
152    *         <code>regiondir</code>
153    */
154   public static Path getRegionDirRecoveredEditsDir(final Path regiondir) {
155     return new Path(regiondir, HConstants.RECOVERED_EDITS_DIR);
156   }
157 
158   /**
159    * Move aside a bad edits file.
160    * 
161    * @param fs
162    * @param edits
163    *          Edits file to move aside.
164    * @return The name of the moved aside file.
165    * @throws IOException
166    */
167   public static Path moveAsideBadEditsFile(final FileSystem fs, final Path edits)
168       throws IOException {
169     Path moveAsideName = new Path(edits.getParent(), edits.getName() + "."
170         + System.currentTimeMillis());
171     if (!fs.rename(edits, moveAsideName)) {
172       LOG.warn("Rename failed from " + edits + " to " + moveAsideName);
173     }
174     return moveAsideName;
175   }
176 
177   /**
178    * @param path
179    *          - the path to analyze. Expected format, if it's in hlog directory:
180    *          / [base directory for hbase] / hbase / .logs / ServerName /
181    *          logfile
182    * @return null if it's not a log file. Returns the ServerName of the region
183    *         server that created this log file otherwise.
184    */
185   public static ServerName getServerNameFromHLogDirectoryName(
186       Configuration conf, String path) throws IOException {
187     if (path == null
188         || path.length() <= HConstants.HREGION_LOGDIR_NAME.length()) {
189       return null;
190     }
191 
192     if (conf == null) {
193       throw new IllegalArgumentException("parameter conf must be set");
194     }
195 
196     final String rootDir = conf.get(HConstants.HBASE_DIR);
197     if (rootDir == null || rootDir.isEmpty()) {
198       throw new IllegalArgumentException(HConstants.HBASE_DIR
199           + " key not found in conf.");
200     }
201 
202     final StringBuilder startPathSB = new StringBuilder(rootDir);
203     if (!rootDir.endsWith("/"))
204       startPathSB.append('/');
205     startPathSB.append(HConstants.HREGION_LOGDIR_NAME);
206     if (!HConstants.HREGION_LOGDIR_NAME.endsWith("/"))
207       startPathSB.append('/');
208     final String startPath = startPathSB.toString();
209 
210     String fullPath;
211     try {
212       fullPath = FileSystem.get(conf).makeQualified(new Path(path)).toString();
213     } catch (IllegalArgumentException e) {
214       LOG.info("Call to makeQualified failed on " + path + " " + e.getMessage());
215       return null;
216     }
217 
218     if (!fullPath.startsWith(startPath)) {
219       return null;
220     }
221 
222     final String serverNameAndFile = fullPath.substring(startPath.length());
223 
224     if (serverNameAndFile.indexOf('/') < "a,0,0".length()) {
225       // Either it's a file (not a directory) or it's not a ServerName format
226       return null;
227     }
228 
229     final String serverName = serverNameAndFile.substring(0,
230         serverNameAndFile.indexOf('/') - 1);
231 
232     if (!ServerName.isFullServerName(serverName)) {
233       return null;
234     }
235 
236     return ServerName.parseServerName(serverName);
237   }
238 
239   /**
240    * Returns sorted set of edit files made by wal-log splitter, excluding files
241    * with '.temp' suffix.
242    * 
243    * @param fs
244    * @param regiondir
245    * @return Files in passed <code>regiondir</code> as a sorted set.
246    * @throws IOException
247    */
248   public static NavigableSet<Path> getSplitEditFilesSorted(final FileSystem fs,
249       final Path regiondir) throws IOException {
250     NavigableSet<Path> filesSorted = new TreeSet<Path>();
251     Path editsdir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);
252     if (!fs.exists(editsdir))
253       return filesSorted;
254     FileStatus[] files = FSUtils.listStatus(fs, editsdir, new PathFilter() {
255       @Override
256       public boolean accept(Path p) {
257         boolean result = false;
258         try {
259           // Return files and only files that match the editfile names pattern.
260           // There can be other files in this directory other than edit files.
261           // In particular, on error, we'll move aside the bad edit file giving
262           // it a timestamp suffix. See moveAsideBadEditsFile.
263           Matcher m = HLog.EDITFILES_NAME_PATTERN.matcher(p.getName());
264           result = fs.isFile(p) && m.matches();
265           // Skip the file whose name ends with RECOVERED_LOG_TMPFILE_SUFFIX,
266           // because it means splithlog thread is writting this file.
267           if (p.getName().endsWith(HLog.RECOVERED_LOG_TMPFILE_SUFFIX)) {
268             result = false;
269           }
270         } catch (IOException e) {
271           LOG.warn("Failed isFile check on " + p);
272         }
273         return result;
274       }
275     });
276     if (files == null)
277       return filesSorted;
278     for (FileStatus status : files) {
279       filesSorted.add(status.getPath());
280     }
281     return filesSorted;
282   }
283 
284   public static boolean isMetaFile(Path p) {
285     if (p.getName().endsWith(HLog.META_HLOG_FILE_EXTN)) {
286       return true;
287     }
288     return false;
289   }
290 }