1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.snapshot; 19 20 import java.io.IOException; 21 22 import org.apache.commons.logging.Log; 23 import org.apache.commons.logging.LogFactory; 24 import org.apache.hadoop.conf.Configuration; 25 import org.apache.hadoop.fs.FSDataInputStream; 26 import org.apache.hadoop.fs.FSDataOutputStream; 27 import org.apache.hadoop.fs.FileSystem; 28 import org.apache.hadoop.fs.Path; 29 import org.apache.hadoop.fs.permission.FsPermission; 30 import org.apache.hadoop.hbase.HConstants; 31 import org.apache.hadoop.hbase.HTableDescriptor; 32 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription; 33 import org.apache.hadoop.hbase.util.Bytes; 34 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; 35 import org.apache.hadoop.hbase.util.FSUtils; 36 37 /** 38 * Utility class to help manage {@link SnapshotDescription SnapshotDesriptions}. 39 * <p> 40 * Snapshots are laid out on disk like this: 41 * 42 * <pre> 43 * /hbase/.snapshots 44 * /.tmp <---- working directory 45 * /[snapshot name] <----- completed snapshot 46 * </pre> 47 * 48 * A completed snapshot named 'completed' then looks like (multiple regions, servers, files, etc. 49 * signified by '...' on the same directory depth). 50 * 51 * <pre> 52 * /hbase/.snapshots/completed 53 * .snapshotinfo <--- Description of the snapshot 54 * .tableinfo <--- Copy of the tableinfo 55 * /.logs 56 * /[server_name] 57 * /... [log files] 58 * ... 59 * /[region name] <---- All the region's information 60 * .regioninfo <---- Copy of the HRegionInfo 61 * /[column family name] 62 * /[hfile name] <--- name of the hfile in the real region 63 * ... 64 * ... 65 * ... 66 * </pre> 67 * 68 * Utility methods in this class are useful for getting the correct locations for different parts of 69 * the snapshot, as well as moving completed snapshots into place (see 70 * {@link #completeSnapshot}, and writing the 71 * {@link SnapshotDescription} to the working snapshot directory. 72 */ 73 public class SnapshotDescriptionUtils { 74 75 /** 76 * Filter that only accepts completed snapshot directories 77 */ 78 public static class CompletedSnaphotDirectoriesFilter extends FSUtils.DirFilter { 79 80 /** 81 * @param fs 82 */ 83 public CompletedSnaphotDirectoriesFilter(FileSystem fs) { 84 super(fs); 85 } 86 87 @Override 88 public boolean accept(Path path) { 89 // only accept directories that aren't the tmp directory 90 if (super.accept(path)) { 91 return !path.getName().equals(SNAPSHOT_TMP_DIR_NAME); 92 } 93 return false; 94 } 95 96 } 97 98 private static final Log LOG = LogFactory.getLog(SnapshotDescriptionUtils.class); 99 /** 100 * Version of the fs layout for a snapshot. Future snapshots may have different file layouts, 101 * which we may need to read in differently. 102 */ 103 public static final int SNAPSHOT_LAYOUT_VERSION = 0; 104 105 // snapshot directory constants 106 /** 107 * The file contains the snapshot basic information and it is under the directory of a snapshot. 108 */ 109 public static final String SNAPSHOTINFO_FILE = ".snapshotinfo"; 110 111 /** Temporary directory under the snapshot directory to store in-progress snapshots */ 112 public static final String SNAPSHOT_TMP_DIR_NAME = ".tmp"; 113 // snapshot operation values 114 /** Default value if no start time is specified */ 115 public static final long NO_SNAPSHOT_START_TIME_SPECIFIED = 0; 116 117 public static final String MASTER_SNAPSHOT_TIMEOUT_MILLIS = "hbase.snapshot.master.timeout.millis"; 118 119 /** By default, wait 60 seconds for a snapshot to complete */ 120 public static final long DEFAULT_MAX_WAIT_TIME = 60000; 121 122 private SnapshotDescriptionUtils() { 123 // private constructor for utility class 124 } 125 126 /** 127 * Check to make sure that the description of the snapshot requested is valid 128 * @param snapshot description of the snapshot 129 * @throws IllegalArgumentException if the name of the snapshot or the name of the table to 130 * snapshot are not valid names. 131 */ 132 public static void assertSnapshotRequestIsValid(SnapshotDescription snapshot) 133 throws IllegalArgumentException { 134 // FIXME these method names is really bad - trunk will probably change 135 // .META. and -ROOT- snapshots are not allowed 136 if (HTableDescriptor.isMetaTable(Bytes.toBytes(snapshot.getTable()))) { 137 throw new IllegalArgumentException(".META. and -ROOT- snapshots are not allowed"); 138 } 139 // make sure the snapshot name is valid 140 HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getName()), true); 141 // make sure the table name is valid 142 HTableDescriptor.isLegalTableName(Bytes.toBytes(snapshot.getTable())); 143 } 144 145 /** 146 * @param conf {@link Configuration} from which to check for the timeout 147 * @param type type of snapshot being taken 148 * @param defaultMaxWaitTime Default amount of time to wait, if none is in the configuration 149 * @return the max amount of time the master should wait for a snapshot to complete 150 */ 151 public static long getMaxMasterTimeout(Configuration conf, SnapshotDescription.Type type, 152 long defaultMaxWaitTime) { 153 String confKey; 154 switch (type) { 155 case DISABLED: 156 default: 157 confKey = MASTER_SNAPSHOT_TIMEOUT_MILLIS; 158 } 159 return conf.getLong(confKey, defaultMaxWaitTime); 160 } 161 162 /** 163 * Get the snapshot root directory. All the snapshots are kept under this directory, i.e. 164 * ${hbase.rootdir}/.snapshot 165 * @param rootDir hbase root directory 166 * @return the base directory in which all snapshots are kept 167 */ 168 public static Path getSnapshotRootDir(final Path rootDir) { 169 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); 170 } 171 172 /** 173 * Get the directory for a specified snapshot. This directory is a sub-directory of snapshot root 174 * directory and all the data files for a snapshot are kept under this directory. 175 * @param snapshot snapshot being taken 176 * @param rootDir hbase root directory 177 * @return the final directory for the completed snapshot 178 */ 179 public static Path getCompletedSnapshotDir(final SnapshotDescription snapshot, final Path rootDir) { 180 return getCompletedSnapshotDir(snapshot.getName(), rootDir); 181 } 182 183 /** 184 * Get the directory for a completed snapshot. This directory is a sub-directory of snapshot root 185 * directory and all the data files for a snapshot are kept under this directory. 186 * @param snapshotName name of the snapshot being taken 187 * @param rootDir hbase root directory 188 * @return the final directory for the completed snapshot 189 */ 190 public static Path getCompletedSnapshotDir(final String snapshotName, final Path rootDir) { 191 return getCompletedSnapshotDir(getSnapshotsDir(rootDir), snapshotName); 192 } 193 194 /** 195 * Get the general working directory for snapshots - where they are built, where they are 196 * temporarily copied on export, etc. 197 * @param rootDir root directory of the HBase installation 198 * @return Path to the snapshot tmp directory, relative to the passed root directory 199 */ 200 public static Path getWorkingSnapshotDir(final Path rootDir) { 201 return new Path(getSnapshotsDir(rootDir), SNAPSHOT_TMP_DIR_NAME); 202 } 203 204 /** 205 * Get the directory to build a snapshot, before it is finalized 206 * @param snapshot snapshot that will be built 207 * @param rootDir root directory of the hbase installation 208 * @return {@link Path} where one can build a snapshot 209 */ 210 public static Path getWorkingSnapshotDir(SnapshotDescription snapshot, final Path rootDir) { 211 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshot.getName()); 212 } 213 214 /** 215 * Get the directory to build a snapshot, before it is finalized 216 * @param snapshotName name of the snapshot 217 * @param rootDir root directory of the hbase installation 218 * @return {@link Path} where one can build a snapshot 219 */ 220 public static Path getWorkingSnapshotDir(String snapshotName, final Path rootDir) { 221 return getCompletedSnapshotDir(getWorkingSnapshotDir(rootDir), snapshotName); 222 } 223 224 /** 225 * Get the directory to store the snapshot instance 226 * @param snapshotsDir hbase-global directory for storing all snapshots 227 * @param snapshotName name of the snapshot to take 228 * @return 229 */ 230 private static final Path getCompletedSnapshotDir(final Path snapshotsDir, String snapshotName) { 231 return new Path(snapshotsDir, snapshotName); 232 } 233 234 /** 235 * @param rootDir hbase root directory 236 * @return the directory for all completed snapshots; 237 */ 238 public static final Path getSnapshotsDir(Path rootDir) { 239 return new Path(rootDir, HConstants.SNAPSHOT_DIR_NAME); 240 } 241 242 /** 243 * Convert the passed snapshot description into a 'full' snapshot description based on default 244 * parameters, if none have been supplied. This resolves any 'optional' parameters that aren't 245 * supplied to their default values. 246 * @param snapshot general snapshot descriptor 247 * @param conf Configuration to read configured snapshot defaults if snapshot is not complete 248 * @return a valid snapshot description 249 * @throws IllegalArgumentException if the {@link SnapshotDescription} is not a complete 250 * {@link SnapshotDescription}. 251 */ 252 public static SnapshotDescription validate(SnapshotDescription snapshot, Configuration conf) 253 throws IllegalArgumentException { 254 if (!snapshot.hasTable()) { 255 throw new IllegalArgumentException( 256 "Descriptor doesn't apply to a table, so we can't build it."); 257 } 258 259 // set the creation time, if one hasn't been set 260 long time = snapshot.getCreationTime(); 261 if (time == SnapshotDescriptionUtils.NO_SNAPSHOT_START_TIME_SPECIFIED) { 262 time = EnvironmentEdgeManager.currentTimeMillis(); 263 LOG.debug("Creation time not specified, setting to:" + time + " (current time:" 264 + EnvironmentEdgeManager.currentTimeMillis() + ")."); 265 SnapshotDescription.Builder builder = snapshot.toBuilder(); 266 builder.setCreationTime(time); 267 snapshot = builder.build(); 268 } 269 return snapshot; 270 } 271 272 /** 273 * Write the snapshot description into the working directory of a snapshot 274 * @param snapshot description of the snapshot being taken 275 * @param workingDir working directory of the snapshot 276 * @param fs {@link FileSystem} on which the snapshot should be taken 277 * @throws IOException if we can't reach the filesystem and the file cannot be cleaned up on 278 * failure 279 */ 280 public static void writeSnapshotInfo(SnapshotDescription snapshot, Path workingDir, FileSystem fs) 281 throws IOException { 282 FsPermission perms = FSUtils.getFilePermissions(fs, fs.getConf(), 283 HConstants.DATA_FILE_UMASK_KEY); 284 Path snapshotInfo = new Path(workingDir, SnapshotDescriptionUtils.SNAPSHOTINFO_FILE); 285 try { 286 FSDataOutputStream out = FSUtils.create(fs, snapshotInfo, perms, true); 287 try { 288 snapshot.writeTo(out); 289 } finally { 290 out.close(); 291 } 292 } catch (IOException e) { 293 // if we get an exception, try to remove the snapshot info 294 if (!fs.delete(snapshotInfo, false)) { 295 String msg = "Couldn't delete snapshot info file: " + snapshotInfo; 296 LOG.error(msg); 297 throw new IOException(msg); 298 } 299 } 300 } 301 302 /** 303 * Read in the {@link SnapshotDescription} stored for the snapshot in the passed directory 304 * @param fs filesystem where the snapshot was taken 305 * @param snapshotDir directory where the snapshot was stored 306 * @return the stored snapshot description 307 * @throws CorruptedSnapshotException if the snapshot cannot be read 308 */ 309 public static SnapshotDescription readSnapshotInfo(FileSystem fs, Path snapshotDir) 310 throws CorruptedSnapshotException { 311 Path snapshotInfo = new Path(snapshotDir, SNAPSHOTINFO_FILE); 312 try { 313 FSDataInputStream in = null; 314 try { 315 in = fs.open(snapshotInfo); 316 return SnapshotDescription.parseFrom(in); 317 } finally { 318 if (in != null) in.close(); 319 } 320 } catch (IOException e) { 321 throw new CorruptedSnapshotException("Couldn't read snapshot info from:" + snapshotInfo, e); 322 } 323 } 324 325 /** 326 * Move the finished snapshot to its final, publicly visible directory - this marks the snapshot 327 * as 'complete'. 328 * @param snapshot description of the snapshot being tabken 329 * @param rootdir root directory of the hbase installation 330 * @param workingDir directory where the in progress snapshot was built 331 * @param fs {@link FileSystem} where the snapshot was built 332 * @throws SnapshotCreationException if the snapshot could not be moved 333 * @throws IOException the filesystem could not be reached 334 */ 335 public static void completeSnapshot(SnapshotDescription snapshot, Path rootdir, Path workingDir, 336 FileSystem fs) throws SnapshotCreationException, IOException { 337 Path finishedDir = getCompletedSnapshotDir(snapshot, rootdir); 338 LOG.debug("Snapshot is done, just moving the snapshot from " + workingDir + " to " 339 + finishedDir); 340 if (!fs.rename(workingDir, finishedDir)) { 341 throw new SnapshotCreationException("Failed to move working directory(" + workingDir 342 + ") to completed directory(" + finishedDir + ").", snapshot); 343 } 344 } 345 346 /** 347 * Returns a single line (no \n) representation of snapshot metadata. Use this instead of 348 * {@link SnapshotDescription#toString()}. We don't replace SnapshotDescrpition's toString 349 * because it is auto-generated by protoc. 350 * @param ssd 351 * @return Single line string with a summary of the snapshot parameters 352 */ 353 public static String toString(SnapshotDescription ssd) { 354 if (ssd == null) { 355 return null; 356 } 357 return "{ ss=" + ssd.getName() + " table=" + ssd.getTable() 358 + " type=" + ssd.getType() + " }"; 359 } 360 }