1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.NavigableMap;
26 import java.util.Set;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReentrantLock;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.fs.PathFilter;
38 import org.apache.hadoop.hbase.ClusterId;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
45 import org.apache.hadoop.hbase.RemoteExceptionHandler;
46 import org.apache.hadoop.hbase.Server;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.backup.HFileArchiver;
49 import org.apache.hadoop.hbase.catalog.MetaReader;
50 import org.apache.hadoop.hbase.client.Result;
51 import org.apache.hadoop.hbase.exceptions.DeserializationException;
52 import org.apache.hadoop.hbase.fs.HFileSystem;
53 import org.apache.hadoop.hbase.regionserver.HRegion;
54 import org.apache.hadoop.hbase.regionserver.wal.HLog;
55 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
56 import org.apache.hadoop.hbase.util.Bytes;
57 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
58 import org.apache.hadoop.hbase.util.FSTableDescriptors;
59 import org.apache.hadoop.hbase.util.FSUtils;
60 import org.apache.zookeeper.KeeperException;
61
62
63
64
65
66
67 @InterfaceAudience.Private
68 public class MasterFileSystem {
69 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
70
71 Configuration conf;
72
73 Server master;
74
75 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
76
77 private ClusterId clusterId;
78
79 private final FileSystem fs;
80
81 private volatile boolean fsOk = true;
82
83 private final Path oldLogDir;
84
85 private final Path rootdir;
86
87 private final Path tempdir;
88
89 final Lock splitLogLock = new ReentrantLock();
90 final boolean distributedLogReplay;
91 final SplitLogManager splitLogManager;
92 private final MasterServices services;
93
94 final static PathFilter META_FILTER = new PathFilter() {
95 public boolean accept(Path p) {
96 return HLogUtil.isMetaFile(p);
97 }
98 };
99
100 final static PathFilter NON_META_FILTER = new PathFilter() {
101 public boolean accept(Path p) {
102 return !HLogUtil.isMetaFile(p);
103 }
104 };
105
106 public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
107 throws IOException {
108 this.conf = master.getConfiguration();
109 this.master = master;
110 this.services = services;
111
112
113
114
115 this.rootdir = FSUtils.getRootDir(conf);
116 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
117
118
119 this.fs = this.rootdir.getFileSystem(conf);
120 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
121
122 fs.setConf(conf);
123 this.distributedLogReplay = this.conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY,
124 HConstants.DEFAULT_DISTRIBUTED_LOG_REPLAY_CONFIG);
125
126
127 this.oldLogDir = createInitialFileSystemLayout();
128 HFileSystem.addLocationsOrderInterceptor(conf);
129 this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
130 master.getConfiguration(), master, services,
131 master.getServerName(), masterRecovery);
132 }
133
134
135
136
137
138
139
140
141
142
143
144 private Path createInitialFileSystemLayout() throws IOException {
145
146 checkRootDir(this.rootdir, conf, this.fs);
147
148
149 checkTempDir(this.tempdir, conf, this.fs);
150
151 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
152
153
154 if(!this.fs.exists(oldLogDir)) {
155 this.fs.mkdirs(oldLogDir);
156 }
157
158 return oldLogDir;
159 }
160
161 public FileSystem getFileSystem() {
162 return this.fs;
163 }
164
165
166
167
168
169 public Path getOldLogDir() {
170 return this.oldLogDir;
171 }
172
173
174
175
176
177
178 public boolean checkFileSystem() {
179 if (this.fsOk) {
180 try {
181 FSUtils.checkFileSystemAvailable(this.fs);
182 FSUtils.checkDfsSafeMode(this.conf);
183 } catch (IOException e) {
184 master.abort("Shutting down HBase cluster: file system not available", e);
185 this.fsOk = false;
186 }
187 }
188 return this.fsOk;
189 }
190
191
192
193
194 public Path getRootDir() {
195 return this.rootdir;
196 }
197
198
199
200
201 public Path getTempDir() {
202 return this.tempdir;
203 }
204
205
206
207
208 public ClusterId getClusterId() {
209 return clusterId;
210 }
211
212
213
214
215
216 Set<ServerName> getFailedServersFromLogFolders() {
217 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
218 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
219
220 Set<ServerName> serverNames = new HashSet<ServerName>();
221 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
222
223 do {
224 if (master.isStopped()) {
225 LOG.warn("Master stopped while trying to get failed servers.");
226 break;
227 }
228 try {
229 if (!this.fs.exists(logsDirPath)) return serverNames;
230 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
231
232
233 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
234 .keySet();
235
236 if (logFolders == null || logFolders.length == 0) {
237 LOG.debug("No log files to split, proceeding...");
238 return serverNames;
239 }
240 for (FileStatus status : logFolders) {
241 String sn = status.getPath().getName();
242
243 if (sn.endsWith(HLog.SPLITTING_EXT)) {
244 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
245 }
246 ServerName serverName = ServerName.parseServerName(sn);
247 if (!onlineServers.contains(serverName)) {
248 LOG.info("Log folder " + status.getPath() + " doesn't belong "
249 + "to a known region server, splitting");
250 serverNames.add(serverName);
251 } else {
252 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
253 }
254 }
255 retrySplitting = false;
256 } catch (IOException ioe) {
257 LOG.warn("Failed getting failed servers to be recovered.", ioe);
258 if (!checkFileSystem()) {
259 LOG.warn("Bad Filesystem, exiting");
260 Runtime.getRuntime().halt(1);
261 }
262 try {
263 if (retrySplitting) {
264 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
265 }
266 } catch (InterruptedException e) {
267 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
268 Thread.currentThread().interrupt();
269 retrySplitting = false;
270 Runtime.getRuntime().halt(1);
271 }
272 }
273 } while (retrySplitting);
274
275 return serverNames;
276 }
277
278 public void splitLog(final ServerName serverName) throws IOException {
279 Set<ServerName> serverNames = new HashSet<ServerName>();
280 serverNames.add(serverName);
281 splitLog(serverNames);
282 }
283
284
285
286
287
288
289 public void splitMetaLog(final ServerName serverName) throws IOException {
290 Set<ServerName> serverNames = new HashSet<ServerName>();
291 serverNames.add(serverName);
292 splitMetaLog(serverNames);
293 }
294
295
296
297
298
299
300 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
301 splitLog(serverNames, META_FILTER);
302 }
303
304 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
305 List<Path> logDirs = new ArrayList<Path>();
306 boolean needReleaseLock = false;
307 if (!this.services.isInitialized()) {
308
309 this.splitLogLock.lock();
310 needReleaseLock = true;
311 }
312 try {
313 for (ServerName serverName : serverNames) {
314 Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
315 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
316
317 if (fs.exists(logDir)) {
318 if (!this.fs.rename(logDir, splitDir)) {
319 throw new IOException("Failed fs.rename for log split: " + logDir);
320 }
321 logDir = splitDir;
322 LOG.debug("Renamed region directory: " + splitDir);
323 } else if (!fs.exists(splitDir)) {
324 LOG.info("Log dir for server " + serverName + " does not exist");
325 continue;
326 }
327 logDirs.add(splitDir);
328 }
329 } finally {
330 if (needReleaseLock) {
331 this.splitLogLock.unlock();
332 }
333 }
334 return logDirs;
335 }
336
337
338
339
340
341
342
343 public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
344 if (!this.distributedLogReplay) {
345 return;
346 }
347
348 for (ServerName serverName : serverNames) {
349 NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
350 if (regions == null) {
351 continue;
352 }
353 try {
354 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
355 } catch (KeeperException e) {
356 throw new IOException(e);
357 }
358 }
359 }
360
361
362
363
364
365
366
367 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
368 if (!this.distributedLogReplay) {
369 return;
370 }
371
372 if (regions == null || regions.isEmpty()) {
373 return;
374 }
375 try {
376 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
377 } catch (KeeperException e) {
378 throw new IOException(e);
379 }
380 }
381
382 public void splitLog(final Set<ServerName> serverNames) throws IOException {
383 splitLog(serverNames, NON_META_FILTER);
384 }
385
386
387
388
389
390
391 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
392 throws KeeperException {
393 this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
394 }
395
396
397
398
399
400
401
402
403 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
404 long splitTime = 0, splitLogSize = 0;
405 List<Path> logDirs = getLogDirs(serverNames);
406
407 splitLogManager.handleDeadWorkers(serverNames);
408 splitTime = EnvironmentEdgeManager.currentTimeMillis();
409 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
410 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
411
412 if (this.metricsMasterFilesystem != null) {
413 if (filter == META_FILTER) {
414 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
415 } else {
416 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
417 }
418 }
419 }
420
421
422
423
424
425
426
427
428
429
430 @SuppressWarnings("deprecation")
431 private Path checkRootDir(final Path rd, final Configuration c,
432 final FileSystem fs)
433 throws IOException {
434
435 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
436
437 try {
438 if (!fs.exists(rd)) {
439 fs.mkdirs(rd);
440
441
442
443
444
445
446
447 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
448 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
449 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
450 } else {
451 if (!fs.isDirectory(rd)) {
452 throw new IllegalArgumentException(rd.toString() + " is not a directory");
453 }
454
455 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
456 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
457 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
458 }
459 } catch (DeserializationException de) {
460 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
461 IOException ioe = new IOException();
462 ioe.initCause(de);
463 throw ioe;
464 } catch (IllegalArgumentException iae) {
465 LOG.fatal("Please fix invalid configuration for "
466 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
467 throw iae;
468 }
469
470 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
471 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
472 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
473 }
474 clusterId = FSUtils.getClusterId(fs, rd);
475
476
477 if (!FSUtils.metaRegionExists(fs, rd)) {
478 bootstrap(rd, c);
479 } else {
480
481 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
482 .migrateFSTableDescriptorsIfNecessary(fs, rd);
483 }
484
485
486 new FSTableDescriptors(fs, rd).createTableDescriptor(HTableDescriptor.META_TABLEDESC);
487
488 return rd;
489 }
490
491
492
493
494
495 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
496 throws IOException {
497
498 if (fs.exists(tmpdir)) {
499
500
501 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
502 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
503 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
504 }
505 }
506 if (!fs.delete(tmpdir, true)) {
507 throw new IOException("Unable to clean the temp directory: " + tmpdir);
508 }
509 }
510
511
512 if (!fs.mkdirs(tmpdir)) {
513 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
514 }
515 }
516
517 private static void bootstrap(final Path rd, final Configuration c)
518 throws IOException {
519 LOG.info("BOOTSTRAP: creating hbase:meta region");
520 try {
521
522
523
524
525 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
526 setInfoFamilyCachingForMeta(false);
527 HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
528 HTableDescriptor.META_TABLEDESC);
529 setInfoFamilyCachingForMeta(true);
530 HRegion.closeHRegion(meta);
531 } catch (IOException e) {
532 e = RemoteExceptionHandler.checkIOException(e);
533 LOG.error("bootstrap", e);
534 throw e;
535 }
536 }
537
538
539
540
541 public static void setInfoFamilyCachingForMeta(final boolean b) {
542 for (HColumnDescriptor hcd:
543 HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
544 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
545 hcd.setBlockCacheEnabled(b);
546 hcd.setInMemory(b);
547 }
548 }
549 }
550
551
552 public void deleteRegion(HRegionInfo region) throws IOException {
553 HFileArchiver.archiveRegion(conf, fs, region);
554 }
555
556 public void deleteTable(TableName tableName) throws IOException {
557 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
558 }
559
560
561
562
563
564
565
566 public Path moveTableToTemp(TableName tableName) throws IOException {
567 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
568 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
569
570
571 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
572 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
573 }
574
575 if (!fs.rename(srcPath, tempPath)) {
576 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
577 }
578
579 return tempPath;
580 }
581
582 public void updateRegionInfo(HRegionInfo region) {
583
584
585
586 }
587
588 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
589 throws IOException {
590
591 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
592 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
593
594
595 Path familyDir = new Path(tableDir,
596 new Path(region.getEncodedName(), Bytes.toString(familyName)));
597 if (fs.delete(familyDir, true) == false) {
598 throw new IOException("Could not delete family "
599 + Bytes.toString(familyName) + " from FileSystem for region "
600 + region.getRegionNameAsString() + "(" + region.getEncodedName()
601 + ")");
602 }
603 }
604
605 public void stop() {
606 if (splitLogManager != null) {
607 this.splitLogManager.stop();
608 }
609 }
610
611
612
613
614
615
616
617
618 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
619 throws IOException {
620 LOG.info("DeleteColumn. Table = " + tableName
621 + " family = " + Bytes.toString(familyName));
622 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
623 htd.removeFamily(familyName);
624 this.services.getTableDescriptors().add(htd);
625 return htd;
626 }
627
628
629
630
631
632
633
634
635 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
636 throws IOException {
637 LOG.info("AddModifyColumn. Table = " + tableName
638 + " HCD = " + hcd.toString());
639
640 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
641 byte [] familyName = hcd.getName();
642 if(!htd.hasFamily(familyName)) {
643 throw new InvalidFamilyOperationException("Family '" +
644 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
645 }
646 htd.addFamily(hcd);
647 this.services.getTableDescriptors().add(htd);
648 return htd;
649 }
650
651
652
653
654
655
656
657
658 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
659 throws IOException {
660 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
661 hcd.toString());
662 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
663 if (htd == null) {
664 throw new InvalidFamilyOperationException("Family '" +
665 hcd.getNameAsString() + "' cannot be modified as HTD is null");
666 }
667 htd.addFamily(hcd);
668 this.services.getTableDescriptors().add(htd);
669 return htd;
670 }
671
672 private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
673 throws IOException {
674 if (!this.master.isStopped()) {
675 try {
676 this.master.getCatalogTracker().waitForMeta();
677 return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
678 } catch (InterruptedException e) {
679 Thread.currentThread().interrupt();
680 throw new IOException("Interrupted", e);
681 }
682 }
683 return null;
684 }
685 }