1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.NavigableMap;
27 import java.util.Set;
28 import java.util.concurrent.locks.Lock;
29 import java.util.concurrent.locks.ReentrantLock;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.catalog.MetaReader;
34 import org.apache.hadoop.hbase.classification.InterfaceAudience;
35 import org.apache.hadoop.hbase.client.Result;
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileStatus;
38 import org.apache.hadoop.fs.FileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.fs.PathFilter;
41 import org.apache.hadoop.fs.permission.FsPermission;
42 import org.apache.hadoop.hbase.ClusterId;
43 import org.apache.hadoop.hbase.TableName;
44 import org.apache.hadoop.hbase.HColumnDescriptor;
45 import org.apache.hadoop.hbase.HConstants;
46 import org.apache.hadoop.hbase.HRegionInfo;
47 import org.apache.hadoop.hbase.HTableDescriptor;
48 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
49 import org.apache.hadoop.hbase.RemoteExceptionHandler;
50 import org.apache.hadoop.hbase.Server;
51 import org.apache.hadoop.hbase.ServerName;
52 import org.apache.hadoop.hbase.backup.HFileArchiver;
53 import org.apache.hadoop.hbase.exceptions.DeserializationException;
54 import org.apache.hadoop.hbase.fs.HFileSystem;
55 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
56 import org.apache.hadoop.hbase.regionserver.HRegion;
57 import org.apache.hadoop.hbase.regionserver.wal.HLog;
58 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
59 import org.apache.hadoop.hbase.util.Bytes;
60 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
61 import org.apache.hadoop.hbase.util.FSTableDescriptors;
62 import org.apache.hadoop.hbase.util.FSUtils;
63 import org.apache.zookeeper.KeeperException;
64
65
66
67
68
69
70 @InterfaceAudience.Private
71 public class MasterFileSystem {
72 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
73
74 Configuration conf;
75
76 Server master;
77
78 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
79
80 private ClusterId clusterId;
81
82 private final FileSystem fs;
83
84 private volatile boolean fsOk = true;
85
86 private final Path oldLogDir;
87
88 private final Path rootdir;
89
90 private final Path tempdir;
91
92 final Lock splitLogLock = new ReentrantLock();
93 final boolean distributedLogReplay;
94 final SplitLogManager splitLogManager;
95 private final MasterServices services;
96
97 final static PathFilter META_FILTER = new PathFilter() {
98 @Override
99 public boolean accept(Path p) {
100 return HLogUtil.isMetaFile(p);
101 }
102 };
103
104 final static PathFilter NON_META_FILTER = new PathFilter() {
105 @Override
106 public boolean accept(Path p) {
107 return !HLogUtil.isMetaFile(p);
108 }
109 };
110
111 public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
112 throws IOException {
113 this.conf = master.getConfiguration();
114 this.master = master;
115 this.services = services;
116
117
118
119
120 this.rootdir = FSUtils.getRootDir(conf);
121 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
122
123
124 this.fs = this.rootdir.getFileSystem(conf);
125 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
126
127 fs.setConf(conf);
128
129
130 this.oldLogDir = createInitialFileSystemLayout();
131 HFileSystem.addLocationsOrderInterceptor(conf);
132 try {
133 this.splitLogManager = new SplitLogManager(master.getZooKeeper(), master.getConfiguration(),
134 master, services,
135 master.getServerName(), masterRecovery);
136 } catch (KeeperException e) {
137 throw new IOException(e);
138 }
139 this.distributedLogReplay = (this.splitLogManager.getRecoveryMode() == RecoveryMode.LOG_REPLAY);
140 }
141
142
143
144
145
146
147
148
149
150
151
152 private Path createInitialFileSystemLayout() throws IOException {
153
154 checkRootDir(this.rootdir, conf, this.fs);
155
156
157 checkTempDir(this.tempdir, conf, this.fs);
158
159 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
160
161
162 if(!this.fs.exists(oldLogDir)) {
163 this.fs.mkdirs(oldLogDir);
164 }
165
166 return oldLogDir;
167 }
168
169 public FileSystem getFileSystem() {
170 return this.fs;
171 }
172
173
174
175
176
177 public Path getOldLogDir() {
178 return this.oldLogDir;
179 }
180
181
182
183
184
185
186 public boolean checkFileSystem() {
187 if (this.fsOk) {
188 try {
189 FSUtils.checkFileSystemAvailable(this.fs);
190 FSUtils.checkDfsSafeMode(this.conf);
191 } catch (IOException e) {
192 master.abort("Shutting down HBase cluster: file system not available", e);
193 this.fsOk = false;
194 }
195 }
196 return this.fsOk;
197 }
198
199
200
201
202 public Path getRootDir() {
203 return this.rootdir;
204 }
205
206
207
208
209 public Path getTempDir() {
210 return this.tempdir;
211 }
212
213
214
215
216 public ClusterId getClusterId() {
217 return clusterId;
218 }
219
220
221
222
223
224 Set<ServerName> getFailedServersFromLogFolders() {
225 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
226 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
227
228 Set<ServerName> serverNames = new HashSet<ServerName>();
229 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
230
231 do {
232 if (master.isStopped()) {
233 LOG.warn("Master stopped while trying to get failed servers.");
234 break;
235 }
236 try {
237 if (!this.fs.exists(logsDirPath)) return serverNames;
238 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
239
240
241 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
242 .keySet();
243
244 if (logFolders == null || logFolders.length == 0) {
245 LOG.debug("No log files to split, proceeding...");
246 return serverNames;
247 }
248 for (FileStatus status : logFolders) {
249 String sn = status.getPath().getName();
250
251 if (sn.endsWith(HLog.SPLITTING_EXT)) {
252 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
253 }
254 ServerName serverName = ServerName.parseServerName(sn);
255 if (!onlineServers.contains(serverName)) {
256 LOG.info("Log folder " + status.getPath() + " doesn't belong "
257 + "to a known region server, splitting");
258 serverNames.add(serverName);
259 } else {
260 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
261 }
262 }
263 retrySplitting = false;
264 } catch (IOException ioe) {
265 LOG.warn("Failed getting failed servers to be recovered.", ioe);
266 if (!checkFileSystem()) {
267 LOG.warn("Bad Filesystem, exiting");
268 Runtime.getRuntime().halt(1);
269 }
270 try {
271 if (retrySplitting) {
272 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
273 }
274 } catch (InterruptedException e) {
275 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
276 Thread.currentThread().interrupt();
277 retrySplitting = false;
278 Runtime.getRuntime().halt(1);
279 }
280 }
281 } while (retrySplitting);
282
283 return serverNames;
284 }
285
286 public void splitLog(final ServerName serverName) throws IOException {
287 Set<ServerName> serverNames = new HashSet<ServerName>();
288 serverNames.add(serverName);
289 splitLog(serverNames);
290 }
291
292
293
294
295
296
297 public void splitMetaLog(final ServerName serverName) throws IOException {
298 Set<ServerName> serverNames = new HashSet<ServerName>();
299 serverNames.add(serverName);
300 splitMetaLog(serverNames);
301 }
302
303
304
305
306
307
308 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
309 splitLog(serverNames, META_FILTER);
310 }
311
312 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
313 List<Path> logDirs = new ArrayList<Path>();
314 boolean needReleaseLock = false;
315 if (!this.services.isInitialized()) {
316
317 this.splitLogLock.lock();
318 needReleaseLock = true;
319 }
320 try {
321 for (ServerName serverName : serverNames) {
322 Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
323 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
324
325 if (fs.exists(logDir)) {
326 if (!this.fs.rename(logDir, splitDir)) {
327 throw new IOException("Failed fs.rename for log split: " + logDir);
328 }
329 logDir = splitDir;
330 LOG.debug("Renamed region directory: " + splitDir);
331 } else if (!fs.exists(splitDir)) {
332 LOG.info("Log dir for server " + serverName + " does not exist");
333 continue;
334 }
335 logDirs.add(splitDir);
336 }
337 } finally {
338 if (needReleaseLock) {
339 this.splitLogLock.unlock();
340 }
341 }
342 return logDirs;
343 }
344
345
346
347
348
349
350
351 public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
352 if (!this.distributedLogReplay) {
353 return;
354 }
355
356 for (ServerName serverName : serverNames) {
357 NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
358 if (regions == null) {
359 continue;
360 }
361 try {
362 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
363 } catch (KeeperException e) {
364 throw new IOException(e);
365 }
366 }
367 }
368
369
370
371
372
373
374
375 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
376 if (!this.distributedLogReplay) {
377 return;
378 }
379
380 if (regions == null || regions.isEmpty()) {
381 return;
382 }
383 try {
384 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
385 } catch (KeeperException e) {
386 throw new IOException(e);
387 }
388 }
389
390 public void splitLog(final Set<ServerName> serverNames) throws IOException {
391 splitLog(serverNames, NON_META_FILTER);
392 }
393
394
395
396
397
398
399 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
400 throws KeeperException {
401 this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
402 }
403
404
405
406
407
408
409
410
411 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
412 long splitTime = 0, splitLogSize = 0;
413 List<Path> logDirs = getLogDirs(serverNames);
414
415 splitLogManager.handleDeadWorkers(serverNames);
416 splitTime = EnvironmentEdgeManager.currentTimeMillis();
417 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
418 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
419
420 if (this.metricsMasterFilesystem != null) {
421 if (filter == META_FILTER) {
422 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
423 } else {
424 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
425 }
426 }
427 }
428
429
430
431
432
433
434
435
436
437
438 @SuppressWarnings("deprecation")
439 private Path checkRootDir(final Path rd, final Configuration c,
440 final FileSystem fs)
441 throws IOException {
442
443 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
444
445 boolean isSecurityEnabled = "kerberos".equalsIgnoreCase(c.get("hbase.security.authentication"));
446 FsPermission rootDirPerms = new FsPermission(c.get("hbase.rootdir.perms", "700"));
447
448
449 try {
450 if (!fs.exists(rd)) {
451 if (isSecurityEnabled) {
452 fs.mkdirs(rd, rootDirPerms);
453 } else {
454 fs.mkdirs(rd);
455 }
456
457
458
459
460
461
462
463 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
464 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
465 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
466 } else {
467 if (!fs.isDirectory(rd)) {
468 throw new IllegalArgumentException(rd.toString() + " is not a directory");
469 }
470 if (isSecurityEnabled && !rootDirPerms.equals(fs.getFileStatus(rd).getPermission())) {
471
472 LOG.warn("Found rootdir permissions NOT matching expected \"hbase.rootdir.perms\" for "
473 + "rootdir=" + rd.toString() + " permissions=" + fs.getFileStatus(rd).getPermission()
474 + " and \"hbase.rootdir.perms\" configured as "
475 + c.get("hbase.rootdir.perms", "700") + ". Automatically setting the permissions. You"
476 + " can change the permissions by setting \"hbase.rootdir.perms\" in hbase-site.xml "
477 + "and restarting the master");
478 fs.setPermission(rd, rootDirPerms);
479 }
480
481 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
482 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
483 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
484 }
485 } catch (DeserializationException de) {
486 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
487 IOException ioe = new IOException();
488 ioe.initCause(de);
489 throw ioe;
490 } catch (IllegalArgumentException iae) {
491 LOG.fatal("Please fix invalid configuration for "
492 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
493 throw iae;
494 }
495
496 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
497 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
498 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
499 }
500 clusterId = FSUtils.getClusterId(fs, rd);
501
502
503 if (!FSUtils.metaRegionExists(fs, rd)) {
504 bootstrap(rd, c);
505 } else {
506
507 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
508 .migrateFSTableDescriptorsIfNecessary(fs, rd);
509 }
510
511
512
513
514
515 FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
516 fsd.createTableDescriptor(
517 new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
518
519 return rd;
520 }
521
522
523
524
525
526 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
527 throws IOException {
528
529 if (fs.exists(tmpdir)) {
530
531
532 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
533 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
534 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
535 }
536 }
537 if (!fs.delete(tmpdir, true)) {
538 throw new IOException("Unable to clean the temp directory: " + tmpdir);
539 }
540 }
541
542
543 if (!fs.mkdirs(tmpdir)) {
544 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
545 }
546 }
547
548 private static void bootstrap(final Path rd, final Configuration c)
549 throws IOException {
550 LOG.info("BOOTSTRAP: creating hbase:meta region");
551 try {
552
553
554
555
556 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
557 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
558 setInfoFamilyCachingForMeta(metaDescriptor, false);
559 HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
560 setInfoFamilyCachingForMeta(metaDescriptor, true);
561 HRegion.closeHRegion(meta);
562 } catch (IOException e) {
563 e = RemoteExceptionHandler.checkIOException(e);
564 LOG.error("bootstrap", e);
565 throw e;
566 }
567 }
568
569
570
571
572 public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
573 final boolean b) {
574 for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
575 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
576 hcd.setBlockCacheEnabled(b);
577 hcd.setInMemory(b);
578 }
579 }
580 }
581
582 public void deleteRegion(HRegionInfo region) throws IOException {
583 HFileArchiver.archiveRegion(conf, fs, region);
584 }
585
586 public void deleteTable(TableName tableName) throws IOException {
587 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
588 }
589
590
591
592
593
594
595
596 public Path moveTableToTemp(TableName tableName) throws IOException {
597 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
598 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
599
600
601 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
602 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
603 }
604
605 if (!fs.rename(srcPath, tempPath)) {
606 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
607 }
608
609 return tempPath;
610 }
611
612 public void updateRegionInfo(HRegionInfo region) {
613
614
615
616 }
617
618 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
619 throws IOException {
620
621 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
622 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
623
624
625 Path familyDir = new Path(tableDir,
626 new Path(region.getEncodedName(), Bytes.toString(familyName)));
627 if (fs.delete(familyDir, true) == false) {
628 throw new IOException("Could not delete family "
629 + Bytes.toString(familyName) + " from FileSystem for region "
630 + region.getRegionNameAsString() + "(" + region.getEncodedName()
631 + ")");
632 }
633 }
634
635 public void stop() {
636 if (splitLogManager != null) {
637 this.splitLogManager.stop();
638 }
639 }
640
641
642
643
644
645
646
647
648 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
649 throws IOException {
650 LOG.info("DeleteColumn. Table = " + tableName
651 + " family = " + Bytes.toString(familyName));
652 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
653 htd.removeFamily(familyName);
654 this.services.getTableDescriptors().add(htd);
655 return htd;
656 }
657
658
659
660
661
662
663
664
665 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
666 throws IOException {
667 LOG.info("AddModifyColumn. Table = " + tableName
668 + " HCD = " + hcd.toString());
669
670 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
671 byte [] familyName = hcd.getName();
672 if(!htd.hasFamily(familyName)) {
673 throw new InvalidFamilyOperationException("Family '" +
674 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
675 }
676 htd.addFamily(hcd);
677 this.services.getTableDescriptors().add(htd);
678 return htd;
679 }
680
681
682
683
684
685
686
687
688 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
689 throws IOException {
690 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
691 hcd.toString());
692 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
693 if (htd == null) {
694 throw new InvalidFamilyOperationException("Family '" +
695 hcd.getNameAsString() + "' cannot be modified as HTD is null");
696 }
697 htd.addFamily(hcd);
698 this.services.getTableDescriptors().add(htd);
699 return htd;
700 }
701
702 private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
703 throws IOException {
704 if (!this.master.isStopped()) {
705 try {
706 this.master.getCatalogTracker().waitForMeta();
707 return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
708 } catch (InterruptedException e) {
709 throw (InterruptedIOException)new InterruptedIOException().initCause(e);
710 }
711 }
712 return null;
713 }
714
715
716
717
718
719
720
721 public void setLogRecoveryMode() throws IOException {
722 try {
723 this.splitLogManager.setRecoveryMode(false);
724 } catch (KeeperException e) {
725 throw new IOException(e);
726 }
727 }
728
729 public RecoveryMode getLogRecoveryMode() {
730 return this.splitLogManager.getRecoveryMode();
731 }
732 }