1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.NavigableMap;
27 import java.util.Set;
28 import java.util.concurrent.locks.Lock;
29 import java.util.concurrent.locks.ReentrantLock;
30
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.apache.hadoop.hbase.classification.InterfaceAudience;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.fs.FileStatus;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.fs.PathFilter;
39 import org.apache.hadoop.hbase.ClusterId;
40 import org.apache.hadoop.hbase.TableName;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HConstants;
43 import org.apache.hadoop.hbase.HRegionInfo;
44 import org.apache.hadoop.hbase.HTableDescriptor;
45 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
46 import org.apache.hadoop.hbase.RemoteExceptionHandler;
47 import org.apache.hadoop.hbase.Server;
48 import org.apache.hadoop.hbase.ServerName;
49 import org.apache.hadoop.hbase.backup.HFileArchiver;
50 import org.apache.hadoop.hbase.catalog.MetaReader;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.exceptions.DeserializationException;
53 import org.apache.hadoop.hbase.fs.HFileSystem;
54 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
55 import org.apache.hadoop.hbase.regionserver.HRegion;
56 import org.apache.hadoop.hbase.regionserver.wal.HLog;
57 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
58 import org.apache.hadoop.hbase.util.Bytes;
59 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
60 import org.apache.hadoop.hbase.util.FSTableDescriptors;
61 import org.apache.hadoop.hbase.util.FSUtils;
62 import org.apache.zookeeper.KeeperException;
63
64
65
66
67
68
69 @InterfaceAudience.Private
70 public class MasterFileSystem {
71 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
72
73 Configuration conf;
74
75 Server master;
76
77 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
78
79 private ClusterId clusterId;
80
81 private final FileSystem fs;
82
83 private volatile boolean fsOk = true;
84
85 private final Path oldLogDir;
86
87 private final Path rootdir;
88
89 private final Path tempdir;
90
91 final Lock splitLogLock = new ReentrantLock();
92 final boolean distributedLogReplay;
93 final SplitLogManager splitLogManager;
94 private final MasterServices services;
95
96 final static PathFilter META_FILTER = new PathFilter() {
97 @Override
98 public boolean accept(Path p) {
99 return HLogUtil.isMetaFile(p);
100 }
101 };
102
103 final static PathFilter NON_META_FILTER = new PathFilter() {
104 @Override
105 public boolean accept(Path p) {
106 return !HLogUtil.isMetaFile(p);
107 }
108 };
109
110 public MasterFileSystem(Server master, MasterServices services, boolean masterRecovery)
111 throws IOException {
112 this.conf = master.getConfiguration();
113 this.master = master;
114 this.services = services;
115
116
117
118
119 this.rootdir = FSUtils.getRootDir(conf);
120 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
121
122
123 this.fs = this.rootdir.getFileSystem(conf);
124 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
125
126 fs.setConf(conf);
127
128
129 this.oldLogDir = createInitialFileSystemLayout();
130 HFileSystem.addLocationsOrderInterceptor(conf);
131 try {
132 this.splitLogManager = new SplitLogManager(master.getZooKeeper(), master.getConfiguration(),
133 master, services,
134 master.getServerName(), masterRecovery);
135 } catch (KeeperException e) {
136 throw new IOException(e);
137 }
138 this.distributedLogReplay = (this.splitLogManager.getRecoveryMode() == RecoveryMode.LOG_REPLAY);
139 }
140
141
142
143
144
145
146
147
148
149
150
151 private Path createInitialFileSystemLayout() throws IOException {
152
153 checkRootDir(this.rootdir, conf, this.fs);
154
155
156 checkTempDir(this.tempdir, conf, this.fs);
157
158 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
159
160
161 if(!this.fs.exists(oldLogDir)) {
162 this.fs.mkdirs(oldLogDir);
163 }
164
165 return oldLogDir;
166 }
167
168 public FileSystem getFileSystem() {
169 return this.fs;
170 }
171
172
173
174
175
176 public Path getOldLogDir() {
177 return this.oldLogDir;
178 }
179
180
181
182
183
184
185 public boolean checkFileSystem() {
186 if (this.fsOk) {
187 try {
188 FSUtils.checkFileSystemAvailable(this.fs);
189 FSUtils.checkDfsSafeMode(this.conf);
190 } catch (IOException e) {
191 master.abort("Shutting down HBase cluster: file system not available", e);
192 this.fsOk = false;
193 }
194 }
195 return this.fsOk;
196 }
197
198
199
200
201 public Path getRootDir() {
202 return this.rootdir;
203 }
204
205
206
207
208 public Path getTempDir() {
209 return this.tempdir;
210 }
211
212
213
214
215 public ClusterId getClusterId() {
216 return clusterId;
217 }
218
219
220
221
222
223 Set<ServerName> getFailedServersFromLogFolders() {
224 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
225 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
226
227 Set<ServerName> serverNames = new HashSet<ServerName>();
228 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
229
230 do {
231 if (master.isStopped()) {
232 LOG.warn("Master stopped while trying to get failed servers.");
233 break;
234 }
235 try {
236 if (!this.fs.exists(logsDirPath)) return serverNames;
237 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
238
239
240 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
241 .keySet();
242
243 if (logFolders == null || logFolders.length == 0) {
244 LOG.debug("No log files to split, proceeding...");
245 return serverNames;
246 }
247 for (FileStatus status : logFolders) {
248 String sn = status.getPath().getName();
249
250 if (sn.endsWith(HLog.SPLITTING_EXT)) {
251 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
252 }
253 ServerName serverName = ServerName.parseServerName(sn);
254 if (!onlineServers.contains(serverName)) {
255 LOG.info("Log folder " + status.getPath() + " doesn't belong "
256 + "to a known region server, splitting");
257 serverNames.add(serverName);
258 } else {
259 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
260 }
261 }
262 retrySplitting = false;
263 } catch (IOException ioe) {
264 LOG.warn("Failed getting failed servers to be recovered.", ioe);
265 if (!checkFileSystem()) {
266 LOG.warn("Bad Filesystem, exiting");
267 Runtime.getRuntime().halt(1);
268 }
269 try {
270 if (retrySplitting) {
271 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
272 }
273 } catch (InterruptedException e) {
274 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
275 Thread.currentThread().interrupt();
276 retrySplitting = false;
277 Runtime.getRuntime().halt(1);
278 }
279 }
280 } while (retrySplitting);
281
282 return serverNames;
283 }
284
285 public void splitLog(final ServerName serverName) throws IOException {
286 Set<ServerName> serverNames = new HashSet<ServerName>();
287 serverNames.add(serverName);
288 splitLog(serverNames);
289 }
290
291
292
293
294
295
296 public void splitMetaLog(final ServerName serverName) throws IOException {
297 Set<ServerName> serverNames = new HashSet<ServerName>();
298 serverNames.add(serverName);
299 splitMetaLog(serverNames);
300 }
301
302
303
304
305
306
307 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
308 splitLog(serverNames, META_FILTER);
309 }
310
311 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
312 List<Path> logDirs = new ArrayList<Path>();
313 boolean needReleaseLock = false;
314 if (!this.services.isInitialized()) {
315
316 this.splitLogLock.lock();
317 needReleaseLock = true;
318 }
319 try {
320 for (ServerName serverName : serverNames) {
321 Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
322 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
323
324 if (fs.exists(logDir)) {
325 if (!this.fs.rename(logDir, splitDir)) {
326 throw new IOException("Failed fs.rename for log split: " + logDir);
327 }
328 logDir = splitDir;
329 LOG.debug("Renamed region directory: " + splitDir);
330 } else if (!fs.exists(splitDir)) {
331 LOG.info("Log dir for server " + serverName + " does not exist");
332 continue;
333 }
334 logDirs.add(splitDir);
335 }
336 } finally {
337 if (needReleaseLock) {
338 this.splitLogLock.unlock();
339 }
340 }
341 return logDirs;
342 }
343
344
345
346
347
348
349
350 public void prepareLogReplay(Set<ServerName> serverNames) throws IOException {
351 if (!this.distributedLogReplay) {
352 return;
353 }
354
355 for (ServerName serverName : serverNames) {
356 NavigableMap<HRegionInfo, Result> regions = this.getServerUserRegions(serverName);
357 if (regions == null) {
358 continue;
359 }
360 try {
361 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions.keySet());
362 } catch (KeeperException e) {
363 throw new IOException(e);
364 }
365 }
366 }
367
368
369
370
371
372
373
374 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
375 if (!this.distributedLogReplay) {
376 return;
377 }
378
379 if (regions == null || regions.isEmpty()) {
380 return;
381 }
382 try {
383 this.splitLogManager.markRegionsRecoveringInZK(serverName, regions);
384 } catch (KeeperException e) {
385 throw new IOException(e);
386 }
387 }
388
389 public void splitLog(final Set<ServerName> serverNames) throws IOException {
390 splitLog(serverNames, NON_META_FILTER);
391 }
392
393
394
395
396
397
398 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
399 throws KeeperException {
400 this.splitLogManager.removeStaleRecoveringRegionsFromZK(failedServers);
401 }
402
403
404
405
406
407
408
409
410 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
411 long splitTime = 0, splitLogSize = 0;
412 List<Path> logDirs = getLogDirs(serverNames);
413
414 splitLogManager.handleDeadWorkers(serverNames);
415 splitTime = EnvironmentEdgeManager.currentTimeMillis();
416 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
417 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
418
419 if (this.metricsMasterFilesystem != null) {
420 if (filter == META_FILTER) {
421 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
422 } else {
423 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
424 }
425 }
426 }
427
428
429
430
431
432
433
434
435
436
437 @SuppressWarnings("deprecation")
438 private Path checkRootDir(final Path rd, final Configuration c,
439 final FileSystem fs)
440 throws IOException {
441
442 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
443
444 try {
445 if (!fs.exists(rd)) {
446 fs.mkdirs(rd);
447
448
449
450
451
452
453
454 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
455 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
456 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
457 } else {
458 if (!fs.isDirectory(rd)) {
459 throw new IllegalArgumentException(rd.toString() + " is not a directory");
460 }
461
462 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
463 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
464 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
465 }
466 } catch (DeserializationException de) {
467 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
468 IOException ioe = new IOException();
469 ioe.initCause(de);
470 throw ioe;
471 } catch (IllegalArgumentException iae) {
472 LOG.fatal("Please fix invalid configuration for "
473 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
474 throw iae;
475 }
476
477 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
478 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
479 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
480 }
481 clusterId = FSUtils.getClusterId(fs, rd);
482
483
484 if (!FSUtils.metaRegionExists(fs, rd)) {
485 bootstrap(rd, c);
486 } else {
487
488 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
489 .migrateFSTableDescriptorsIfNecessary(fs, rd);
490 }
491
492
493 new FSTableDescriptors(fs, rd).createTableDescriptor(HTableDescriptor.META_TABLEDESC);
494
495 return rd;
496 }
497
498
499
500
501
502 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
503 throws IOException {
504
505 if (fs.exists(tmpdir)) {
506
507
508 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
509 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
510 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
511 }
512 }
513 if (!fs.delete(tmpdir, true)) {
514 throw new IOException("Unable to clean the temp directory: " + tmpdir);
515 }
516 }
517
518
519 if (!fs.mkdirs(tmpdir)) {
520 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
521 }
522 }
523
524 private static void bootstrap(final Path rd, final Configuration c)
525 throws IOException {
526 LOG.info("BOOTSTRAP: creating hbase:meta region");
527 try {
528
529
530
531
532 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
533 setInfoFamilyCachingForMeta(false);
534 HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
535 HTableDescriptor.META_TABLEDESC, null, true, true);
536 setInfoFamilyCachingForMeta(true);
537 HRegion.closeHRegion(meta);
538 } catch (IOException e) {
539 e = RemoteExceptionHandler.checkIOException(e);
540 LOG.error("bootstrap", e);
541 throw e;
542 }
543 }
544
545
546
547
548 public static void setInfoFamilyCachingForMeta(final boolean b) {
549 for (HColumnDescriptor hcd:
550 HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
551 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
552 hcd.setBlockCacheEnabled(b);
553 hcd.setInMemory(b);
554 }
555 }
556 }
557
558
559 public void deleteRegion(HRegionInfo region) throws IOException {
560 HFileArchiver.archiveRegion(conf, fs, region);
561 }
562
563 public void deleteTable(TableName tableName) throws IOException {
564 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
565 }
566
567
568
569
570
571
572
573 public Path moveTableToTemp(TableName tableName) throws IOException {
574 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
575 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
576
577
578 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
579 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
580 }
581
582 if (!fs.rename(srcPath, tempPath)) {
583 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
584 }
585
586 return tempPath;
587 }
588
589 public void updateRegionInfo(HRegionInfo region) {
590
591
592
593 }
594
595 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
596 throws IOException {
597
598 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
599 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
600
601
602 Path familyDir = new Path(tableDir,
603 new Path(region.getEncodedName(), Bytes.toString(familyName)));
604 if (fs.delete(familyDir, true) == false) {
605 throw new IOException("Could not delete family "
606 + Bytes.toString(familyName) + " from FileSystem for region "
607 + region.getRegionNameAsString() + "(" + region.getEncodedName()
608 + ")");
609 }
610 }
611
612 public void stop() {
613 if (splitLogManager != null) {
614 this.splitLogManager.stop();
615 }
616 }
617
618
619
620
621
622
623
624
625 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
626 throws IOException {
627 LOG.info("DeleteColumn. Table = " + tableName
628 + " family = " + Bytes.toString(familyName));
629 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
630 htd.removeFamily(familyName);
631 this.services.getTableDescriptors().add(htd);
632 return htd;
633 }
634
635
636
637
638
639
640
641
642 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
643 throws IOException {
644 LOG.info("AddModifyColumn. Table = " + tableName
645 + " HCD = " + hcd.toString());
646
647 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
648 byte [] familyName = hcd.getName();
649 if(!htd.hasFamily(familyName)) {
650 throw new InvalidFamilyOperationException("Family '" +
651 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
652 }
653 htd.addFamily(hcd);
654 this.services.getTableDescriptors().add(htd);
655 return htd;
656 }
657
658
659
660
661
662
663
664
665 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
666 throws IOException {
667 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
668 hcd.toString());
669 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
670 if (htd == null) {
671 throw new InvalidFamilyOperationException("Family '" +
672 hcd.getNameAsString() + "' cannot be modified as HTD is null");
673 }
674 htd.addFamily(hcd);
675 this.services.getTableDescriptors().add(htd);
676 return htd;
677 }
678
679 private NavigableMap<HRegionInfo, Result> getServerUserRegions(ServerName serverName)
680 throws IOException {
681 if (!this.master.isStopped()) {
682 try {
683 this.master.getCatalogTracker().waitForMeta();
684 return MetaReader.getServerUserRegions(this.master.getCatalogTracker(), serverName);
685 } catch (InterruptedException e) {
686 throw (InterruptedIOException)new InterruptedIOException().initCause(e);
687 }
688 }
689 return null;
690 }
691
692
693
694
695
696
697
698 public void setLogRecoveryMode() throws IOException {
699 try {
700 this.splitLogManager.setRecoveryMode(false);
701 } catch (KeeperException e) {
702 throw new IOException(e);
703 }
704 }
705
706 public RecoveryMode getLogRecoveryMode() {
707 return this.splitLogManager.getRecoveryMode();
708 }
709 }