1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.io.InterruptedIOException;
23 import java.util.ArrayList;
24 import java.util.HashSet;
25 import java.util.List;
26 import java.util.Set;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReentrantLock;
29
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.hbase.classification.InterfaceAudience;
33 import org.apache.hadoop.conf.Configuration;
34 import org.apache.hadoop.fs.FileStatus;
35 import org.apache.hadoop.fs.FileSystem;
36 import org.apache.hadoop.fs.Path;
37 import org.apache.hadoop.fs.PathFilter;
38 import org.apache.hadoop.hbase.ClusterId;
39 import org.apache.hadoop.hbase.TableName;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.InvalidFamilyOperationException;
45 import org.apache.hadoop.hbase.RemoteExceptionHandler;
46 import org.apache.hadoop.hbase.Server;
47 import org.apache.hadoop.hbase.ServerName;
48 import org.apache.hadoop.hbase.backup.HFileArchiver;
49 import org.apache.hadoop.hbase.exceptions.DeserializationException;
50 import org.apache.hadoop.hbase.fs.HFileSystem;
51 import org.apache.hadoop.hbase.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode;
52 import org.apache.hadoop.hbase.regionserver.HRegion;
53 import org.apache.hadoop.hbase.wal.DefaultWALProvider;
54 import org.apache.hadoop.hbase.wal.WALSplitter;
55 import org.apache.hadoop.hbase.util.Bytes;
56 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
57 import org.apache.hadoop.hbase.util.FSTableDescriptors;
58 import org.apache.hadoop.hbase.util.FSUtils;
59
60
61
62
63
64
65 @InterfaceAudience.Private
66 public class MasterFileSystem {
67 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
68
69 Configuration conf;
70
71 Server master;
72
73 private final MetricsMasterFileSystem metricsMasterFilesystem = new MetricsMasterFileSystem();
74
75 private ClusterId clusterId;
76
77 private final FileSystem fs;
78
79 private volatile boolean fsOk = true;
80
81 private final Path oldLogDir;
82
83 private final Path rootdir;
84
85 private final Path tempdir;
86
87 final Lock splitLogLock = new ReentrantLock();
88 final boolean distributedLogReplay;
89 final SplitLogManager splitLogManager;
90 private final MasterServices services;
91
92 final static PathFilter META_FILTER = new PathFilter() {
93 @Override
94 public boolean accept(Path p) {
95 return DefaultWALProvider.isMetaFile(p);
96 }
97 };
98
99 final static PathFilter NON_META_FILTER = new PathFilter() {
100 @Override
101 public boolean accept(Path p) {
102 return !DefaultWALProvider.isMetaFile(p);
103 }
104 };
105
106 public MasterFileSystem(Server master, MasterServices services)
107 throws IOException {
108 this.conf = master.getConfiguration();
109 this.master = master;
110 this.services = services;
111
112
113
114
115 this.rootdir = FSUtils.getRootDir(conf);
116 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
117
118
119 this.fs = this.rootdir.getFileSystem(conf);
120 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
121
122 fs.setConf(conf);
123
124
125 this.oldLogDir = createInitialFileSystemLayout();
126 HFileSystem.addLocationsOrderInterceptor(conf);
127 this.splitLogManager =
128 new SplitLogManager(master, master.getConfiguration(), master, services,
129 master.getServerName());
130 this.distributedLogReplay = this.splitLogManager.isLogReplaying();
131 }
132
133
134
135
136
137
138
139
140
141
142
143 private Path createInitialFileSystemLayout() throws IOException {
144
145 checkRootDir(this.rootdir, conf, this.fs);
146
147
148 checkTempDir(this.tempdir, conf, this.fs);
149
150 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
151
152
153 if(!this.fs.exists(oldLogDir)) {
154 this.fs.mkdirs(oldLogDir);
155 }
156
157 return oldLogDir;
158 }
159
160 public FileSystem getFileSystem() {
161 return this.fs;
162 }
163
164
165
166
167
168 public Path getOldLogDir() {
169 return this.oldLogDir;
170 }
171
172
173
174
175
176
177 public boolean checkFileSystem() {
178 if (this.fsOk) {
179 try {
180 FSUtils.checkFileSystemAvailable(this.fs);
181 FSUtils.checkDfsSafeMode(this.conf);
182 } catch (IOException e) {
183 master.abort("Shutting down HBase cluster: file system not available", e);
184 this.fsOk = false;
185 }
186 }
187 return this.fsOk;
188 }
189
190
191
192
193 public Path getRootDir() {
194 return this.rootdir;
195 }
196
197
198
199
200 public Path getTempDir() {
201 return this.tempdir;
202 }
203
204
205
206
207 public ClusterId getClusterId() {
208 return clusterId;
209 }
210
211
212
213
214
215 Set<ServerName> getFailedServersFromLogFolders() {
216 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
217 WALSplitter.SPLIT_SKIP_ERRORS_DEFAULT);
218
219 Set<ServerName> serverNames = new HashSet<ServerName>();
220 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
221
222 do {
223 if (master.isStopped()) {
224 LOG.warn("Master stopped while trying to get failed servers.");
225 break;
226 }
227 try {
228 if (!this.fs.exists(logsDirPath)) return serverNames;
229 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
230
231
232 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
233 .keySet();
234
235 if (logFolders == null || logFolders.length == 0) {
236 LOG.debug("No log files to split, proceeding...");
237 return serverNames;
238 }
239 for (FileStatus status : logFolders) {
240 FileStatus[] curLogFiles = FSUtils.listStatus(this.fs, status.getPath(), null);
241 if (curLogFiles == null || curLogFiles.length == 0) {
242
243 continue;
244 }
245 final ServerName serverName = DefaultWALProvider.getServerNameFromWALDirectoryName(
246 status.getPath());
247 if (null == serverName) {
248 LOG.warn("Log folder " + status.getPath() + " doesn't look like its name includes a " +
249 "region server name; leaving in place. If you see later errors about missing " +
250 "write ahead logs they may be saved in this location.");
251 } else if (!onlineServers.contains(serverName)) {
252 LOG.info("Log folder " + status.getPath() + " doesn't belong "
253 + "to a known region server, splitting");
254 serverNames.add(serverName);
255 } else {
256 LOG.info("Log folder " + status.getPath() + " belongs to an existing region server");
257 }
258 }
259 retrySplitting = false;
260 } catch (IOException ioe) {
261 LOG.warn("Failed getting failed servers to be recovered.", ioe);
262 if (!checkFileSystem()) {
263 LOG.warn("Bad Filesystem, exiting");
264 Runtime.getRuntime().halt(1);
265 }
266 try {
267 if (retrySplitting) {
268 Thread.sleep(conf.getInt("hbase.hlog.split.failure.retry.interval", 30 * 1000));
269 }
270 } catch (InterruptedException e) {
271 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
272 Thread.currentThread().interrupt();
273 retrySplitting = false;
274 Runtime.getRuntime().halt(1);
275 }
276 }
277 } while (retrySplitting);
278
279 return serverNames;
280 }
281
282 public void splitLog(final ServerName serverName) throws IOException {
283 Set<ServerName> serverNames = new HashSet<ServerName>();
284 serverNames.add(serverName);
285 splitLog(serverNames);
286 }
287
288
289
290
291
292
293 public void splitMetaLog(final ServerName serverName) throws IOException {
294 Set<ServerName> serverNames = new HashSet<ServerName>();
295 serverNames.add(serverName);
296 splitMetaLog(serverNames);
297 }
298
299
300
301
302
303
304 public void splitMetaLog(final Set<ServerName> serverNames) throws IOException {
305 splitLog(serverNames, META_FILTER);
306 }
307
308 @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="UL_UNRELEASED_LOCK", justification=
309 "We only release this lock when we set it. Updates to code that uses it should verify use " +
310 "of the guard boolean.")
311 private List<Path> getLogDirs(final Set<ServerName> serverNames) throws IOException {
312 List<Path> logDirs = new ArrayList<Path>();
313 boolean needReleaseLock = false;
314 if (!this.services.isInitialized()) {
315
316 this.splitLogLock.lock();
317 needReleaseLock = true;
318 }
319 try {
320 for (ServerName serverName : serverNames) {
321 Path logDir = new Path(this.rootdir,
322 DefaultWALProvider.getWALDirectoryName(serverName.toString()));
323 Path splitDir = logDir.suffix(DefaultWALProvider.SPLITTING_EXT);
324
325 if (fs.exists(logDir)) {
326 if (!this.fs.rename(logDir, splitDir)) {
327 throw new IOException("Failed fs.rename for log split: " + logDir);
328 }
329 logDir = splitDir;
330 LOG.debug("Renamed region directory: " + splitDir);
331 } else if (!fs.exists(splitDir)) {
332 LOG.info("Log dir for server " + serverName + " does not exist");
333 continue;
334 }
335 logDirs.add(splitDir);
336 }
337 } finally {
338 if (needReleaseLock) {
339 this.splitLogLock.unlock();
340 }
341 }
342 return logDirs;
343 }
344
345
346
347
348
349
350
351 public void prepareLogReplay(ServerName serverName, Set<HRegionInfo> regions) throws IOException {
352 if (!this.distributedLogReplay) {
353 return;
354 }
355
356 if (regions == null || regions.isEmpty()) {
357 return;
358 }
359 this.splitLogManager.markRegionsRecovering(serverName, regions);
360 }
361
362 public void splitLog(final Set<ServerName> serverNames) throws IOException {
363 splitLog(serverNames, NON_META_FILTER);
364 }
365
366
367
368
369
370
371 void removeStaleRecoveringRegionsFromZK(final Set<ServerName> failedServers)
372 throws IOException, InterruptedIOException {
373 this.splitLogManager.removeStaleRecoveringRegions(failedServers);
374 }
375
376
377
378
379
380
381
382
383
384 public void splitLog(final Set<ServerName> serverNames, PathFilter filter) throws IOException {
385 long splitTime = 0, splitLogSize = 0;
386 List<Path> logDirs = getLogDirs(serverNames);
387
388 splitLogManager.handleDeadWorkers(serverNames);
389 splitTime = EnvironmentEdgeManager.currentTime();
390 splitLogSize = splitLogManager.splitLogDistributed(serverNames, logDirs, filter);
391 splitTime = EnvironmentEdgeManager.currentTime() - splitTime;
392
393 if (this.metricsMasterFilesystem != null) {
394 if (filter == META_FILTER) {
395 this.metricsMasterFilesystem.addMetaWALSplit(splitTime, splitLogSize);
396 } else {
397 this.metricsMasterFilesystem.addSplit(splitTime, splitLogSize);
398 }
399 }
400 }
401
402
403
404
405
406
407
408
409
410
411 @SuppressWarnings("deprecation")
412 private Path checkRootDir(final Path rd, final Configuration c,
413 final FileSystem fs)
414 throws IOException {
415
416 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
417
418 try {
419 if (!fs.exists(rd)) {
420 fs.mkdirs(rd);
421
422
423
424
425
426
427
428 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
429 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
430 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
431 } else {
432 if (!fs.isDirectory(rd)) {
433 throw new IllegalArgumentException(rd.toString() + " is not a directory");
434 }
435
436 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
437 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
438 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
439 }
440 } catch (DeserializationException de) {
441 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
442 IOException ioe = new IOException();
443 ioe.initCause(de);
444 throw ioe;
445 } catch (IllegalArgumentException iae) {
446 LOG.fatal("Please fix invalid configuration for "
447 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
448 throw iae;
449 }
450
451 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
452 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
453 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
454 }
455 clusterId = FSUtils.getClusterId(fs, rd);
456
457
458 if (!FSUtils.metaRegionExists(fs, rd)) {
459 bootstrap(rd, c);
460 } else {
461
462 org.apache.hadoop.hbase.util.FSTableDescriptorMigrationToSubdir
463 .migrateFSTableDescriptorsIfNecessary(fs, rd);
464 }
465
466
467
468
469
470 FSTableDescriptors fsd = new FSTableDescriptors(c, fs, rd);
471 fsd.createTableDescriptor(
472 new HTableDescriptor(fsd.get(TableName.META_TABLE_NAME)));
473
474 return rd;
475 }
476
477
478
479
480
481 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
482 throws IOException {
483
484 if (fs.exists(tmpdir)) {
485
486
487 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
488 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
489 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
490 }
491 }
492 if (!fs.delete(tmpdir, true)) {
493 throw new IOException("Unable to clean the temp directory: " + tmpdir);
494 }
495 }
496
497
498 if (!fs.mkdirs(tmpdir)) {
499 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
500 }
501 }
502
503 private static void bootstrap(final Path rd, final Configuration c)
504 throws IOException {
505 LOG.info("BOOTSTRAP: creating hbase:meta region");
506 try {
507
508
509
510
511 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
512 HTableDescriptor metaDescriptor = new FSTableDescriptors(c).get(TableName.META_TABLE_NAME);
513 setInfoFamilyCachingForMeta(metaDescriptor, false);
514 HRegion meta = HRegion.createHRegion(metaHRI, rd, c, metaDescriptor);
515 setInfoFamilyCachingForMeta(metaDescriptor, true);
516 HRegion.closeHRegion(meta);
517 } catch (IOException e) {
518 e = RemoteExceptionHandler.checkIOException(e);
519 LOG.error("bootstrap", e);
520 throw e;
521 }
522 }
523
524
525
526
527 public static void setInfoFamilyCachingForMeta(final HTableDescriptor metaDescriptor,
528 final boolean b) {
529 for (HColumnDescriptor hcd: metaDescriptor.getColumnFamilies()) {
530 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
531 hcd.setBlockCacheEnabled(b);
532 hcd.setInMemory(b);
533 }
534 }
535 }
536
537 public void deleteRegion(HRegionInfo region) throws IOException {
538 HFileArchiver.archiveRegion(conf, fs, region);
539 }
540
541 public void deleteTable(TableName tableName) throws IOException {
542 fs.delete(FSUtils.getTableDir(rootdir, tableName), true);
543 }
544
545
546
547
548
549
550
551 public Path moveTableToTemp(TableName tableName) throws IOException {
552 Path srcPath = FSUtils.getTableDir(rootdir, tableName);
553 Path tempPath = FSUtils.getTableDir(this.tempdir, tableName);
554
555
556 if (!fs.exists(tempPath.getParent()) && !fs.mkdirs(tempPath.getParent())) {
557 throw new IOException("HBase temp directory '" + tempPath.getParent() + "' creation failure.");
558 }
559
560 if (!fs.rename(srcPath, tempPath)) {
561 throw new IOException("Unable to move '" + srcPath + "' to temp '" + tempPath + "'");
562 }
563
564 return tempPath;
565 }
566
567 public void updateRegionInfo(HRegionInfo region) {
568
569
570
571 }
572
573 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
574 throws IOException {
575
576 Path tableDir = FSUtils.getTableDir(rootdir, region.getTable());
577 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
578
579
580 Path familyDir = new Path(tableDir,
581 new Path(region.getEncodedName(), Bytes.toString(familyName)));
582 if (fs.delete(familyDir, true) == false) {
583 throw new IOException("Could not delete family "
584 + Bytes.toString(familyName) + " from FileSystem for region "
585 + region.getRegionNameAsString() + "(" + region.getEncodedName()
586 + ")");
587 }
588 }
589
590 public void stop() {
591 if (splitLogManager != null) {
592 this.splitLogManager.stop();
593 }
594 }
595
596
597
598
599
600
601
602
603 public HTableDescriptor deleteColumn(TableName tableName, byte[] familyName)
604 throws IOException {
605 LOG.info("DeleteColumn. Table = " + tableName
606 + " family = " + Bytes.toString(familyName));
607 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
608 htd.removeFamily(familyName);
609 this.services.getTableDescriptors().add(htd);
610 return htd;
611 }
612
613
614
615
616
617
618
619
620 public HTableDescriptor modifyColumn(TableName tableName, HColumnDescriptor hcd)
621 throws IOException {
622 LOG.info("AddModifyColumn. Table = " + tableName
623 + " HCD = " + hcd.toString());
624
625 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
626 byte [] familyName = hcd.getName();
627 if(!htd.hasFamily(familyName)) {
628 throw new InvalidFamilyOperationException("Family '" +
629 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
630 }
631 htd.modifyFamily(hcd);
632 this.services.getTableDescriptors().add(htd);
633 return htd;
634 }
635
636
637
638
639
640
641
642
643 public HTableDescriptor addColumn(TableName tableName, HColumnDescriptor hcd)
644 throws IOException {
645 LOG.info("AddColumn. Table = " + tableName + " HCD = " +
646 hcd.toString());
647 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
648 if (htd == null) {
649 throw new InvalidFamilyOperationException("Family '" +
650 hcd.getNameAsString() + "' cannot be modified as HTD is null");
651 }
652 htd.addFamily(hcd);
653 this.services.getTableDescriptors().add(htd);
654 return htd;
655 }
656
657
658
659
660
661
662 public void setLogRecoveryMode() throws IOException {
663 this.splitLogManager.setRecoveryMode(false);
664 }
665
666 public RecoveryMode getLogRecoveryMode() {
667 return this.splitLogManager.getRecoveryMode();
668 }
669 }