1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.master;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Set;
25 import java.util.concurrent.locks.Lock;
26 import java.util.concurrent.locks.ReentrantLock;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.classification.InterfaceAudience;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.FileStatus;
33 import org.apache.hadoop.fs.FileSystem;
34 import org.apache.hadoop.fs.Path;
35 import org.apache.hadoop.fs.PathFilter;
36 import org.apache.hadoop.hbase.ClusterId;
37 import org.apache.hadoop.hbase.exceptions.DeserializationException;
38 import org.apache.hadoop.hbase.HColumnDescriptor;
39 import org.apache.hadoop.hbase.HConstants;
40 import org.apache.hadoop.hbase.HRegionInfo;
41 import org.apache.hadoop.hbase.HTableDescriptor;
42 import org.apache.hadoop.hbase.exceptions.InvalidFamilyOperationException;
43 import org.apache.hadoop.hbase.RemoteExceptionHandler;
44 import org.apache.hadoop.hbase.Server;
45 import org.apache.hadoop.hbase.ServerName;
46 import org.apache.hadoop.hbase.backup.HFileArchiver;
47 import org.apache.hadoop.hbase.fs.HFileSystem;
48 import org.apache.hadoop.hbase.regionserver.HRegion;
49 import org.apache.hadoop.hbase.regionserver.wal.HLog;
50 import org.apache.hadoop.hbase.regionserver.wal.HLogSplitter;
51 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
52 import org.apache.hadoop.hbase.exceptions.OrphanHLogAfterSplitException;
53 import org.apache.hadoop.hbase.util.Bytes;
54 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
55 import org.apache.hadoop.hbase.util.FSTableDescriptors;
56 import org.apache.hadoop.hbase.util.FSUtils;
57
58
59
60
61
62
63 @InterfaceAudience.Private
64 public class MasterFileSystem {
65 private static final Log LOG = LogFactory.getLog(MasterFileSystem.class.getName());
66
67 Configuration conf;
68
69 Server master;
70
71 MetricsMaster metricsMaster;
72
73 private ClusterId clusterId;
74
75 private final FileSystem fs;
76
77 private volatile boolean fsOk = true;
78
79 private final Path oldLogDir;
80
81 private final Path rootdir;
82
83 private final Path tempdir;
84
85 final Lock splitLogLock = new ReentrantLock();
86 final boolean distributedLogSplitting;
87 final SplitLogManager splitLogManager;
88 private final MasterServices services;
89
90 private final static PathFilter META_FILTER = new PathFilter() {
91 public boolean accept(Path p) {
92 return HLogUtil.isMetaFile(p);
93 }
94 };
95
96 private final static PathFilter NON_META_FILTER = new PathFilter() {
97 public boolean accept(Path p) {
98 return !HLogUtil.isMetaFile(p);
99 }
100 };
101
102 public MasterFileSystem(Server master, MasterServices services,
103 MetricsMaster metricsMaster, boolean masterRecovery)
104 throws IOException {
105 this.conf = master.getConfiguration();
106 this.master = master;
107 this.services = services;
108 this.metricsMaster = metricsMaster;
109
110
111
112
113 this.rootdir = FSUtils.getRootDir(conf);
114 this.tempdir = new Path(this.rootdir, HConstants.HBASE_TEMP_DIRECTORY);
115
116
117 this.fs = this.rootdir.getFileSystem(conf);
118 FSUtils.setFsDefault(conf, new Path(this.fs.getUri()));
119
120 fs.setConf(conf);
121 this.distributedLogSplitting =
122 conf.getBoolean(HConstants.DISTRIBUTED_LOG_SPLITTING_KEY, true);
123 if (this.distributedLogSplitting) {
124 this.splitLogManager = new SplitLogManager(master.getZooKeeper(),
125 master.getConfiguration(), master, services, master.getServerName());
126 this.splitLogManager.finishInitialization(masterRecovery);
127 } else {
128 this.splitLogManager = null;
129 }
130
131
132 this.oldLogDir = createInitialFileSystemLayout();
133 HFileSystem.addLocationsOrderInterceptor(conf);
134 }
135
136
137
138
139
140
141
142
143
144
145
146 private Path createInitialFileSystemLayout() throws IOException {
147
148 checkRootDir(this.rootdir, conf, this.fs);
149
150
151 checkTempDir(this.tempdir, conf, this.fs);
152
153 Path oldLogDir = new Path(this.rootdir, HConstants.HREGION_OLDLOGDIR_NAME);
154
155
156 if(!this.fs.exists(oldLogDir)) {
157 this.fs.mkdirs(oldLogDir);
158 }
159
160 return oldLogDir;
161 }
162
163 public FileSystem getFileSystem() {
164 return this.fs;
165 }
166
167
168
169
170
171 public Path getOldLogDir() {
172 return this.oldLogDir;
173 }
174
175
176
177
178
179
180 public boolean checkFileSystem() {
181 if (this.fsOk) {
182 try {
183 FSUtils.checkFileSystemAvailable(this.fs);
184 FSUtils.checkDfsSafeMode(this.conf);
185 } catch (IOException e) {
186 master.abort("Shutting down HBase cluster: file system not available", e);
187 this.fsOk = false;
188 }
189 }
190 return this.fsOk;
191 }
192
193
194
195
196 public Path getRootDir() {
197 return this.rootdir;
198 }
199
200
201
202
203 public Path getTempDir() {
204 return this.tempdir;
205 }
206
207
208
209
210 public ClusterId getClusterId() {
211 return clusterId;
212 }
213
214
215
216
217
218 void splitLogAfterStartup() {
219 boolean retrySplitting = !conf.getBoolean("hbase.hlog.split.skip.errors",
220 HLog.SPLIT_SKIP_ERRORS_DEFAULT);
221 Path logsDirPath = new Path(this.rootdir, HConstants.HREGION_LOGDIR_NAME);
222 do {
223 if (master.isStopped()) {
224 LOG.warn("Master stopped while splitting logs");
225 break;
226 }
227 List<ServerName> serverNames = new ArrayList<ServerName>();
228 try {
229 if (!this.fs.exists(logsDirPath)) return;
230 FileStatus[] logFolders = FSUtils.listStatus(this.fs, logsDirPath, null);
231
232
233 Set<ServerName> onlineServers = ((HMaster) master).getServerManager().getOnlineServers()
234 .keySet();
235
236 if (logFolders == null || logFolders.length == 0) {
237 LOG.debug("No log files to split, proceeding...");
238 return;
239 }
240 for (FileStatus status : logFolders) {
241 String sn = status.getPath().getName();
242
243 if (sn.endsWith(HLog.SPLITTING_EXT)) {
244 sn = sn.substring(0, sn.length() - HLog.SPLITTING_EXT.length());
245 }
246 ServerName serverName = ServerName.parseServerName(sn);
247 if (!onlineServers.contains(serverName)) {
248 LOG.info("Log folder " + status.getPath() + " doesn't belong "
249 + "to a known region server, splitting");
250 serverNames.add(serverName);
251 } else {
252 LOG.info("Log folder " + status.getPath()
253 + " belongs to an existing region server");
254 }
255 }
256 splitLog(serverNames, META_FILTER);
257 splitLog(serverNames, NON_META_FILTER);
258 retrySplitting = false;
259 } catch (IOException ioe) {
260 LOG.warn("Failed splitting of " + serverNames, ioe);
261 if (!checkFileSystem()) {
262 LOG.warn("Bad Filesystem, exiting");
263 Runtime.getRuntime().halt(1);
264 }
265 try {
266 if (retrySplitting) {
267 Thread.sleep(conf.getInt(
268 "hbase.hlog.split.failure.retry.interval", 30 * 1000));
269 }
270 } catch (InterruptedException e) {
271 LOG.warn("Interrupted, aborting since cannot return w/o splitting");
272 Thread.currentThread().interrupt();
273 retrySplitting = false;
274 Runtime.getRuntime().halt(1);
275 }
276 }
277 } while (retrySplitting);
278 }
279
280 public void splitLog(final ServerName serverName) throws IOException {
281 List<ServerName> serverNames = new ArrayList<ServerName>();
282 serverNames.add(serverName);
283 splitLog(serverNames);
284 }
285
286
287
288
289
290
291 public void splitMetaLog(final ServerName serverName) throws IOException {
292 long splitTime = 0, splitLogSize = 0;
293 List<ServerName> serverNames = new ArrayList<ServerName>();
294 serverNames.add(serverName);
295 List<Path> logDirs = getLogDirs(serverNames);
296 if (logDirs.isEmpty()) {
297 LOG.info("No meta logs to split");
298 return;
299 }
300 splitLogManager.handleDeadWorkers(serverNames);
301 splitTime = EnvironmentEdgeManager.currentTimeMillis();
302 splitLogSize = splitLogManager.splitLogDistributed(logDirs, META_FILTER);
303 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
304 if (this.metricsMaster != null) {
305 this.metricsMaster.addSplit(splitTime, splitLogSize);
306 }
307 }
308
309 private List<Path> getLogDirs(final List<ServerName> serverNames) throws IOException {
310 List<Path> logDirs = new ArrayList<Path>();
311 for (ServerName serverName: serverNames) {
312 Path logDir = new Path(this.rootdir, HLogUtil.getHLogDirectoryName(serverName.toString()));
313 Path splitDir = logDir.suffix(HLog.SPLITTING_EXT);
314
315 if (fs.exists(logDir)) {
316 if (!this.fs.rename(logDir, splitDir)) {
317 throw new IOException("Failed fs.rename for log split: " + logDir);
318 }
319 logDir = splitDir;
320 LOG.debug("Renamed region directory: " + splitDir);
321 } else if (!fs.exists(splitDir)) {
322 LOG.info("Log dir for server " + serverName + " does not exist");
323 continue;
324 }
325 logDirs.add(splitDir);
326 }
327 return logDirs;
328 }
329
330 public void splitLog(final List<ServerName> serverNames) throws IOException {
331 splitLog(serverNames, NON_META_FILTER);
332 }
333
334
335
336
337
338
339
340
341 public void splitLog(final List<ServerName> serverNames, PathFilter filter) throws IOException {
342 long splitTime = 0, splitLogSize = 0;
343 List<Path> logDirs = getLogDirs(serverNames);
344
345 if (logDirs.isEmpty()) {
346 LOG.info("No logs to split");
347 return;
348 }
349
350 if (distributedLogSplitting) {
351 splitLogManager.handleDeadWorkers(serverNames);
352 splitTime = EnvironmentEdgeManager.currentTimeMillis();
353 splitLogSize = splitLogManager.splitLogDistributed(logDirs,filter);
354 splitTime = EnvironmentEdgeManager.currentTimeMillis() - splitTime;
355 } else {
356 for(Path logDir: logDirs){
357
358
359 this.splitLogLock.lock();
360 try {
361 HLogSplitter splitter = HLogSplitter.createLogSplitter(
362 conf, rootdir, logDir, oldLogDir, this.fs);
363 try {
364
365 FSUtils.waitOnSafeMode(conf, conf.getInt(HConstants.THREAD_WAKE_FREQUENCY, 1000));
366 splitter.splitLog();
367 } catch (OrphanHLogAfterSplitException e) {
368 LOG.warn("Retrying splitting because of:", e);
369
370 splitter = HLogSplitter.createLogSplitter(conf, rootdir, logDir,
371 oldLogDir, this.fs);
372 splitter.splitLog();
373 }
374 splitTime = splitter.getTime();
375 splitLogSize = splitter.getSize();
376 } finally {
377 this.splitLogLock.unlock();
378 }
379 }
380 }
381
382 if (this.metricsMaster != null) {
383 this.metricsMaster.addSplit(splitTime, splitLogSize);
384 }
385 }
386
387
388
389
390
391
392
393
394
395
396 private Path checkRootDir(final Path rd, final Configuration c,
397 final FileSystem fs)
398 throws IOException {
399
400 FSUtils.waitOnSafeMode(c, c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
401
402 try {
403 if (!fs.exists(rd)) {
404 fs.mkdirs(rd);
405
406
407
408
409
410
411
412 FSUtils.setVersion(fs, rd, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
413 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
414 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
415 } else {
416 if (!fs.isDirectory(rd)) {
417 throw new IllegalArgumentException(rd.toString() + " is not a directory");
418 }
419
420 FSUtils.checkVersion(fs, rd, true, c.getInt(HConstants.THREAD_WAKE_FREQUENCY,
421 10 * 1000), c.getInt(HConstants.VERSION_FILE_WRITE_ATTEMPTS,
422 HConstants.DEFAULT_VERSION_FILE_WRITE_ATTEMPTS));
423 }
424 } catch (DeserializationException de) {
425 LOG.fatal("Please fix invalid configuration for " + HConstants.HBASE_DIR, de);
426 IOException ioe = new IOException();
427 ioe.initCause(de);
428 throw ioe;
429 } catch (IllegalArgumentException iae) {
430 LOG.fatal("Please fix invalid configuration for "
431 + HConstants.HBASE_DIR + " " + rd.toString(), iae);
432 throw iae;
433 }
434
435 if (!FSUtils.checkClusterIdExists(fs, rd, c.getInt(
436 HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000))) {
437 FSUtils.setClusterId(fs, rd, new ClusterId(), c.getInt(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000));
438 }
439 clusterId = FSUtils.getClusterId(fs, rd);
440
441
442 if (!FSUtils.metaRegionExists(fs, rd)) {
443 bootstrap(rd, c);
444 }
445
446
447 FSTableDescriptors.createTableDescriptor(fs, rd, HTableDescriptor.META_TABLEDESC, false);
448
449 return rd;
450 }
451
452
453
454
455
456 private void checkTempDir(final Path tmpdir, final Configuration c, final FileSystem fs)
457 throws IOException {
458
459 if (fs.exists(tmpdir)) {
460
461
462 for (Path tabledir: FSUtils.getTableDirs(fs, tmpdir)) {
463 for (Path regiondir: FSUtils.getRegionDirs(fs, tabledir)) {
464 HFileArchiver.archiveRegion(fs, this.rootdir, tabledir, regiondir);
465 }
466 }
467 if (!fs.delete(tmpdir, true)) {
468 throw new IOException("Unable to clean the temp directory: " + tmpdir);
469 }
470 }
471
472
473 if (!fs.mkdirs(tmpdir)) {
474 throw new IOException("HBase temp directory '" + tmpdir + "' creation failure.");
475 }
476 }
477
478 private static void bootstrap(final Path rd, final Configuration c)
479 throws IOException {
480 LOG.info("BOOTSTRAP: creating first META region");
481 try {
482
483
484
485
486 HRegionInfo metaHRI = new HRegionInfo(HRegionInfo.FIRST_META_REGIONINFO);
487 setInfoFamilyCachingForMeta(false);
488 HRegion meta = HRegion.createHRegion(metaHRI, rd, c,
489 HTableDescriptor.META_TABLEDESC);
490 setInfoFamilyCachingForMeta(true);
491 HRegion.closeHRegion(meta);
492 } catch (IOException e) {
493 e = RemoteExceptionHandler.checkIOException(e);
494 LOG.error("bootstrap", e);
495 throw e;
496 }
497 }
498
499
500
501
502 public static void setInfoFamilyCachingForMeta(final boolean b) {
503 for (HColumnDescriptor hcd:
504 HTableDescriptor.META_TABLEDESC.getColumnFamilies()) {
505 if (Bytes.equals(hcd.getName(), HConstants.CATALOG_FAMILY)) {
506 hcd.setBlockCacheEnabled(b);
507 hcd.setInMemory(b);
508 }
509 }
510 }
511
512
513 public void deleteRegion(HRegionInfo region) throws IOException {
514 HFileArchiver.archiveRegion(conf, fs, region);
515 }
516
517 public void deleteTable(byte[] tableName) throws IOException {
518 fs.delete(new Path(rootdir, Bytes.toString(tableName)), true);
519 }
520
521
522
523
524
525
526
527 public Path moveToTemp(final Path path) throws IOException {
528 Path tempPath = new Path(this.tempdir, path.getName());
529
530
531 if (!fs.exists(tempdir) && !fs.mkdirs(tempdir)) {
532 throw new IOException("HBase temp directory '" + tempdir + "' creation failure.");
533 }
534
535 if (!fs.rename(path, tempPath)) {
536 throw new IOException("Unable to move '" + path + "' to temp '" + tempPath + "'");
537 }
538
539 return tempPath;
540 }
541
542
543
544
545
546
547
548 public Path moveTableToTemp(byte[] tableName) throws IOException {
549 return moveToTemp(HTableDescriptor.getTableDir(this.rootdir, tableName));
550 }
551
552 public void updateRegionInfo(HRegionInfo region) {
553
554
555
556 }
557
558 public void deleteFamilyFromFS(HRegionInfo region, byte[] familyName)
559 throws IOException {
560
561 Path tableDir = new Path(rootdir, region.getTableNameAsString());
562 HFileArchiver.archiveFamily(fs, conf, region, tableDir, familyName);
563
564
565 Path familyDir = new Path(tableDir,
566 new Path(region.getEncodedName(), Bytes.toString(familyName)));
567 if (fs.delete(familyDir, true) == false) {
568 throw new IOException("Could not delete family "
569 + Bytes.toString(familyName) + " from FileSystem for region "
570 + region.getRegionNameAsString() + "(" + region.getEncodedName()
571 + ")");
572 }
573 }
574
575 public void stop() {
576 if (splitLogManager != null) {
577 this.splitLogManager.stop();
578 }
579 }
580
581
582
583
584
585
586 public void createTableDescriptor(HTableDescriptor htableDescriptor)
587 throws IOException {
588 FSTableDescriptors.createTableDescriptor(htableDescriptor, conf);
589 }
590
591
592
593
594
595
596
597
598 public HTableDescriptor deleteColumn(byte[] tableName, byte[] familyName)
599 throws IOException {
600 LOG.info("DeleteColumn. Table = " + Bytes.toString(tableName)
601 + " family = " + Bytes.toString(familyName));
602 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
603 htd.removeFamily(familyName);
604 this.services.getTableDescriptors().add(htd);
605 return htd;
606 }
607
608
609
610
611
612
613
614
615 public HTableDescriptor modifyColumn(byte[] tableName, HColumnDescriptor hcd)
616 throws IOException {
617 LOG.info("AddModifyColumn. Table = " + Bytes.toString(tableName)
618 + " HCD = " + hcd.toString());
619
620 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
621 byte [] familyName = hcd.getName();
622 if(!htd.hasFamily(familyName)) {
623 throw new InvalidFamilyOperationException("Family '" +
624 Bytes.toString(familyName) + "' doesn't exists so cannot be modified");
625 }
626 htd.addFamily(hcd);
627 this.services.getTableDescriptors().add(htd);
628 return htd;
629 }
630
631
632
633
634
635
636
637
638 public HTableDescriptor addColumn(byte[] tableName, HColumnDescriptor hcd)
639 throws IOException {
640 LOG.info("AddColumn. Table = " + Bytes.toString(tableName) + " HCD = " +
641 hcd.toString());
642 HTableDescriptor htd = this.services.getTableDescriptors().get(tableName);
643 if (htd == null) {
644 throw new InvalidFamilyOperationException("Family '" +
645 hcd.getNameAsString() + "' cannot be modified as HTD is null");
646 }
647 htd.addFamily(hcd);
648 this.services.getTableDescriptors().add(htd);
649 return htd;
650 }
651 }