1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.util;
20
21 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertErrors;
22 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.assertNoErrors;
23 import static org.apache.hadoop.hbase.util.hbck.HbckTestingUtil.doFsck;
24 import static org.junit.Assert.assertEquals;
25 import static org.junit.Assert.assertFalse;
26 import static org.junit.Assert.assertNotEquals;
27 import static org.junit.Assert.assertNotNull;
28 import static org.junit.Assert.assertTrue;
29 import static org.junit.Assert.fail;
30
31 import java.io.IOException;
32 import java.util.ArrayList;
33 import java.util.Collection;
34 import java.util.HashMap;
35 import java.util.LinkedList;
36 import java.util.List;
37 import java.util.Map;
38 import java.util.Map.Entry;
39 import java.util.concurrent.CountDownLatch;
40 import java.util.concurrent.ExecutorService;
41 import java.util.concurrent.ScheduledThreadPoolExecutor;
42 import java.util.concurrent.SynchronousQueue;
43 import java.util.concurrent.ThreadPoolExecutor;
44 import java.util.concurrent.TimeUnit;
45
46 import org.apache.commons.io.IOUtils;
47 import org.apache.commons.logging.Log;
48 import org.apache.commons.logging.LogFactory;
49 import org.apache.hadoop.conf.Configuration;
50 import org.apache.hadoop.fs.FileStatus;
51 import org.apache.hadoop.fs.FileSystem;
52 import org.apache.hadoop.fs.Path;
53 import org.apache.hadoop.hbase.ClusterStatus;
54 import org.apache.hadoop.hbase.HBaseTestingUtility;
55 import org.apache.hadoop.hbase.HColumnDescriptor;
56 import org.apache.hadoop.hbase.HConstants;
57 import org.apache.hadoop.hbase.HRegionInfo;
58 import org.apache.hadoop.hbase.HRegionLocation;
59 import org.apache.hadoop.hbase.HTableDescriptor;
60 import org.apache.hadoop.hbase.LargeTests;
61 import org.apache.hadoop.hbase.MiniHBaseCluster;
62 import org.apache.hadoop.hbase.ServerName;
63 import org.apache.hadoop.hbase.TableName;
64 import org.apache.hadoop.hbase.catalog.MetaEditor;
65 import org.apache.hadoop.hbase.client.Delete;
66 import org.apache.hadoop.hbase.client.Durability;
67 import org.apache.hadoop.hbase.client.Get;
68 import org.apache.hadoop.hbase.client.HBaseAdmin;
69 import org.apache.hadoop.hbase.client.HConnection;
70 import org.apache.hadoop.hbase.client.HConnectionManager;
71 import org.apache.hadoop.hbase.client.HTable;
72 import org.apache.hadoop.hbase.client.MetaScanner;
73 import org.apache.hadoop.hbase.client.Put;
74 import org.apache.hadoop.hbase.client.Result;
75 import org.apache.hadoop.hbase.client.ResultScanner;
76 import org.apache.hadoop.hbase.client.Scan;
77 import org.apache.hadoop.hbase.io.hfile.TestHFile;
78 import org.apache.hadoop.hbase.master.AssignmentManager;
79 import org.apache.hadoop.hbase.master.HMaster;
80 import org.apache.hadoop.hbase.master.RegionStates;
81 import org.apache.hadoop.hbase.master.TableLockManager;
82 import org.apache.hadoop.hbase.master.TableLockManager.TableLock;
83 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
84 import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
85 import org.apache.hadoop.hbase.regionserver.HRegion;
86 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
87 import org.apache.hadoop.hbase.regionserver.HRegionServer;
88 import org.apache.hadoop.hbase.regionserver.TestEndToEndSplitTransaction;
89 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter;
90 import org.apache.hadoop.hbase.util.HBaseFsck.ErrorReporter.ERROR_CODE;
91 import org.apache.hadoop.hbase.util.HBaseFsck.HbckInfo;
92 import org.apache.hadoop.hbase.util.HBaseFsck.PrintingErrorReporter;
93 import org.apache.hadoop.hbase.util.HBaseFsck.TableInfo;
94 import org.apache.hadoop.hbase.util.hbck.HFileCorruptionChecker;
95 import org.apache.hadoop.hbase.util.hbck.HbckTestingUtil;
96 import org.apache.hadoop.hbase.zookeeper.MetaRegionTracker;
97 import org.apache.zookeeper.KeeperException;
98 import org.junit.AfterClass;
99 import org.junit.BeforeClass;
100 import org.junit.Ignore;
101 import org.junit.Test;
102 import org.junit.experimental.categories.Category;
103 import org.junit.rules.TestName;
104
105 import com.google.common.collect.Multimap;
106
107
108
109
110 @Category(LargeTests.class)
111 public class TestHBaseFsck {
112 final static Log LOG = LogFactory.getLog(TestHBaseFsck.class);
113 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
114 private final static Configuration conf = TEST_UTIL.getConfiguration();
115 private final static String FAM_STR = "fam";
116 private final static byte[] FAM = Bytes.toBytes(FAM_STR);
117 private final static int REGION_ONLINE_TIMEOUT = 800;
118 private static RegionStates regionStates;
119 private static ExecutorService executorService;
120
121
122 private HTable tbl;
123 private final static byte[][] SPLITS = new byte[][] { Bytes.toBytes("A"),
124 Bytes.toBytes("B"), Bytes.toBytes("C") };
125
126 private final static byte[][] ROWKEYS= new byte[][] {
127 Bytes.toBytes("00"), Bytes.toBytes("50"), Bytes.toBytes("A0"), Bytes.toBytes("A5"),
128 Bytes.toBytes("B0"), Bytes.toBytes("B5"), Bytes.toBytes("C0"), Bytes.toBytes("C5") };
129
130 @BeforeClass
131 public static void setUpBeforeClass() throws Exception {
132 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.handler.count", 2);
133 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.metahandler.count", 2);
134 TEST_UTIL.startMiniCluster(3);
135
136 executorService = new ThreadPoolExecutor(1, Integer.MAX_VALUE, 60, TimeUnit.SECONDS,
137 new SynchronousQueue<Runnable>(), Threads.newDaemonThreadFactory("testhbck"));
138
139 AssignmentManager assignmentManager =
140 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager();
141 regionStates = assignmentManager.getRegionStates();
142 TEST_UTIL.getHBaseAdmin().setBalancerRunning(false, true);
143 }
144
145 @AfterClass
146 public static void tearDownAfterClass() throws Exception {
147 TEST_UTIL.shutdownMiniCluster();
148 }
149
150 @Test
151 public void testHBaseFsck() throws Exception {
152 assertNoErrors(doFsck(conf, false));
153 String table = "tableBadMetaAssign";
154 TEST_UTIL.createTable(Bytes.toBytes(table), FAM);
155
156
157 assertNoErrors(doFsck(conf, false));
158
159
160
161 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName(),
162 executorService);
163 Scan scan = new Scan();
164 scan.setStartRow(Bytes.toBytes(table+",,"));
165 ResultScanner scanner = meta.getScanner(scan);
166 HRegionInfo hri = null;
167
168 Result res = scanner.next();
169 ServerName currServer =
170 ServerName.parseFrom(res.getValue(HConstants.CATALOG_FAMILY,
171 HConstants.SERVER_QUALIFIER));
172 long startCode = Bytes.toLong(res.getValue(HConstants.CATALOG_FAMILY,
173 HConstants.STARTCODE_QUALIFIER));
174
175 for (JVMClusterUtil.RegionServerThread rs :
176 TEST_UTIL.getHBaseCluster().getRegionServerThreads()) {
177
178 ServerName sn = rs.getRegionServer().getServerName();
179
180
181 if (!currServer.getHostAndPort().equals(sn.getHostAndPort()) ||
182 startCode != sn.getStartcode()) {
183 Put put = new Put(res.getRow());
184 put.setDurability(Durability.SKIP_WAL);
185 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
186 Bytes.toBytes(sn.getHostAndPort()));
187 put.add(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER,
188 Bytes.toBytes(sn.getStartcode()));
189 meta.put(put);
190 hri = HRegionInfo.getHRegionInfo(res);
191 break;
192 }
193 }
194
195
196 assertErrors(doFsck(conf, true), new ERROR_CODE[]{
197 ERROR_CODE.SERVER_DOES_NOT_MATCH_META});
198
199 TEST_UTIL.getHBaseCluster().getMaster()
200 .getAssignmentManager().waitForAssignment(hri);
201
202
203 assertNoErrors(doFsck(conf, false));
204
205
206 HTable t = new HTable(conf, Bytes.toBytes(table), executorService);
207 ResultScanner s = t.getScanner(new Scan());
208 s.close();
209 t.close();
210
211 scanner.close();
212 meta.close();
213 }
214
215 @Test(timeout=180000)
216 public void testFixAssignmentsWhenMETAinTransition() throws Exception {
217 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
218 HBaseAdmin admin = null;
219 try {
220 admin = new HBaseAdmin(TEST_UTIL.getConfiguration());
221 admin.closeRegion(cluster.getServerHoldingMeta(),
222 HRegionInfo.FIRST_META_REGIONINFO);
223 } finally {
224 if (admin != null) {
225 admin.close();
226 }
227 }
228 regionStates.regionOffline(HRegionInfo.FIRST_META_REGIONINFO);
229 MetaRegionTracker.deleteMetaLocation(cluster.getMaster().getZooKeeper());
230 assertFalse(regionStates.isRegionOnline(HRegionInfo.FIRST_META_REGIONINFO));
231 HBaseFsck hbck = doFsck(conf, true);
232 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.UNKNOWN, ERROR_CODE.NO_META_REGION,
233 ERROR_CODE.NULL_META_REGION });
234 assertNoErrors(doFsck(conf, false));
235 }
236
237
238
239
240 private HRegionInfo createRegion(Configuration conf, final HTableDescriptor
241 htd, byte[] startKey, byte[] endKey)
242 throws IOException {
243 HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);
244 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
245 MetaEditor.addRegionToMeta(meta, hri);
246 meta.close();
247 return hri;
248 }
249
250
251
252
253 private void dumpMeta(TableName tableName) throws IOException {
254 List<byte[]> metaRows = TEST_UTIL.getMetaTableRows(tableName);
255 for (byte[] row : metaRows) {
256 LOG.info(Bytes.toString(row));
257 }
258 }
259
260
261
262
263
264 private void undeployRegion(HBaseAdmin admin, ServerName sn,
265 HRegionInfo hri) throws IOException, InterruptedException {
266 try {
267 HBaseFsckRepair.closeRegionSilentlyAndWait(admin, sn, hri);
268 if (!hri.isMetaTable()) {
269 admin.offline(hri.getRegionName());
270 }
271 } catch (IOException ioe) {
272 LOG.warn("Got exception when attempting to offline region "
273 + Bytes.toString(hri.getRegionName()), ioe);
274 }
275 }
276
277
278
279
280
281
282 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
283 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
284 boolean hdfs) throws IOException, InterruptedException {
285 deleteRegion(conf, htd, startKey, endKey, unassign, metaRow, hdfs, false);
286 }
287
288
289
290
291
292
293
294
295 private void deleteRegion(Configuration conf, final HTableDescriptor htd,
296 byte[] startKey, byte[] endKey, boolean unassign, boolean metaRow,
297 boolean hdfs, boolean regionInfoOnly) throws IOException, InterruptedException {
298 LOG.info("** Before delete:");
299 dumpMeta(htd.getTableName());
300
301 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
302 for (Entry<HRegionInfo, ServerName> e: hris.entrySet()) {
303 HRegionInfo hri = e.getKey();
304 ServerName hsa = e.getValue();
305 if (Bytes.compareTo(hri.getStartKey(), startKey) == 0
306 && Bytes.compareTo(hri.getEndKey(), endKey) == 0) {
307
308 LOG.info("RegionName: " +hri.getRegionNameAsString());
309 byte[] deleteRow = hri.getRegionName();
310
311 if (unassign) {
312 LOG.info("Undeploying region " + hri + " from server " + hsa);
313 undeployRegion(new HBaseAdmin(conf), hsa, hri);
314 }
315
316 if (regionInfoOnly) {
317 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
318 Path rootDir = FSUtils.getRootDir(conf);
319 FileSystem fs = rootDir.getFileSystem(conf);
320 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
321 hri.getEncodedName());
322 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
323 fs.delete(hriPath, true);
324 }
325
326 if (hdfs) {
327 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
328 Path rootDir = FSUtils.getRootDir(conf);
329 FileSystem fs = rootDir.getFileSystem(conf);
330 Path p = new Path(FSUtils.getTableDir(rootDir, htd.getTableName()),
331 hri.getEncodedName());
332 HBaseFsck.debugLsr(conf, p);
333 boolean success = fs.delete(p, true);
334 LOG.info("Deleted " + p + " sucessfully? " + success);
335 HBaseFsck.debugLsr(conf, p);
336 }
337
338 if (metaRow) {
339 HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);
340 Delete delete = new Delete(deleteRow);
341 meta.delete(delete);
342 }
343 }
344 LOG.info(hri.toString() + hsa.toString());
345 }
346
347 TEST_UTIL.getMetaTableRows(htd.getTableName());
348 LOG.info("*** After delete:");
349 dumpMeta(htd.getTableName());
350 }
351
352
353
354
355
356
357
358
359 HTable setupTable(TableName tablename) throws Exception {
360 HTableDescriptor desc = new HTableDescriptor(tablename);
361 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
362 desc.addFamily(hcd);
363 TEST_UTIL.getHBaseAdmin().createTable(desc, SPLITS);
364 tbl = new HTable(TEST_UTIL.getConfiguration(), tablename, executorService);
365
366 List<Put> puts = new ArrayList<Put>();
367 for (byte[] row : ROWKEYS) {
368 Put p = new Put(row);
369 p.add(FAM, Bytes.toBytes("val"), row);
370 puts.add(p);
371 }
372 tbl.put(puts);
373 tbl.flushCommits();
374 return tbl;
375 }
376
377
378
379
380 int countRows() throws IOException {
381 Scan s = new Scan();
382 ResultScanner rs = tbl.getScanner(s);
383 int i = 0;
384 while(rs.next() !=null) {
385 i++;
386 }
387 return i;
388 }
389
390
391
392
393
394
395
396 void deleteTable(TableName tablename) throws IOException {
397 HBaseAdmin admin = new HBaseAdmin(conf);
398 admin.getConnection().clearRegionCache();
399 if (admin.isTableEnabled(tablename)) {
400 admin.disableTableAsync(tablename);
401 }
402 long totalWait = 0;
403 long maxWait = 30*1000;
404 long sleepTime = 250;
405 while (!admin.isTableDisabled(tablename)) {
406 try {
407 Thread.sleep(sleepTime);
408 totalWait += sleepTime;
409 if (totalWait >= maxWait) {
410 fail("Waited too long for table to be disabled + " + tablename);
411 }
412 } catch (InterruptedException e) {
413 e.printStackTrace();
414 fail("Interrupted when trying to disable table " + tablename);
415 }
416 }
417 admin.deleteTable(tablename);
418 }
419
420
421
422
423 @Test
424 public void testHBaseFsckClean() throws Exception {
425 assertNoErrors(doFsck(conf, false));
426 TableName table = TableName.valueOf("tableClean");
427 try {
428 HBaseFsck hbck = doFsck(conf, false);
429 assertNoErrors(hbck);
430
431 setupTable(table);
432 assertEquals(ROWKEYS.length, countRows());
433
434
435 hbck = doFsck(conf, false);
436 assertNoErrors(hbck);
437 assertEquals(0, hbck.getOverlapGroups(table).size());
438 assertEquals(ROWKEYS.length, countRows());
439 } finally {
440 deleteTable(table);
441 }
442 }
443
444
445
446
447 @Test
448 public void testHbckThreadpooling() throws Exception {
449 TableName table =
450 TableName.valueOf("tableDupeStartKey");
451 try {
452
453 setupTable(table);
454
455
456 Configuration newconf = new Configuration(conf);
457 newconf.setInt("hbasefsck.numthreads", 1);
458 assertNoErrors(doFsck(newconf, false));
459
460
461 } finally {
462 deleteTable(table);
463 }
464 }
465
466 @Test
467 public void testHbckFixOrphanTable() throws Exception {
468 TableName table = TableName.valueOf("tableInfo");
469 FileSystem fs = null;
470 Path tableinfo = null;
471 try {
472 setupTable(table);
473 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
474
475 Path hbaseTableDir = FSUtils.getTableDir(
476 FSUtils.getRootDir(conf), table);
477 fs = hbaseTableDir.getFileSystem(conf);
478 FileStatus status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
479 tableinfo = status.getPath();
480 fs.rename(tableinfo, new Path("/.tableinfo"));
481
482
483 HBaseFsck hbck = doFsck(conf, false);
484 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_TABLEINFO_FILE });
485
486
487 hbck = doFsck(conf, true);
488 assertNoErrors(hbck);
489 status = null;
490 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
491 assertNotNull(status);
492
493 HTableDescriptor htd = admin.getTableDescriptor(table);
494 htd.setValue("NOT_DEFAULT", "true");
495 admin.disableTable(table);
496 admin.modifyTable(table, htd);
497 admin.enableTable(table);
498 fs.delete(status.getPath(), true);
499
500
501 htd = admin.getTableDescriptor(table);
502 hbck = doFsck(conf, true);
503 assertNoErrors(hbck);
504 status = null;
505 status = FSTableDescriptors.getTableInfoPath(fs, hbaseTableDir);
506 assertNotNull(status);
507 htd = admin.getTableDescriptor(table);
508 assertEquals(htd.getValue("NOT_DEFAULT"), "true");
509 } finally {
510 fs.rename(new Path("/.tableinfo"), tableinfo);
511 deleteTable(table);
512 }
513 }
514
515
516
517
518
519 @Test
520 public void testDupeStartKey() throws Exception {
521 TableName table =
522 TableName.valueOf("tableDupeStartKey");
523 try {
524 setupTable(table);
525 assertNoErrors(doFsck(conf, false));
526 assertEquals(ROWKEYS.length, countRows());
527
528
529 HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
530 Bytes.toBytes("A"), Bytes.toBytes("A2"));
531 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
532 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
533 .waitForAssignment(hriDupe);
534 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
535 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
536
537 HBaseFsck hbck = doFsck(conf, false);
538 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
539 ERROR_CODE.DUPE_STARTKEYS});
540 assertEquals(2, hbck.getOverlapGroups(table).size());
541 assertEquals(ROWKEYS.length, countRows());
542
543
544 doFsck(conf,true);
545
546
547 HBaseFsck hbck2 = doFsck(conf,false);
548 assertNoErrors(hbck2);
549 assertEquals(0, hbck2.getOverlapGroups(table).size());
550 assertEquals(ROWKEYS.length, countRows());
551 } finally {
552 deleteTable(table);
553 }
554 }
555
556
557
558
559 Map<ServerName, List<String>> getDeployedHRIs(
560 final HBaseAdmin admin) throws IOException {
561 ClusterStatus status = admin.getClusterStatus();
562 Collection<ServerName> regionServers = status.getServers();
563 Map<ServerName, List<String>> mm =
564 new HashMap<ServerName, List<String>>();
565 HConnection connection = admin.getConnection();
566 for (ServerName hsi : regionServers) {
567 AdminProtos.AdminService.BlockingInterface server = connection.getAdmin(hsi);
568
569
570 List<HRegionInfo> regions = ProtobufUtil.getOnlineRegions(server);
571 List<String> regionNames = new ArrayList<String>();
572 for (HRegionInfo hri : regions) {
573 regionNames.add(hri.getRegionNameAsString());
574 }
575 mm.put(hsi, regionNames);
576 }
577 return mm;
578 }
579
580
581
582
583 ServerName findDeployedHSI(Map<ServerName, List<String>> mm, HRegionInfo hri) {
584 for (Map.Entry<ServerName,List <String>> e : mm.entrySet()) {
585 if (e.getValue().contains(hri.getRegionNameAsString())) {
586 return e.getKey();
587 }
588 }
589 return null;
590 }
591
592
593
594
595
596 @Test
597 public void testDupeRegion() throws Exception {
598 TableName table =
599 TableName.valueOf("tableDupeRegion");
600 try {
601 setupTable(table);
602 assertNoErrors(doFsck(conf, false));
603 assertEquals(ROWKEYS.length, countRows());
604
605
606 HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
607 Bytes.toBytes("A"), Bytes.toBytes("B"));
608
609 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
610 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
611 .waitForAssignment(hriDupe);
612 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
613 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
614
615
616
617
618
619 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
620 while (findDeployedHSI(getDeployedHRIs(admin), hriDupe) == null) {
621 Thread.sleep(250);
622 }
623
624 LOG.debug("Finished assignment of dupe region");
625
626
627 HBaseFsck hbck = doFsck(conf, false);
628 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DUPE_STARTKEYS,
629 ERROR_CODE.DUPE_STARTKEYS});
630 assertEquals(2, hbck.getOverlapGroups(table).size());
631 assertEquals(ROWKEYS.length, countRows());
632
633
634 doFsck(conf,true);
635
636
637 HBaseFsck hbck2 = doFsck(conf,false);
638 assertNoErrors(hbck2);
639 assertEquals(0, hbck2.getOverlapGroups(table).size());
640 assertEquals(ROWKEYS.length, countRows());
641 } finally {
642 deleteTable(table);
643 }
644 }
645
646
647
648
649 @Test
650 public void testDegenerateRegions() throws Exception {
651 TableName table =
652 TableName.valueOf("tableDegenerateRegions");
653 try {
654 setupTable(table);
655 assertNoErrors(doFsck(conf,false));
656 assertEquals(ROWKEYS.length, countRows());
657
658
659 HRegionInfo hriDupe = createRegion(conf, tbl.getTableDescriptor(),
660 Bytes.toBytes("B"), Bytes.toBytes("B"));
661 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriDupe);
662 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
663 .waitForAssignment(hriDupe);
664 ServerName server = regionStates.getRegionServerOfRegion(hriDupe);
665 TEST_UTIL.assertRegionOnServer(hriDupe, server, REGION_ONLINE_TIMEOUT);
666
667 HBaseFsck hbck = doFsck(conf,false);
668 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.DEGENERATE_REGION,
669 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.DUPE_STARTKEYS});
670 assertEquals(2, hbck.getOverlapGroups(table).size());
671 assertEquals(ROWKEYS.length, countRows());
672
673
674 doFsck(conf,true);
675
676
677 HBaseFsck hbck2 = doFsck(conf,false);
678 assertNoErrors(hbck2);
679 assertEquals(0, hbck2.getOverlapGroups(table).size());
680 assertEquals(ROWKEYS.length, countRows());
681 } finally {
682 deleteTable(table);
683 }
684 }
685
686
687
688
689
690 @Test
691 public void testContainedRegionOverlap() throws Exception {
692 TableName table =
693 TableName.valueOf("tableContainedRegionOverlap");
694 try {
695 setupTable(table);
696 assertEquals(ROWKEYS.length, countRows());
697
698
699 HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
700 Bytes.toBytes("A2"), Bytes.toBytes("B"));
701 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
702 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
703 .waitForAssignment(hriOverlap);
704 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
705 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
706
707 HBaseFsck hbck = doFsck(conf, false);
708 assertErrors(hbck, new ERROR_CODE[] {
709 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
710 assertEquals(2, hbck.getOverlapGroups(table).size());
711 assertEquals(ROWKEYS.length, countRows());
712
713
714 doFsck(conf, true);
715
716
717 HBaseFsck hbck2 = doFsck(conf,false);
718 assertNoErrors(hbck2);
719 assertEquals(0, hbck2.getOverlapGroups(table).size());
720 assertEquals(ROWKEYS.length, countRows());
721 } finally {
722 deleteTable(table);
723 }
724 }
725
726
727
728
729
730
731
732 @Test
733 public void testSidelineOverlapRegion() throws Exception {
734 TableName table =
735 TableName.valueOf("testSidelineOverlapRegion");
736 try {
737 setupTable(table);
738 assertEquals(ROWKEYS.length, countRows());
739
740
741 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
742 HMaster master = cluster.getMaster();
743 HRegionInfo hriOverlap1 = createRegion(conf, tbl.getTableDescriptor(),
744 Bytes.toBytes("A"), Bytes.toBytes("AB"));
745 master.assignRegion(hriOverlap1);
746 master.getAssignmentManager().waitForAssignment(hriOverlap1);
747 HRegionInfo hriOverlap2 = createRegion(conf, tbl.getTableDescriptor(),
748 Bytes.toBytes("AB"), Bytes.toBytes("B"));
749 master.assignRegion(hriOverlap2);
750 master.getAssignmentManager().waitForAssignment(hriOverlap2);
751
752 HBaseFsck hbck = doFsck(conf, false);
753 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.DUPE_STARTKEYS,
754 ERROR_CODE.DUPE_STARTKEYS, ERROR_CODE.OVERLAP_IN_REGION_CHAIN});
755 assertEquals(3, hbck.getOverlapGroups(table).size());
756 assertEquals(ROWKEYS.length, countRows());
757
758
759 Multimap<byte[], HbckInfo> overlapGroups = hbck.getOverlapGroups(table);
760 ServerName serverName = null;
761 byte[] regionName = null;
762 for (HbckInfo hbi: overlapGroups.values()) {
763 if ("A".equals(Bytes.toString(hbi.getStartKey()))
764 && "B".equals(Bytes.toString(hbi.getEndKey()))) {
765 regionName = hbi.getRegionName();
766
767
768 int k = cluster.getServerWith(regionName);
769 for (int i = 0; i < 3; i++) {
770 if (i != k) {
771 HRegionServer rs = cluster.getRegionServer(i);
772 serverName = rs.getServerName();
773 break;
774 }
775 }
776
777 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
778 HBaseFsckRepair.closeRegionSilentlyAndWait(admin,
779 cluster.getRegionServer(k).getServerName(), hbi.getHdfsHRI());
780 admin.offline(regionName);
781 break;
782 }
783 }
784
785 assertNotNull(regionName);
786 assertNotNull(serverName);
787 HTable meta = new HTable(conf, TableName.META_TABLE_NAME, executorService);
788 Put put = new Put(regionName);
789 put.add(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER,
790 Bytes.toBytes(serverName.getHostAndPort()));
791 meta.put(put);
792
793
794 HBaseFsck fsck = new HBaseFsck(conf);
795 fsck.connect();
796 fsck.setDisplayFullReport();
797 fsck.setTimeLag(0);
798 fsck.setFixAssignments(true);
799 fsck.setFixMeta(true);
800 fsck.setFixHdfsHoles(true);
801 fsck.setFixHdfsOverlaps(true);
802 fsck.setFixHdfsOrphans(true);
803 fsck.setFixVersionFile(true);
804 fsck.setSidelineBigOverlaps(true);
805 fsck.setMaxMerge(2);
806 fsck.onlineHbck();
807
808
809
810 HBaseFsck hbck2 = doFsck(conf,false);
811 assertNoErrors(hbck2);
812 assertEquals(0, hbck2.getOverlapGroups(table).size());
813 assertTrue(ROWKEYS.length > countRows());
814 } finally {
815 deleteTable(table);
816 }
817 }
818
819
820
821
822
823 @Test
824 public void testOverlapAndOrphan() throws Exception {
825 TableName table =
826 TableName.valueOf("tableOverlapAndOrphan");
827 try {
828 setupTable(table);
829 assertEquals(ROWKEYS.length, countRows());
830
831
832 TEST_UTIL.getHBaseAdmin().disableTable(table);
833 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
834 Bytes.toBytes("B"), true, true, false, true);
835 TEST_UTIL.getHBaseAdmin().enableTable(table);
836
837 HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
838 Bytes.toBytes("A2"), Bytes.toBytes("B"));
839 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
840 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
841 .waitForAssignment(hriOverlap);
842 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
843 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
844
845 HBaseFsck hbck = doFsck(conf, false);
846 assertErrors(hbck, new ERROR_CODE[] {
847 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
848 ERROR_CODE.HOLE_IN_REGION_CHAIN});
849
850
851 doFsck(conf, true);
852
853
854 HBaseFsck hbck2 = doFsck(conf,false);
855 assertNoErrors(hbck2);
856 assertEquals(0, hbck2.getOverlapGroups(table).size());
857 assertEquals(ROWKEYS.length, countRows());
858 } finally {
859 deleteTable(table);
860 }
861 }
862
863
864
865
866
867
868 @Test
869 public void testCoveredStartKey() throws Exception {
870 TableName table =
871 TableName.valueOf("tableCoveredStartKey");
872 try {
873 setupTable(table);
874 assertEquals(ROWKEYS.length, countRows());
875
876
877 HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
878 Bytes.toBytes("A2"), Bytes.toBytes("B2"));
879 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
880 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
881 .waitForAssignment(hriOverlap);
882 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
883 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
884
885 HBaseFsck hbck = doFsck(conf, false);
886 assertErrors(hbck, new ERROR_CODE[] {
887 ERROR_CODE.OVERLAP_IN_REGION_CHAIN,
888 ERROR_CODE.OVERLAP_IN_REGION_CHAIN });
889 assertEquals(3, hbck.getOverlapGroups(table).size());
890 assertEquals(ROWKEYS.length, countRows());
891
892
893 doFsck(conf, true);
894
895
896 HBaseFsck hbck2 = doFsck(conf, false);
897 assertErrors(hbck2, new ERROR_CODE[0]);
898 assertEquals(0, hbck2.getOverlapGroups(table).size());
899 assertEquals(ROWKEYS.length, countRows());
900 } finally {
901 deleteTable(table);
902 }
903 }
904
905
906
907
908
909 @Test
910 public void testRegionHole() throws Exception {
911 TableName table =
912 TableName.valueOf("tableRegionHole");
913 try {
914 setupTable(table);
915 assertEquals(ROWKEYS.length, countRows());
916
917
918 TEST_UTIL.getHBaseAdmin().disableTable(table);
919 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
920 Bytes.toBytes("C"), true, true, true);
921 TEST_UTIL.getHBaseAdmin().enableTable(table);
922
923 HBaseFsck hbck = doFsck(conf, false);
924 assertErrors(hbck, new ERROR_CODE[] {
925 ERROR_CODE.HOLE_IN_REGION_CHAIN});
926
927 assertEquals(0, hbck.getOverlapGroups(table).size());
928
929
930 doFsck(conf, true);
931
932
933 assertNoErrors(doFsck(conf,false));
934 assertEquals(ROWKEYS.length - 2 , countRows());
935 } finally {
936 deleteTable(table);
937 }
938 }
939
940
941
942
943
944 @Test
945 public void testHDFSRegioninfoMissing() throws Exception {
946 TableName table =
947 TableName.valueOf("tableHDFSRegioininfoMissing");
948 try {
949 setupTable(table);
950 assertEquals(ROWKEYS.length, countRows());
951
952
953 TEST_UTIL.getHBaseAdmin().disableTable(table);
954 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
955 Bytes.toBytes("C"), true, true, false, true);
956 TEST_UTIL.getHBaseAdmin().enableTable(table);
957
958 HBaseFsck hbck = doFsck(conf, false);
959 assertErrors(hbck, new ERROR_CODE[] {
960 ERROR_CODE.ORPHAN_HDFS_REGION,
961 ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
962 ERROR_CODE.HOLE_IN_REGION_CHAIN});
963
964 assertEquals(0, hbck.getOverlapGroups(table).size());
965
966
967 doFsck(conf, true);
968
969
970 assertNoErrors(doFsck(conf, false));
971 assertEquals(ROWKEYS.length, countRows());
972 } finally {
973 deleteTable(table);
974 }
975 }
976
977
978
979
980
981 @Test
982 public void testNotInMetaOrDeployedHole() throws Exception {
983 TableName table =
984 TableName.valueOf("tableNotInMetaOrDeployedHole");
985 try {
986 setupTable(table);
987 assertEquals(ROWKEYS.length, countRows());
988
989
990 TEST_UTIL.getHBaseAdmin().disableTable(table);
991 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
992 Bytes.toBytes("C"), true, true, false);
993 TEST_UTIL.getHBaseAdmin().enableTable(table);
994
995 HBaseFsck hbck = doFsck(conf, false);
996 assertErrors(hbck, new ERROR_CODE[] {
997 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
998
999 assertEquals(0, hbck.getOverlapGroups(table).size());
1000
1001
1002 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1003 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1004
1005
1006 assertNoErrors(doFsck(conf,false));
1007 assertEquals(ROWKEYS.length, countRows());
1008 } finally {
1009 deleteTable(table);
1010 }
1011 }
1012
1013
1014
1015
1016 @Test
1017 public void testNotInMetaHole() throws Exception {
1018 TableName table =
1019 TableName.valueOf("tableNotInMetaHole");
1020 try {
1021 setupTable(table);
1022 assertEquals(ROWKEYS.length, countRows());
1023
1024
1025 TEST_UTIL.getHBaseAdmin().disableTable(table);
1026 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1027 Bytes.toBytes("C"), false, true, false);
1028 TEST_UTIL.getHBaseAdmin().enableTable(table);
1029
1030 HBaseFsck hbck = doFsck(conf, false);
1031 assertErrors(hbck, new ERROR_CODE[] {
1032 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1033
1034 assertEquals(0, hbck.getOverlapGroups(table).size());
1035
1036
1037 assertErrors(doFsck(conf, true) , new ERROR_CODE[] {
1038 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1039
1040
1041 assertNoErrors(doFsck(conf,false));
1042 assertEquals(ROWKEYS.length, countRows());
1043 } finally {
1044 deleteTable(table);
1045 }
1046 }
1047
1048
1049
1050
1051
1052 @Test
1053 public void testNotInHdfs() throws Exception {
1054 TableName table =
1055 TableName.valueOf("tableNotInHdfs");
1056 try {
1057 setupTable(table);
1058 assertEquals(ROWKEYS.length, countRows());
1059
1060
1061 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1062
1063
1064 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1065 Bytes.toBytes("C"), false, false, true);
1066
1067 HBaseFsck hbck = doFsck(conf, false);
1068 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1069
1070 assertEquals(0, hbck.getOverlapGroups(table).size());
1071
1072
1073 doFsck(conf, true);
1074
1075
1076 assertNoErrors(doFsck(conf,false));
1077 assertEquals(ROWKEYS.length - 2, countRows());
1078 } finally {
1079 deleteTable(table);
1080 }
1081 }
1082
1083
1084
1085
1086
1087 @Test
1088 public void testNoHdfsTable() throws Exception {
1089 TableName table = TableName.valueOf("NoHdfsTable");
1090 setupTable(table);
1091 assertEquals(ROWKEYS.length, countRows());
1092
1093
1094 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1095
1096
1097 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""),
1098 Bytes.toBytes("A"), false, false, true);
1099 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1100 Bytes.toBytes("B"), false, false, true);
1101 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1102 Bytes.toBytes("C"), false, false, true);
1103 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"),
1104 Bytes.toBytes(""), false, false, true);
1105
1106
1107 deleteTableDir(table);
1108
1109 HBaseFsck hbck = doFsck(conf, false);
1110 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS,
1111 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS,
1112 ERROR_CODE.NOT_IN_HDFS,});
1113
1114 assertEquals(0, hbck.getOverlapGroups(table).size());
1115
1116
1117 doFsck(conf, true);
1118
1119
1120 assertNoErrors(doFsck(conf,false));
1121 assertFalse("Table "+ table + " should have been deleted",
1122 TEST_UTIL.getHBaseAdmin().tableExists(table));
1123 }
1124
1125 public void deleteTableDir(TableName table) throws IOException {
1126 Path rootDir = FSUtils.getRootDir(conf);
1127 FileSystem fs = rootDir.getFileSystem(conf);
1128 Path p = FSUtils.getTableDir(rootDir, table);
1129 HBaseFsck.debugLsr(conf, p);
1130 boolean success = fs.delete(p, true);
1131 LOG.info("Deleted " + p + " sucessfully? " + success);
1132 }
1133
1134
1135
1136
1137 @Test
1138 public void testNoVersionFile() throws Exception {
1139
1140 Path rootDir = FSUtils.getRootDir(conf);
1141 FileSystem fs = rootDir.getFileSystem(conf);
1142 Path versionFile = new Path(rootDir, HConstants.VERSION_FILE_NAME);
1143 fs.delete(versionFile, true);
1144
1145
1146 HBaseFsck hbck = doFsck(conf, false);
1147 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_VERSION_FILE });
1148
1149 doFsck(conf, true);
1150
1151
1152 assertNoErrors(doFsck(conf, false));
1153 }
1154
1155
1156
1157
1158 @Test
1159 public void testRegionShouldNotBeDeployed() throws Exception {
1160 TableName table =
1161 TableName.valueOf("tableRegionShouldNotBeDeployed");
1162 try {
1163 LOG.info("Starting testRegionShouldNotBeDeployed.");
1164 MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1165 assertTrue(cluster.waitForActiveAndReadyMaster());
1166
1167
1168 byte[][] SPLIT_KEYS = new byte[][] { new byte[0], Bytes.toBytes("aaa"),
1169 Bytes.toBytes("bbb"), Bytes.toBytes("ccc"), Bytes.toBytes("ddd") };
1170 HTableDescriptor htdDisabled = new HTableDescriptor(table);
1171 htdDisabled.addFamily(new HColumnDescriptor(FAM));
1172
1173
1174 FSTableDescriptors fstd = new FSTableDescriptors(conf);
1175 fstd.createTableDescriptor(htdDisabled);
1176 List<HRegionInfo> disabledRegions = TEST_UTIL.createMultiRegionsInMeta(
1177 TEST_UTIL.getConfiguration(), htdDisabled, SPLIT_KEYS);
1178
1179
1180 HRegionServer hrs = cluster.getRegionServer(0);
1181
1182
1183 TEST_UTIL.getHBaseAdmin().disableTable(table);
1184 TEST_UTIL.getHBaseAdmin().enableTable(table);
1185
1186
1187 TEST_UTIL.getHBaseAdmin().disableTable(table);
1188 HRegionInfo region = disabledRegions.remove(0);
1189 byte[] regionName = region.getRegionName();
1190
1191
1192 assertTrue(cluster.getServerWith(regionName) == -1);
1193
1194
1195
1196
1197
1198 HRegion r = HRegion.openHRegion(
1199 region, htdDisabled, hrs.getWAL(region), conf);
1200 hrs.addToOnlineRegions(r);
1201
1202 HBaseFsck hbck = doFsck(conf, false);
1203 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.SHOULD_NOT_BE_DEPLOYED });
1204
1205
1206 doFsck(conf, true);
1207
1208
1209 assertNoErrors(doFsck(conf, false));
1210 } finally {
1211 TEST_UTIL.getHBaseAdmin().enableTable(table);
1212 deleteTable(table);
1213 }
1214 }
1215
1216
1217
1218
1219 @Test
1220 public void testFixByTable() throws Exception {
1221 TableName table1 =
1222 TableName.valueOf("testFixByTable1");
1223 TableName table2 =
1224 TableName.valueOf("testFixByTable2");
1225 try {
1226 setupTable(table1);
1227
1228 TEST_UTIL.getHBaseAdmin().flush(table1.getName());
1229
1230 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1231 Bytes.toBytes("C"), false, false, true);
1232
1233 setupTable(table2);
1234
1235 TEST_UTIL.getHBaseAdmin().flush(table2.getName());
1236
1237 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1238 Bytes.toBytes("C"), false, false, true);
1239
1240 HBaseFsck hbck = doFsck(conf, false);
1241 assertErrors(hbck, new ERROR_CODE[] {
1242 ERROR_CODE.NOT_IN_HDFS, ERROR_CODE.NOT_IN_HDFS});
1243
1244
1245 doFsck(conf, true, table1);
1246
1247 assertNoErrors(doFsck(conf, false, table1));
1248
1249 assertErrors(doFsck(conf, false, table2),
1250 new ERROR_CODE[] {ERROR_CODE.NOT_IN_HDFS});
1251
1252
1253 doFsck(conf, true, table2);
1254
1255 assertNoErrors(doFsck(conf, false));
1256 assertEquals(ROWKEYS.length - 2, countRows());
1257 } finally {
1258 deleteTable(table1);
1259 deleteTable(table2);
1260 }
1261 }
1262
1263
1264
1265 @Test
1266 public void testLingeringSplitParent() throws Exception {
1267 TableName table =
1268 TableName.valueOf("testLingeringSplitParent");
1269 HTable meta = null;
1270 try {
1271 setupTable(table);
1272 assertEquals(ROWKEYS.length, countRows());
1273
1274
1275 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1276 HRegionLocation location = tbl.getRegionLocation("B");
1277
1278
1279 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("B"),
1280 Bytes.toBytes("C"), true, true, false);
1281
1282
1283 meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName(),
1284 executorService);
1285 HRegionInfo hri = location.getRegionInfo();
1286
1287 HRegionInfo a = new HRegionInfo(tbl.getName(),
1288 Bytes.toBytes("B"), Bytes.toBytes("BM"));
1289 HRegionInfo b = new HRegionInfo(tbl.getName(),
1290 Bytes.toBytes("BM"), Bytes.toBytes("C"));
1291
1292 hri.setOffline(true);
1293 hri.setSplit(true);
1294
1295 MetaEditor.addRegionToMeta(meta, hri, a, b);
1296 meta.flushCommits();
1297 TEST_UTIL.getHBaseAdmin().flush(TableName.META_TABLE_NAME.getName());
1298
1299 HBaseFsck hbck = doFsck(conf, false);
1300 assertErrors(hbck, new ERROR_CODE[] {
1301 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1302
1303
1304 hbck = doFsck(conf, true);
1305 assertErrors(hbck, new ERROR_CODE[] {
1306 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1307 assertFalse(hbck.shouldRerun());
1308 hbck = doFsck(conf, false);
1309 assertErrors(hbck, new ERROR_CODE[] {
1310 ERROR_CODE.LINGERING_SPLIT_PARENT, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1311
1312
1313 hbck = new HBaseFsck(conf);
1314 hbck.connect();
1315 hbck.setDisplayFullReport();
1316 hbck.setTimeLag(0);
1317 hbck.setFixSplitParents(true);
1318 hbck.onlineHbck();
1319 assertTrue(hbck.shouldRerun());
1320
1321 Get get = new Get(hri.getRegionName());
1322 Result result = meta.get(get);
1323 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1324 HConstants.SPLITA_QUALIFIER).isEmpty());
1325 assertTrue(result.getColumnCells(HConstants.CATALOG_FAMILY,
1326 HConstants.SPLITB_QUALIFIER).isEmpty());
1327 TEST_UTIL.getHBaseAdmin().flush(TableName.META_TABLE_NAME.getName());
1328
1329
1330 doFsck(conf, true);
1331
1332
1333 assertNoErrors(doFsck(conf, false));
1334 assertEquals(ROWKEYS.length, countRows());
1335 } finally {
1336 deleteTable(table);
1337 IOUtils.closeQuietly(meta);
1338 }
1339 }
1340
1341
1342
1343
1344
1345 @Test
1346 public void testValidLingeringSplitParent() throws Exception {
1347 TableName table =
1348 TableName.valueOf("testLingeringSplitParent");
1349 HTable meta = null;
1350 try {
1351 setupTable(table);
1352 assertEquals(ROWKEYS.length, countRows());
1353
1354
1355 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1356 HRegionLocation location = tbl.getRegionLocation("B");
1357
1358 meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName());
1359 HRegionInfo hri = location.getRegionInfo();
1360
1361
1362 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
1363 byte[] regionName = location.getRegionInfo().getRegionName();
1364 admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1365 TestEndToEndSplitTransaction.blockUntilRegionSplit(
1366 TEST_UTIL.getConfiguration(), 60000, regionName, true);
1367
1368
1369
1370
1371 HBaseFsck hbck = doFsck(conf, true, true, false, false, false, true, true, true, false, false, null);
1372 assertErrors(hbck, new ERROR_CODE[] {});
1373
1374
1375 Get get = new Get(hri.getRegionName());
1376 Result result = meta.get(get);
1377 assertNotNull(result);
1378 assertNotNull(HRegionInfo.getHRegionInfo(result));
1379
1380 assertEquals(ROWKEYS.length, countRows());
1381
1382
1383 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1384 assertNoErrors(doFsck(conf, false));
1385 } finally {
1386 deleteTable(table);
1387 IOUtils.closeQuietly(meta);
1388 }
1389 }
1390
1391
1392
1393
1394
1395 @Test(timeout=75000)
1396 public void testSplitDaughtersNotInMeta() throws Exception {
1397 TableName table =
1398 TableName.valueOf("testSplitdaughtersNotInMeta");
1399 HTable meta = null;
1400 try {
1401 setupTable(table);
1402 assertEquals(ROWKEYS.length, countRows());
1403
1404
1405 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1406 HRegionLocation location = tbl.getRegionLocation("B");
1407
1408 meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName());
1409 HRegionInfo hri = location.getRegionInfo();
1410
1411
1412 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
1413 byte[] regionName = location.getRegionInfo().getRegionName();
1414 admin.split(location.getRegionInfo().getRegionName(), Bytes.toBytes("BM"));
1415 TestEndToEndSplitTransaction.blockUntilRegionSplit(
1416 TEST_UTIL.getConfiguration(), 60000, regionName, true);
1417
1418 PairOfSameType<HRegionInfo> daughters = HRegionInfo.getDaughterRegions(meta.get(new Get(regionName)));
1419
1420
1421 Map<HRegionInfo, ServerName> hris = tbl.getRegionLocations();
1422 undeployRegion(admin, hris.get(daughters.getFirst()), daughters.getFirst());
1423 undeployRegion(admin, hris.get(daughters.getSecond()), daughters.getSecond());
1424
1425 meta.delete(new Delete(daughters.getFirst().getRegionName()));
1426 meta.delete(new Delete(daughters.getSecond().getRegionName()));
1427 meta.flushCommits();
1428
1429 HBaseFsck hbck = doFsck(conf, false);
1430 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1431 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1432
1433
1434 hbck = doFsck(conf, true, true, false, false, false, false, false, false, false, false, null);
1435 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1436 ERROR_CODE.NOT_IN_META_OR_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1437
1438
1439 Get get = new Get(hri.getRegionName());
1440 Result result = meta.get(get);
1441 assertNotNull(result);
1442 assertNotNull(HRegionInfo.getHRegionInfo(result));
1443
1444 assertEquals(ROWKEYS.length, countRows());
1445
1446
1447 assertEquals(tbl.getStartKeys().length, SPLITS.length + 1 + 1);
1448 assertNoErrors(doFsck(conf, false));
1449 } finally {
1450 deleteTable(table);
1451 IOUtils.closeQuietly(meta);
1452 }
1453 }
1454
1455
1456
1457
1458
1459 @Test(timeout=120000)
1460 public void testMissingFirstRegion() throws Exception {
1461 TableName table =
1462 TableName.valueOf("testMissingFirstRegion");
1463 try {
1464 setupTable(table);
1465 assertEquals(ROWKEYS.length, countRows());
1466
1467
1468 TEST_UTIL.getHBaseAdmin().disableTable(table);
1469 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes(""), Bytes.toBytes("A"), true,
1470 true, true);
1471 TEST_UTIL.getHBaseAdmin().enableTable(table);
1472
1473 HBaseFsck hbck = doFsck(conf, false);
1474 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY });
1475
1476 doFsck(conf, true);
1477
1478 assertNoErrors(doFsck(conf, false));
1479 } finally {
1480 deleteTable(table);
1481 }
1482 }
1483
1484
1485
1486
1487
1488 @Test(timeout=120000)
1489 public void testMissingLastRegion() throws Exception {
1490 TableName table =
1491 TableName.valueOf("testMissingLastRegion");
1492 try {
1493 setupTable(table);
1494 assertEquals(ROWKEYS.length, countRows());
1495
1496
1497 TEST_UTIL.getHBaseAdmin().disableTable(table);
1498 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("C"), Bytes.toBytes(""), true,
1499 true, true);
1500 TEST_UTIL.getHBaseAdmin().enableTable(table);
1501
1502 HBaseFsck hbck = doFsck(conf, false);
1503 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY });
1504
1505 doFsck(conf, true);
1506
1507 assertNoErrors(doFsck(conf, false));
1508 } finally {
1509 deleteTable(table);
1510 }
1511 }
1512
1513
1514
1515
1516 @Test
1517 public void testFixAssignmentsAndNoHdfsChecking() throws Exception {
1518 TableName table =
1519 TableName.valueOf("testFixAssignmentsAndNoHdfsChecking");
1520 try {
1521 setupTable(table);
1522 assertEquals(ROWKEYS.length, countRows());
1523
1524
1525 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1526 Bytes.toBytes("B"), true, false, false, false);
1527
1528
1529 HBaseFsck hbck = doFsck(conf, false);
1530 assertErrors(hbck, new ERROR_CODE[] {
1531 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1532
1533
1534 HBaseFsck fsck = new HBaseFsck(conf);
1535 fsck.connect();
1536 fsck.setDisplayFullReport();
1537 fsck.setTimeLag(0);
1538 fsck.setCheckHdfs(false);
1539 fsck.onlineHbck();
1540 assertErrors(fsck, new ERROR_CODE[] {
1541 ERROR_CODE.NOT_DEPLOYED, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1542
1543
1544 fsck = new HBaseFsck(conf);
1545 fsck.connect();
1546 fsck.setDisplayFullReport();
1547 fsck.setTimeLag(0);
1548 fsck.setCheckHdfs(false);
1549 fsck.setFixAssignments(true);
1550 fsck.onlineHbck();
1551 assertTrue(fsck.shouldRerun());
1552 fsck.onlineHbck();
1553 assertNoErrors(fsck);
1554
1555 assertEquals(ROWKEYS.length, countRows());
1556 } finally {
1557 deleteTable(table);
1558 }
1559 }
1560
1561
1562
1563
1564
1565
1566 @Test
1567 public void testFixMetaNotWorkingWithNoHdfsChecking() throws Exception {
1568 TableName table =
1569 TableName.valueOf("testFixMetaNotWorkingWithNoHdfsChecking");
1570 try {
1571 setupTable(table);
1572 assertEquals(ROWKEYS.length, countRows());
1573
1574
1575 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1576 Bytes.toBytes("B"), false, true, false, false);
1577
1578
1579 HBaseFsck hbck = doFsck(conf, false);
1580 assertErrors(hbck, new ERROR_CODE[] {
1581 ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1582
1583
1584 HBaseFsck fsck = new HBaseFsck(conf);
1585 fsck.connect();
1586 fsck.setDisplayFullReport();
1587 fsck.setTimeLag(0);
1588 fsck.setCheckHdfs(false);
1589 fsck.onlineHbck();
1590 assertErrors(fsck, new ERROR_CODE[] {
1591 ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1592
1593
1594 fsck = new HBaseFsck(conf);
1595 fsck.connect();
1596 fsck.setDisplayFullReport();
1597 fsck.setTimeLag(0);
1598 fsck.setCheckHdfs(false);
1599 fsck.setFixAssignments(true);
1600 fsck.setFixMeta(true);
1601 fsck.onlineHbck();
1602 assertFalse(fsck.shouldRerun());
1603 assertErrors(fsck, new ERROR_CODE[] {
1604 ERROR_CODE.NOT_IN_META, ERROR_CODE.HOLE_IN_REGION_CHAIN});
1605
1606
1607 fsck = doFsck(conf, true);
1608 assertTrue(fsck.shouldRerun());
1609 fsck = doFsck(conf, true);
1610 assertNoErrors(fsck);
1611 } finally {
1612 deleteTable(table);
1613 }
1614 }
1615
1616
1617
1618
1619
1620 @Test
1621 public void testFixHdfsHolesNotWorkingWithNoHdfsChecking() throws Exception {
1622 TableName table =
1623 TableName.valueOf("testFixHdfsHolesNotWorkingWithNoHdfsChecking");
1624 try {
1625 setupTable(table);
1626 assertEquals(ROWKEYS.length, countRows());
1627
1628
1629 TEST_UTIL.getHBaseAdmin().disableTable(table);
1630 deleteRegion(conf, tbl.getTableDescriptor(), Bytes.toBytes("A"),
1631 Bytes.toBytes("B"), true, true, false, true);
1632 TEST_UTIL.getHBaseAdmin().enableTable(table);
1633
1634 HRegionInfo hriOverlap = createRegion(conf, tbl.getTableDescriptor(),
1635 Bytes.toBytes("A2"), Bytes.toBytes("B"));
1636 TEST_UTIL.getHBaseCluster().getMaster().assignRegion(hriOverlap);
1637 TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager()
1638 .waitForAssignment(hriOverlap);
1639 ServerName server = regionStates.getRegionServerOfRegion(hriOverlap);
1640 TEST_UTIL.assertRegionOnServer(hriOverlap, server, REGION_ONLINE_TIMEOUT);
1641
1642 HBaseFsck hbck = doFsck(conf, false);
1643 assertErrors(hbck, new ERROR_CODE[] {
1644 ERROR_CODE.ORPHAN_HDFS_REGION, ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
1645 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1646
1647
1648 HBaseFsck fsck = new HBaseFsck(conf);
1649 fsck.connect();
1650 fsck.setDisplayFullReport();
1651 fsck.setTimeLag(0);
1652 fsck.setCheckHdfs(false);
1653 fsck.onlineHbck();
1654 assertErrors(fsck, new ERROR_CODE[] {
1655 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1656
1657
1658 fsck = new HBaseFsck(conf);
1659 fsck.connect();
1660 fsck.setDisplayFullReport();
1661 fsck.setTimeLag(0);
1662 fsck.setCheckHdfs(false);
1663 fsck.setFixHdfsHoles(true);
1664 fsck.setFixHdfsOverlaps(true);
1665 fsck.setFixHdfsOrphans(true);
1666 fsck.onlineHbck();
1667 assertFalse(fsck.shouldRerun());
1668 assertErrors(fsck, new ERROR_CODE[] {
1669 ERROR_CODE.HOLE_IN_REGION_CHAIN});
1670 } finally {
1671 if (TEST_UTIL.getHBaseAdmin().isTableDisabled(table)) {
1672 TEST_UTIL.getHBaseAdmin().enableTable(table);
1673 }
1674 deleteTable(table);
1675 }
1676 }
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686 Path getFlushedHFile(FileSystem fs, TableName table) throws IOException {
1687 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
1688 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
1689 Path famDir = new Path(regionDir, FAM_STR);
1690
1691
1692 while (true) {
1693 FileStatus[] hfFss = fs.listStatus(famDir);
1694 if (hfFss.length == 0) {
1695 continue;
1696 }
1697 for (FileStatus hfs : hfFss) {
1698 if (!hfs.isDir()) {
1699 return hfs.getPath();
1700 }
1701 }
1702 }
1703 }
1704
1705
1706
1707
1708 @Test(timeout=180000)
1709 public void testQuarantineCorruptHFile() throws Exception {
1710 TableName table = TableName.valueOf(name.getMethodName());
1711 try {
1712 setupTable(table);
1713 assertEquals(ROWKEYS.length, countRows());
1714 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1715
1716 FileSystem fs = FileSystem.get(conf);
1717 Path hfile = getFlushedHFile(fs, table);
1718
1719
1720 TEST_UTIL.getHBaseAdmin().disableTable(table);
1721
1722
1723 Path corrupt = new Path(hfile.getParent(), "deadbeef");
1724 TestHFile.truncateFile(fs, hfile, corrupt);
1725 LOG.info("Created corrupted file " + corrupt);
1726 HBaseFsck.debugLsr(conf, FSUtils.getRootDir(conf));
1727
1728
1729 HBaseFsck res = HbckTestingUtil.doHFileQuarantine(conf, table);
1730 assertEquals(res.getRetCode(), 0);
1731 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
1732 assertEquals(hfcc.getHFilesChecked(), 5);
1733 assertEquals(hfcc.getCorrupted().size(), 1);
1734 assertEquals(hfcc.getFailures().size(), 0);
1735 assertEquals(hfcc.getQuarantined().size(), 1);
1736 assertEquals(hfcc.getMissing().size(), 0);
1737
1738
1739 TEST_UTIL.getHBaseAdmin().enableTable(table);
1740 } finally {
1741 deleteTable(table);
1742 }
1743 }
1744
1745
1746
1747
1748 private void doQuarantineTest(TableName table, HBaseFsck hbck, int check,
1749 int corrupt, int fail, int quar, int missing) throws Exception {
1750 try {
1751 setupTable(table);
1752 assertEquals(ROWKEYS.length, countRows());
1753 TEST_UTIL.getHBaseAdmin().flush(table.getName());
1754
1755
1756 TEST_UTIL.getHBaseAdmin().disableTable(table);
1757
1758 String[] args = {"-sidelineCorruptHFiles", "-repairHoles", "-ignorePreCheckPermission",
1759 table.getNameAsString()};
1760 ExecutorService exec = new ScheduledThreadPoolExecutor(10);
1761 HBaseFsck res = hbck.exec(exec, args);
1762
1763 HFileCorruptionChecker hfcc = res.getHFilecorruptionChecker();
1764 assertEquals(hfcc.getHFilesChecked(), check);
1765 assertEquals(hfcc.getCorrupted().size(), corrupt);
1766 assertEquals(hfcc.getFailures().size(), fail);
1767 assertEquals(hfcc.getQuarantined().size(), quar);
1768 assertEquals(hfcc.getMissing().size(), missing);
1769
1770
1771 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
1772 admin.enableTableAsync(table);
1773 while (!admin.isTableEnabled(table)) {
1774 try {
1775 Thread.sleep(250);
1776 } catch (InterruptedException e) {
1777 e.printStackTrace();
1778 fail("Interrupted when trying to enable table " + table);
1779 }
1780 }
1781 } finally {
1782 deleteTable(table);
1783 }
1784 }
1785
1786
1787
1788
1789
1790 @Test(timeout=180000)
1791 public void testQuarantineMissingHFile() throws Exception {
1792 TableName table = TableName.valueOf(name.getMethodName());
1793 ExecutorService exec = new ScheduledThreadPoolExecutor(10);
1794
1795 final FileSystem fs = FileSystem.get(conf);
1796 HBaseFsck hbck = new HBaseFsck(conf, exec) {
1797 @Override
1798 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
1799 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
1800 boolean attemptedFirstHFile = false;
1801 @Override
1802 protected void checkHFile(Path p) throws IOException {
1803 if (!attemptedFirstHFile) {
1804 attemptedFirstHFile = true;
1805 assertTrue(fs.delete(p, true));
1806 }
1807 super.checkHFile(p);
1808 }
1809 };
1810 }
1811 };
1812 doQuarantineTest(table, hbck, 4, 0, 0, 0, 1);
1813 }
1814
1815
1816
1817
1818
1819
1820
1821 @Ignore @Test(timeout=180000)
1822 public void testQuarantineMissingFamdir() throws Exception {
1823 TableName table = TableName.valueOf(name.getMethodName());
1824 ExecutorService exec = new ScheduledThreadPoolExecutor(10);
1825
1826 final FileSystem fs = FileSystem.get(conf);
1827 HBaseFsck hbck = new HBaseFsck(conf, exec) {
1828 @Override
1829 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
1830 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
1831 boolean attemptedFirstFamDir = false;
1832 @Override
1833 protected void checkColFamDir(Path p) throws IOException {
1834 if (!attemptedFirstFamDir) {
1835 attemptedFirstFamDir = true;
1836 assertTrue(fs.delete(p, true));
1837 }
1838 super.checkColFamDir(p);
1839 }
1840 };
1841 }
1842 };
1843 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
1844 }
1845
1846
1847
1848
1849
1850 @Test(timeout=180000)
1851 public void testQuarantineMissingRegionDir() throws Exception {
1852 TableName table = TableName.valueOf(name.getMethodName());
1853 ExecutorService exec = new ScheduledThreadPoolExecutor(10);
1854
1855 final FileSystem fs = FileSystem.get(conf);
1856 HBaseFsck hbck = new HBaseFsck(conf, exec) {
1857 @Override
1858 public HFileCorruptionChecker createHFileCorruptionChecker(boolean sidelineCorruptHFiles) throws IOException {
1859 return new HFileCorruptionChecker(conf, executor, sidelineCorruptHFiles) {
1860 boolean attemptedFirstRegionDir = false;
1861 @Override
1862 protected void checkRegionDir(Path p) throws IOException {
1863 if (!attemptedFirstRegionDir) {
1864 attemptedFirstRegionDir = true;
1865 assertTrue(fs.delete(p, true));
1866 }
1867 super.checkRegionDir(p);
1868 }
1869 };
1870 }
1871 };
1872 doQuarantineTest(table, hbck, 3, 0, 0, 0, 1);
1873 }
1874
1875
1876
1877
1878 @Test
1879 public void testLingeringReferenceFile() throws Exception {
1880 TableName table =
1881 TableName.valueOf("testLingeringReferenceFile");
1882 try {
1883 setupTable(table);
1884 assertEquals(ROWKEYS.length, countRows());
1885
1886
1887 FileSystem fs = FileSystem.get(conf);
1888 Path tableDir= FSUtils.getTableDir(FSUtils.getRootDir(conf), table);
1889 Path regionDir = FSUtils.getRegionDirs(fs, tableDir).get(0);
1890 Path famDir = new Path(regionDir, FAM_STR);
1891 Path fakeReferenceFile = new Path(famDir, "fbce357483ceea.12144538");
1892 fs.create(fakeReferenceFile);
1893
1894 HBaseFsck hbck = doFsck(conf, false);
1895 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.LINGERING_REFERENCE_HFILE });
1896
1897 doFsck(conf, true);
1898
1899 assertNoErrors(doFsck(conf, false));
1900 } finally {
1901 deleteTable(table);
1902 }
1903 }
1904
1905
1906
1907
1908 @Test
1909 public void testMissingRegionInfoQualifier() throws Exception {
1910 TableName table =
1911 TableName.valueOf("testMissingRegionInfoQualifier");
1912 try {
1913 setupTable(table);
1914
1915
1916 final List<Delete> deletes = new LinkedList<Delete>();
1917 HTable meta = new HTable(conf, HTableDescriptor.META_TABLEDESC.getTableName());
1918 MetaScanner.metaScan(conf, new MetaScanner.MetaScannerVisitor() {
1919
1920 @Override
1921 public boolean processRow(Result rowResult) throws IOException {
1922 HRegionInfo hri = MetaScanner.getHRegionInfo(rowResult);
1923 if (hri != null && !hri.getTable().isSystemTable()) {
1924 Delete delete = new Delete(rowResult.getRow());
1925 delete.deleteColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
1926 deletes.add(delete);
1927 }
1928 return true;
1929 }
1930
1931 @Override
1932 public void close() throws IOException {
1933 }
1934 });
1935 meta.delete(deletes);
1936
1937
1938 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
1939 HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER, Bytes.toBytes("node1:60020")));
1940 meta.put(new Put(Bytes.toBytes(table + ",,1361911384013.810e28f59a57da91c66")).add(
1941 HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER, Bytes.toBytes(1362150791183L)));
1942 meta.close();
1943
1944 HBaseFsck hbck = doFsck(conf, false);
1945 assertTrue(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
1946
1947
1948 hbck = doFsck(conf, true);
1949
1950
1951 assertFalse(hbck.getErrors().getErrorList().contains(ERROR_CODE.EMPTY_META_CELL));
1952 } finally {
1953 deleteTable(table);
1954 }
1955 }
1956
1957
1958
1959
1960
1961
1962 @Test
1963 public void testErrorReporter() throws Exception {
1964 try {
1965 MockErrorReporter.calledCount = 0;
1966 doFsck(conf, false);
1967 assertEquals(MockErrorReporter.calledCount, 0);
1968
1969 conf.set("hbasefsck.errorreporter", MockErrorReporter.class.getName());
1970 doFsck(conf, false);
1971 assertTrue(MockErrorReporter.calledCount > 20);
1972 } finally {
1973 conf.set("hbasefsck.errorreporter",
1974 PrintingErrorReporter.class.getName());
1975 MockErrorReporter.calledCount = 0;
1976 }
1977 }
1978
1979 static class MockErrorReporter implements ErrorReporter {
1980 static int calledCount = 0;
1981
1982 @Override
1983 public void clear() {
1984 calledCount++;
1985 }
1986
1987 @Override
1988 public void report(String message) {
1989 calledCount++;
1990 }
1991
1992 @Override
1993 public void reportError(String message) {
1994 calledCount++;
1995 }
1996
1997 @Override
1998 public void reportError(ERROR_CODE errorCode, String message) {
1999 calledCount++;
2000 }
2001
2002 @Override
2003 public void reportError(ERROR_CODE errorCode, String message, TableInfo table) {
2004 calledCount++;
2005 }
2006
2007 @Override
2008 public void reportError(ERROR_CODE errorCode,
2009 String message, TableInfo table, HbckInfo info) {
2010 calledCount++;
2011 }
2012
2013 @Override
2014 public void reportError(ERROR_CODE errorCode, String message,
2015 TableInfo table, HbckInfo info1, HbckInfo info2) {
2016 calledCount++;
2017 }
2018
2019 @Override
2020 public int summarize() {
2021 return ++calledCount;
2022 }
2023
2024 @Override
2025 public void detail(String details) {
2026 calledCount++;
2027 }
2028
2029 @Override
2030 public ArrayList<ERROR_CODE> getErrorList() {
2031 calledCount++;
2032 return new ArrayList<ERROR_CODE>();
2033 }
2034
2035 @Override
2036 public void progress() {
2037 calledCount++;
2038 }
2039
2040 @Override
2041 public void print(String message) {
2042 calledCount++;
2043 }
2044
2045 @Override
2046 public void resetErrors() {
2047 calledCount++;
2048 }
2049
2050 @Override
2051 public boolean tableHasErrors(TableInfo table) {
2052 calledCount++;
2053 return false;
2054 }
2055 }
2056
2057 @Test(timeout=60000)
2058 public void testCheckTableLocks() throws Exception {
2059 IncrementingEnvironmentEdge edge = new IncrementingEnvironmentEdge(0);
2060 EnvironmentEdgeManager.injectEdge(edge);
2061
2062 HBaseFsck hbck = doFsck(conf, false);
2063 assertNoErrors(hbck);
2064
2065 ServerName mockName = ServerName.valueOf("localhost", 60000, 1);
2066
2067
2068 final TableLockManager tableLockManager = TableLockManager.createTableLockManager(conf, TEST_UTIL.getZooKeeperWatcher(), mockName);
2069 TableLock writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
2070 "testCheckTableLocks");
2071 writeLock.acquire();
2072 hbck = doFsck(conf, false);
2073 assertNoErrors(hbck);
2074
2075 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2076 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2077
2078 hbck = doFsck(conf, false);
2079 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2080
2081 final CountDownLatch latch = new CountDownLatch(1);
2082 new Thread() {
2083 @Override
2084 public void run() {
2085 TableLock readLock = tableLockManager.writeLock(TableName.valueOf("foo"),
2086 "testCheckTableLocks");
2087 try {
2088 latch.countDown();
2089 readLock.acquire();
2090 } catch (IOException ex) {
2091 fail();
2092 } catch (IllegalStateException ex) {
2093 return;
2094 }
2095 fail("should not have come here");
2096 };
2097 }.start();
2098
2099 latch.await();
2100 Threads.sleep(300);
2101
2102 hbck = doFsck(conf, false);
2103 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK});
2104
2105 edge.incrementTime(conf.getLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT,
2106 TableLockManager.DEFAULT_TABLE_LOCK_EXPIRE_TIMEOUT_MS));
2107
2108 hbck = doFsck(conf, false);
2109 assertErrors(hbck, new ERROR_CODE[] {ERROR_CODE.EXPIRED_TABLE_LOCK, ERROR_CODE.EXPIRED_TABLE_LOCK});
2110
2111 conf.setLong(TableLockManager.TABLE_LOCK_EXPIRE_TIMEOUT, 1);
2112
2113 Threads.sleep(10);
2114 hbck = doFsck(conf, true);
2115
2116 hbck = doFsck(conf, false);
2117 assertNoErrors(hbck);
2118
2119
2120 writeLock = tableLockManager.writeLock(TableName.valueOf("foo"),
2121 "should acquire without blocking");
2122 writeLock.acquire();
2123 writeLock.release();
2124 }
2125
2126 @Test
2127 public void testMetaOffline() throws Exception {
2128
2129 HBaseFsck hbck = doFsck(conf, false);
2130 assertNoErrors(hbck);
2131 deleteMetaRegion(conf, true, false, false);
2132 hbck = doFsck(conf, false);
2133
2134
2135 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2136 hbck = doFsck(conf, true);
2137 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NO_META_REGION, ERROR_CODE.UNKNOWN });
2138 hbck = doFsck(conf, false);
2139 assertNoErrors(hbck);
2140 }
2141
2142 private void deleteMetaRegion(Configuration conf, boolean unassign, boolean hdfs,
2143 boolean regionInfoOnly) throws IOException, InterruptedException {
2144 HConnection connection = HConnectionManager.getConnection(conf);
2145 HRegionLocation metaLocation = connection.locateRegion(TableName.META_TABLE_NAME,
2146 HConstants.EMPTY_START_ROW);
2147 ServerName hsa = metaLocation.getServerName();
2148 HRegionInfo hri = metaLocation.getRegionInfo();
2149 if (unassign) {
2150 LOG.info("Undeploying meta region " + hri + " from server " + hsa);
2151 undeployRegion(new HBaseAdmin(conf), hsa, hri);
2152 }
2153
2154 if (regionInfoOnly) {
2155 LOG.info("deleting hdfs .regioninfo data: " + hri.toString() + hsa.toString());
2156 Path rootDir = FSUtils.getRootDir(conf);
2157 FileSystem fs = rootDir.getFileSystem(conf);
2158 Path p = new Path(rootDir + "/" + HTableDescriptor.META_TABLEDESC.getNameAsString(),
2159 hri.getEncodedName());
2160 Path hriPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
2161 fs.delete(hriPath, true);
2162 }
2163
2164 if (hdfs) {
2165 LOG.info("deleting hdfs data: " + hri.toString() + hsa.toString());
2166 Path rootDir = FSUtils.getRootDir(conf);
2167 FileSystem fs = rootDir.getFileSystem(conf);
2168 Path p = new Path(rootDir + "/" + HTableDescriptor.META_TABLEDESC.getNameAsString(),
2169 hri.getEncodedName());
2170 HBaseFsck.debugLsr(conf, p);
2171 boolean success = fs.delete(p, true);
2172 LOG.info("Deleted " + p + " sucessfully? " + success);
2173 HBaseFsck.debugLsr(conf, p);
2174 }
2175 }
2176
2177 @Test
2178 public void testTableWithNoRegions() throws Exception {
2179
2180
2181 TableName table =
2182 TableName.valueOf(name.getMethodName());
2183 try {
2184
2185 HTableDescriptor desc = new HTableDescriptor(table);
2186 HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toString(FAM));
2187 desc.addFamily(hcd);
2188 TEST_UTIL.getHBaseAdmin().createTable(desc);
2189 tbl = new HTable(TEST_UTIL.getConfiguration(), table, executorService);
2190
2191
2192 deleteRegion(conf, tbl.getTableDescriptor(), HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false,
2193 false, true);
2194
2195 HBaseFsck hbck = doFsck(conf, false);
2196 assertErrors(hbck, new ERROR_CODE[] { ERROR_CODE.NOT_IN_HDFS });
2197
2198 doFsck(conf, true);
2199
2200
2201 doFsck(conf, true);
2202
2203
2204 assertNoErrors(doFsck(conf, false));
2205 } finally {
2206 deleteTable(table);
2207 }
2208
2209 }
2210
2211 @Test
2212 public void testHbckAfterRegionMerge() throws Exception {
2213 TableName table = TableName.valueOf("testMergeRegionFilesInHdfs");
2214 HTable meta = null;
2215 try {
2216
2217 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
2218 setupTable(table);
2219 assertEquals(ROWKEYS.length, countRows());
2220
2221
2222 TEST_UTIL.getHBaseAdmin().flush(table.getName());
2223 HRegionInfo region1 = tbl.getRegionLocation("A").getRegionInfo();
2224 HRegionInfo region2 = tbl.getRegionLocation("B").getRegionInfo();
2225
2226 int regionCountBeforeMerge = tbl.getRegionLocations().size();
2227
2228 assertNotEquals(region1, region2);
2229
2230
2231 HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
2232 admin.mergeRegions(region1.getEncodedNameAsBytes(),
2233 region2.getEncodedNameAsBytes(), false);
2234
2235
2236 long timeout = System.currentTimeMillis() + 30 * 1000;
2237 while (true) {
2238 if (tbl.getRegionLocations().size() < regionCountBeforeMerge) {
2239 break;
2240 } else if (System.currentTimeMillis() > timeout) {
2241 fail("Time out waiting on region " + region1.getEncodedName()
2242 + " and " + region2.getEncodedName() + " be merged");
2243 }
2244 Thread.sleep(10);
2245 }
2246
2247 assertEquals(ROWKEYS.length, countRows());
2248
2249 HBaseFsck hbck = doFsck(conf, false);
2250 assertNoErrors(hbck);
2251
2252 } finally {
2253 TEST_UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(true);
2254 deleteTable(table);
2255 IOUtils.closeQuietly(meta);
2256 }
2257 }
2258
2259 @Test
2260 public void testRegionBoundariesCheck() throws Exception {
2261 HBaseFsck hbck = doFsck(conf, false);
2262 assertNoErrors(hbck);
2263 try {
2264 hbck.checkRegionBoundaries();
2265 } catch (IllegalArgumentException e) {
2266 if (e.getMessage().endsWith("not a valid DFS filename.")) {
2267 fail("Table directory path is not valid." + e.getMessage());
2268 }
2269 }
2270 }
2271
2272 @org.junit.Rule
2273 public TestName name = new TestName();
2274 }