1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.Arrays;
26 import java.util.ArrayList;
27 import java.util.Arrays;
28 import java.util.Collection;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.Set;
32 import java.util.HashSet;
33 import java.util.TreeSet;
34
35 import org.apache.commons.logging.Log;
36 import org.apache.commons.logging.LogFactory;
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.fs.FSDataOutputStream;
39 import org.apache.hadoop.fs.FileSystem;
40 import org.apache.hadoop.fs.FileStatus;
41 import org.apache.hadoop.fs.FSDataInputStream;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.fs.PathFilter;
44 import org.apache.hadoop.hbase.HBaseTestingUtility;
45 import org.apache.hadoop.hbase.HColumnDescriptor;
46 import org.apache.hadoop.hbase.HConstants;
47 import org.apache.hadoop.hbase.HRegionInfo;
48 import org.apache.hadoop.hbase.HTableDescriptor;
49 import org.apache.hadoop.hbase.TableName;
50 import org.apache.hadoop.hbase.TableNotEnabledException;
51 import org.apache.hadoop.hbase.Waiter;
52 import org.apache.hadoop.hbase.client.Durability;
53 import org.apache.hadoop.hbase.client.HBaseAdmin;
54 import org.apache.hadoop.hbase.client.HTable;
55 import org.apache.hadoop.hbase.client.Put;
56 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
57 import org.apache.hadoop.hbase.io.HFileLink;
58 import org.apache.hadoop.hbase.master.HMaster;
59 import org.apache.hadoop.hbase.master.MasterFileSystem;
60 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
61 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
62 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
63 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
64 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
65 import org.apache.hadoop.hbase.regionserver.HRegion;
66 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
67 import org.apache.hadoop.hbase.regionserver.HRegionServer;
68 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
69 import org.apache.hadoop.hbase.util.Bytes;
70 import org.apache.hadoop.hbase.util.FSTableDescriptors;
71 import org.apache.hadoop.hbase.util.FSUtils;
72 import org.apache.hadoop.hbase.util.FSVisitor;
73 import org.apache.hadoop.hbase.util.FSUtils;
74 import org.apache.hadoop.hbase.util.MD5Hash;
75 import org.junit.Assert;
76
77 import com.google.protobuf.ServiceException;
78
79
80
81
82 public class SnapshotTestingUtils {
83
84 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
85 private static byte[] KEYS = Bytes.toBytes("0123456789");
86
87
88
89
90
91
92
93 public static void assertNoSnapshots(HBaseAdmin admin) throws IOException {
94 assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
95 .size());
96 }
97
98
99
100
101
102 public static List<SnapshotDescription> assertExistsMatchingSnapshot(
103 HBaseAdmin admin, String snapshotName, TableName tableName)
104 throws IOException {
105
106 List<SnapshotDescription> snapshots = admin.listSnapshots();
107
108 List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
109 for (SnapshotDescription sd : snapshots) {
110 if (snapshotName.equals(sd.getName()) &&
111 tableName.equals(TableName.valueOf(sd.getTable()))) {
112 returnedSnapshots.add(sd);
113 }
114 }
115
116 Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
117 return returnedSnapshots;
118 }
119
120
121
122
123 public static void assertOneSnapshotThatMatches(HBaseAdmin admin,
124 SnapshotDescription snapshot) throws IOException {
125 assertOneSnapshotThatMatches(admin, snapshot.getName(),
126 TableName.valueOf(snapshot.getTable()));
127 }
128
129
130
131
132
133 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
134 HBaseAdmin admin, String snapshotName, TableName tableName)
135 throws IOException {
136
137 List<SnapshotDescription> snapshots = admin.listSnapshots();
138
139 assertEquals("Should only have 1 snapshot", 1, snapshots.size());
140 assertEquals(snapshotName, snapshots.get(0).getName());
141 assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
142
143 return snapshots;
144 }
145
146
147
148
149
150 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
151 HBaseAdmin admin, byte[] snapshot, TableName tableName) throws IOException {
152 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
153 tableName);
154 }
155
156
157
158
159
160 public static void confirmSnapshotValid(
161 SnapshotDescription snapshotDescriptor, TableName tableName,
162 byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
163 throws IOException {
164 ArrayList nonEmptyTestFamilies = new ArrayList(1);
165 nonEmptyTestFamilies.add(testFamily);
166 confirmSnapshotValid(snapshotDescriptor, tableName,
167 nonEmptyTestFamilies, null, rootDir, admin, fs);
168 }
169
170
171
172
173 public static void confirmEmptySnapshotValid(
174 SnapshotDescription snapshotDescriptor, TableName tableName,
175 byte[] testFamily, Path rootDir, HBaseAdmin admin, FileSystem fs)
176 throws IOException {
177 ArrayList emptyTestFamilies = new ArrayList(1);
178 emptyTestFamilies.add(testFamily);
179 confirmSnapshotValid(snapshotDescriptor, tableName,
180 null, emptyTestFamilies, rootDir, admin, fs);
181 }
182
183
184
185
186
187
188
189 public static void confirmSnapshotValid(
190 SnapshotDescription snapshotDescriptor, TableName tableName,
191 List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
192 Path rootDir, HBaseAdmin admin, FileSystem fs) throws IOException {
193 final Configuration conf = admin.getConfiguration();
194
195
196 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
197 snapshotDescriptor, rootDir);
198 assertTrue(fs.exists(snapshotDir));
199
200 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
201
202
203 final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
204
205 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
206 Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
207 for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
208 SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
209 new SnapshotReferenceUtil.StoreFileVisitor() {
210 @Override
211 public void storeFile(final HRegionInfo regionInfo, final String family,
212 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
213 snapshotFamilies.add(Bytes.toBytes(family));
214 }
215 });
216 }
217
218
219 if (nonEmptyTestFamilies != null) {
220 for (final byte[] familyName: nonEmptyTestFamilies) {
221 assertTrue(snapshotFamilies.contains(familyName));
222 }
223 }
224
225
226 if (emptyTestFamilies != null) {
227 for (final byte[] familyName: emptyTestFamilies) {
228 assertFalse(snapshotFamilies.contains(familyName));
229 }
230 }
231
232
233 List<HRegionInfo> regions = admin.getTableRegions(tableName);
234 assertEquals(regions.size(), regionManifests.size());
235
236
237 for (HRegionInfo info : regions) {
238 String regionName = info.getEncodedName();
239 assertTrue(regionManifests.containsKey(regionName));
240 }
241 }
242
243
244
245
246
247
248
249
250
251
252 public static void waitForSnapshotToComplete(HMaster master,
253 SnapshotDescription snapshot, long sleep) throws ServiceException {
254 final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
255 .setSnapshot(snapshot).build();
256 IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
257 .buildPartial();
258 while (!done.getDone()) {
259 done = master.isSnapshotDone(null, request);
260 try {
261 Thread.sleep(sleep);
262 } catch (InterruptedException e) {
263 throw new ServiceException(e);
264 }
265 }
266 }
267
268
269
270
271
272 public static void snapshot(HBaseAdmin admin,
273 final String snapshotName, final String tableName,
274 SnapshotDescription.Type type, int numTries) throws IOException {
275 int tries = 0;
276 CorruptedSnapshotException lastEx = null;
277 while (tries++ < numTries) {
278 try {
279 admin.snapshot(snapshotName, tableName, type);
280 return;
281 } catch (CorruptedSnapshotException cse) {
282 LOG.warn("Got CorruptedSnapshotException", cse);
283 lastEx = cse;
284 }
285 }
286 throw lastEx;
287 }
288
289 public static void cleanupSnapshot(HBaseAdmin admin, byte[] tableName)
290 throws IOException {
291 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
292 }
293
294 public static void cleanupSnapshot(HBaseAdmin admin, String snapshotName)
295 throws IOException {
296
297 admin.deleteSnapshot(snapshotName);
298 assertNoSnapshots(admin);
299 }
300
301
302
303
304
305
306
307
308
309 public static void expectSnapshotDoneException(HMaster master,
310 IsSnapshotDoneRequest snapshot,
311 Class<? extends HBaseSnapshotException> clazz) {
312 try {
313 master.isSnapshotDone(null, snapshot);
314 Assert.fail("didn't fail to lookup a snapshot");
315 } catch (ServiceException se) {
316 try {
317 throw ProtobufUtil.getRemoteException(se);
318 } catch (HBaseSnapshotException e) {
319 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
320 } catch (Throwable t) {
321 Assert.fail("Threw an unexpected exception:" + t);
322 }
323 }
324 }
325
326
327
328
329
330
331
332
333
334 public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
335 throws IOException {
336 final ArrayList<Path> hfiles = new ArrayList<Path>();
337 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
338 @Override
339 public void storeFile(final String region, final String family, final String hfileName)
340 throws IOException {
341 hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
342 }
343 });
344 return hfiles.toArray(new Path[hfiles.size()]);
345 }
346
347 public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
348 throws IOException {
349 Path[] files = listHFiles(fs, tableDir);
350 String[] names = new String[files.length];
351 for (int i = 0; i < files.length; ++i) {
352 names[i] = files[i].getName();
353 }
354 Arrays.sort(names);
355 return names;
356 }
357
358
359
360
361
362
363 public static void createSnapshotAndValidate(HBaseAdmin admin,
364 TableName tableName, String familyName, String snapshotNameString,
365 Path rootDir, FileSystem fs, boolean onlineSnapshot)
366 throws Exception {
367 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
368 nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
369 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames,
370 snapshotNameString, rootDir, fs, onlineSnapshot);
371 }
372
373
374
375
376
377 public static void createSnapshotAndValidate(HBaseAdmin admin,
378 TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
379 String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
380 throws Exception {
381 if (!onlineSnapshot) {
382 try {
383 admin.disableTable(tableName);
384 } catch (TableNotEnabledException tne) {
385 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
386 "already disabled.");
387 }
388 }
389 admin.snapshot(snapshotNameString, tableName);
390
391 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
392 snapshotNameString, tableName);
393 if (snapshots == null || snapshots.size() != 1) {
394 Assert.fail("Incorrect number of snapshots for table " + tableName);
395 }
396
397 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
398 emptyFamilyNames, rootDir, admin, fs);
399 }
400
401
402
403
404
405
406
407
408
409 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
410 throws IOException {
411 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
412 final FileSystem fs = mfs.getFileSystem();
413
414 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
415 mfs.getRootDir());
416 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
417 final TableName table = TableName.valueOf(snapshotDesc.getTable());
418
419 final ArrayList corruptedFiles = new ArrayList();
420 final Configuration conf = util.getConfiguration();
421 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
422 new SnapshotReferenceUtil.StoreFileVisitor() {
423 @Override
424 public void storeFile(final HRegionInfo regionInfo, final String family,
425 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
426 String region = regionInfo.getEncodedName();
427 String hfile = storeFile.getName();
428 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
429 if (corruptedFiles.size() % 2 == 0) {
430 fs.delete(link.getAvailablePath(fs));
431 corruptedFiles.add(hfile);
432 }
433 }
434 });
435
436 assertTrue(corruptedFiles.size() > 0);
437 return corruptedFiles;
438 }
439
440
441
442
443 public static class SnapshotMock {
444 private final static String TEST_FAMILY = "cf";
445 public final static int TEST_NUM_REGIONS = 4;
446
447 private final Configuration conf;
448 private final FileSystem fs;
449 private final Path rootDir;
450
451 static class RegionData {
452 public HRegionInfo hri;
453 public Path tableDir;
454 public Path[] files;
455
456 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
457 this.tableDir = tableDir;
458 this.hri = hri;
459 this.files = new Path[nfiles];
460 }
461 }
462
463 public static class SnapshotBuilder {
464 private final RegionData[] tableRegions;
465 private final SnapshotDescription desc;
466 private final HTableDescriptor htd;
467 private final Configuration conf;
468 private final FileSystem fs;
469 private final Path rootDir;
470 private Path snapshotDir;
471 private int snapshotted = 0;
472
473 public SnapshotBuilder(final Configuration conf, final FileSystem fs,
474 final Path rootDir, final HTableDescriptor htd,
475 final SnapshotDescription desc, final RegionData[] tableRegions)
476 throws IOException {
477 this.fs = fs;
478 this.conf = conf;
479 this.rootDir = rootDir;
480 this.htd = htd;
481 this.desc = desc;
482 this.tableRegions = tableRegions;
483 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
484 new FSTableDescriptors(conf)
485 .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
486 }
487
488 public HTableDescriptor getTableDescriptor() {
489 return this.htd;
490 }
491
492 public SnapshotDescription getSnapshotDescription() {
493 return this.desc;
494 }
495
496 public Path getSnapshotsDir() {
497 return this.snapshotDir;
498 }
499
500 public Path[] addRegion() throws IOException {
501 return addRegion(desc);
502 }
503
504 public Path[] addRegionV1() throws IOException {
505 return addRegion(desc.toBuilder()
506 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
507 .build());
508 }
509
510 public Path[] addRegionV2() throws IOException {
511 return addRegion(desc.toBuilder()
512 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
513 .build());
514 }
515
516 private Path[] addRegion(final SnapshotDescription desc) throws IOException {
517 if (this.snapshotted == tableRegions.length) {
518 throw new UnsupportedOperationException("No more regions in the table");
519 }
520
521 RegionData regionData = tableRegions[this.snapshotted++];
522 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
523 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
524 manifest.addRegion(regionData.tableDir, regionData.hri);
525 return regionData.files;
526 }
527
528 private void corruptFile(Path p) throws IOException {
529 String manifestName = p.getName();
530
531
532 Path newP = new Path(p.getParent(), manifestName + "1");
533 fs.rename(p, newP);
534
535
536 FSDataOutputStream out = fs.create(p);
537
538
539
540 FSDataInputStream input = fs.open(newP);
541 byte[] buffer = new byte[25];
542 int len = input.read(0, buffer, 0, 25);
543 if (len > 1) {
544 out.write(buffer, 0, len - 1);
545 }
546 out.close();
547
548
549 fs.delete(newP);
550 }
551
552
553
554
555
556
557 public void corruptOneRegionManifest() throws IOException {
558 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
559 @Override public boolean accept(Path path) {
560 return path.getName().startsWith(SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX);
561 }
562 });
563
564 if (manifestFiles.length == 0) return;
565
566
567 Path p = manifestFiles[0].getPath();
568 corruptFile(p);
569 }
570
571
572
573
574
575
576 public void corruptDataManifest() throws IOException {
577 FileStatus[] manifestFiles = FSUtils.listStatus(fs, snapshotDir, new PathFilter() {
578 @Override
579 public boolean accept(Path path) {
580 return path.getName().startsWith(SnapshotManifest.DATA_MANIFEST_NAME);
581 }
582 });
583
584 if (manifestFiles.length == 0) return;
585
586
587 Path p = manifestFiles[0].getPath();
588 corruptFile(p);
589 }
590
591 public Path commit() throws IOException {
592 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
593 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
594 manifest.addTableDescriptor(htd);
595 manifest.consolidate();
596 SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
597 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
598 return snapshotDir;
599 }
600
601 public void consolidate() throws IOException {
602 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
603 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
604 manifest.addTableDescriptor(htd);
605 manifest.consolidate();
606 }
607 }
608
609 public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
610 this.fs = fs;
611 this.conf = conf;
612 this.rootDir = rootDir;
613 }
614
615 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName)
616 throws IOException {
617 return createSnapshot(snapshotName, tableName, SnapshotManifestV1.DESCRIPTOR_VERSION);
618 }
619
620 public SnapshotBuilder createSnapshotV1(final String snapshotName, final String tableName,
621 final int numRegions) throws IOException {
622 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV1.DESCRIPTOR_VERSION);
623 }
624
625 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName)
626 throws IOException {
627 return createSnapshot(snapshotName, tableName, SnapshotManifestV2.DESCRIPTOR_VERSION);
628 }
629
630 public SnapshotBuilder createSnapshotV2(final String snapshotName, final String tableName,
631 final int numRegions) throws IOException {
632 return createSnapshot(snapshotName, tableName, numRegions, SnapshotManifestV2.DESCRIPTOR_VERSION);
633 }
634
635 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
636 final int version) throws IOException {
637 return createSnapshot(snapshotName, tableName, TEST_NUM_REGIONS, version);
638 }
639
640 private SnapshotBuilder createSnapshot(final String snapshotName, final String tableName,
641 final int numRegions, final int version) throws IOException {
642 HTableDescriptor htd = createHtd(tableName);
643 RegionData[] regions = createTable(htd, numRegions);
644
645 SnapshotDescription desc = SnapshotDescription.newBuilder()
646 .setTable(htd.getNameAsString())
647 .setName(snapshotName)
648 .setVersion(version)
649 .build();
650
651 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
652 SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
653 return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
654 }
655
656 public HTableDescriptor createHtd(final String tableName) {
657 HTableDescriptor htd = new HTableDescriptor(tableName);
658 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
659 return htd;
660 }
661
662 private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
663 throws IOException {
664 Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
665 new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
666
667 assertTrue(nregions % 2 == 0);
668 RegionData[] regions = new RegionData[nregions];
669 for (int i = 0; i < regions.length; i += 2) {
670 byte[] startKey = Bytes.toBytes(0 + i * 2);
671 byte[] endKey = Bytes.toBytes(1 + i * 2);
672
673
674 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
675 HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
676 regions[i] = new RegionData(tableDir, hri, 3);
677 for (int j = 0; j < regions[i].files.length; ++j) {
678 Path storeFile = createStoreFile(rfs.createTempName());
679 regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
680 }
681
682
683
684 startKey = Bytes.toBytes(2 + i * 2);
685 endKey = Bytes.toBytes(3 + i * 2);
686 hri = new HRegionInfo(htd.getTableName());
687 rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
688 regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
689 for (int j = 0; j < regions[i].files.length; ++j) {
690 String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
691 Path refFile = createStoreFile(new Path(rootDir, refName));
692 regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
693 }
694 }
695 return regions;
696 }
697
698 private Path createStoreFile(final Path storeFile)
699 throws IOException {
700 FSDataOutputStream out = fs.create(storeFile);
701 try {
702 out.write(Bytes.toBytes(storeFile.toString()));
703 } finally {
704 out.close();
705 }
706 return storeFile;
707 }
708 }
709
710
711
712
713 public static void waitForTableToBeOnline(final HBaseTestingUtility util,
714 final TableName tableName)
715 throws IOException, InterruptedException {
716 HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
717 List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
718 for (HRegion region : onlineRegions) {
719 region.waitForFlushesAndCompactions();
720 }
721
722 final HBaseAdmin hBaseAdmin = util.getHBaseAdmin();
723 util.waitFor(60000, new Waiter.Predicate<IOException>() {
724 @Override
725 public boolean evaluate() throws IOException {
726 return hBaseAdmin.isTableAvailable(tableName);
727 }
728 });
729 }
730
731 public static void createTable(final HBaseTestingUtility util, final TableName tableName,
732 final byte[]... families) throws IOException, InterruptedException {
733 HTableDescriptor htd = new HTableDescriptor(tableName);
734 for (byte[] family : families) {
735 HColumnDescriptor hcd = new HColumnDescriptor(family);
736 htd.addFamily(hcd);
737 }
738 byte[][] splitKeys = new byte[KEYS.length-2][];
739 for (int i = 0; i < splitKeys.length; ++i) {
740 splitKeys[i] = new byte[] { KEYS[i+1] };
741 }
742 util.createTable(htd, splitKeys);
743 assertEquals(KEYS.length-1, util.getHBaseAdmin().getTableRegions(tableName).size());
744 }
745
746 public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
747 byte[]... families) throws IOException, InterruptedException {
748 loadData(util, new HTable(util.getConfiguration(), tableName), rows, families);
749 }
750
751 public static void loadData(final HBaseTestingUtility util, final HTable table, int rows,
752 byte[]... families) throws IOException, InterruptedException {
753 table.setAutoFlush(false, true);
754
755
756 assertTrue(rows >= KEYS.length);
757 for (byte k0: KEYS) {
758 byte[] k = new byte[] { k0 };
759 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
760 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
761 putData(table, families, key, value);
762 rows--;
763 }
764
765
766 while (rows-- > 0) {
767 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
768 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
769 putData(table, families, key, value);
770 }
771 table.flushCommits();
772
773 waitForTableToBeOnline(util, table.getName());
774 }
775
776 private static void putData(final HTable table, final byte[][] families,
777 final byte[] key, final byte[] value) throws IOException {
778 byte[] q = Bytes.toBytes("q");
779 Put put = new Put(key);
780 put.setDurability(Durability.SKIP_WAL);
781 for (byte[] family: families) {
782 put.add(family, q, value);
783 }
784 table.put(put);
785 }
786
787 public static void deleteAllSnapshots(final HBaseAdmin admin)
788 throws IOException {
789
790 for (SnapshotDescription snapshot: admin.listSnapshots()) {
791 admin.deleteSnapshot(snapshot.getName());
792 }
793 SnapshotTestingUtils.assertNoSnapshots(admin);
794 }
795
796 public static void deleteArchiveDirectory(final HBaseTestingUtility util)
797 throws IOException {
798
799 MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
800 Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
801 mfs.getFileSystem().delete(archiveDir, true);
802 }
803
804 public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
805 long expectedRows) throws IOException {
806 HTable table = new HTable(util.getConfiguration(), tableName);
807 try {
808 assertEquals(expectedRows, util.countRows(table));
809 } finally {
810 table.close();
811 }
812 }
813 }