1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.snapshot;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertFalse;
22 import static org.junit.Assert.assertTrue;
23
24 import java.io.IOException;
25 import java.util.Arrays;
26 import java.util.ArrayList;
27 import java.util.HashSet;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.Set;
31 import java.util.TreeSet;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FSDataOutputStream;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.Path;
39 import org.apache.hadoop.hbase.HBaseTestingUtility;
40 import org.apache.hadoop.hbase.HColumnDescriptor;
41 import org.apache.hadoop.hbase.HConstants;
42 import org.apache.hadoop.hbase.HRegionInfo;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.TableName;
45 import org.apache.hadoop.hbase.TableNotEnabledException;
46 import org.apache.hadoop.hbase.Waiter;
47 import org.apache.hadoop.hbase.client.Admin;
48 import org.apache.hadoop.hbase.client.BufferedMutator;
49 import org.apache.hadoop.hbase.client.Durability;
50 import org.apache.hadoop.hbase.client.HBaseAdmin;
51 import org.apache.hadoop.hbase.client.HTable;
52 import org.apache.hadoop.hbase.client.Put;
53 import org.apache.hadoop.hbase.client.Table;
54 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
55 import org.apache.hadoop.hbase.client.RegionReplicaUtil;
56 import org.apache.hadoop.hbase.io.HFileLink;
57 import org.apache.hadoop.hbase.master.HMaster;
58 import org.apache.hadoop.hbase.master.MasterFileSystem;
59 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
60 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
61 import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
62 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneRequest;
63 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.IsSnapshotDoneResponse;
64 import org.apache.hadoop.hbase.regionserver.HRegion;
65 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
66 import org.apache.hadoop.hbase.regionserver.HRegionServer;
67 import org.apache.hadoop.hbase.util.Bytes;
68 import org.apache.hadoop.hbase.util.FSTableDescriptors;
69 import org.apache.hadoop.hbase.util.FSVisitor;
70 import org.apache.hadoop.hbase.util.FSUtils;
71 import org.apache.hadoop.hbase.util.MD5Hash;
72 import org.junit.Assert;
73
74 import com.google.protobuf.ServiceException;
75
76
77
78
79 public class SnapshotTestingUtils {
80
81 private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
82 private static byte[] KEYS = Bytes.toBytes("0123456789");
83
84
85
86
87
88
89
90 public static void assertNoSnapshots(Admin admin) throws IOException {
91 assertEquals("Have some previous snapshots", 0, admin.listSnapshots()
92 .size());
93 }
94
95
96
97
98
99 public static List<SnapshotDescription> assertExistsMatchingSnapshot(
100 Admin admin, String snapshotName, TableName tableName)
101 throws IOException {
102
103 List<SnapshotDescription> snapshots = admin.listSnapshots();
104
105 List<SnapshotDescription> returnedSnapshots = new ArrayList<SnapshotDescription>();
106 for (SnapshotDescription sd : snapshots) {
107 if (snapshotName.equals(sd.getName()) &&
108 tableName.equals(TableName.valueOf(sd.getTable()))) {
109 returnedSnapshots.add(sd);
110 }
111 }
112
113 Assert.assertTrue("No matching snapshots found.", returnedSnapshots.size()>0);
114 return returnedSnapshots;
115 }
116
117
118
119
120 public static void assertOneSnapshotThatMatches(Admin admin,
121 SnapshotDescription snapshot) throws IOException {
122 assertOneSnapshotThatMatches(admin, snapshot.getName(),
123 TableName.valueOf(snapshot.getTable()));
124 }
125
126
127
128
129
130 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
131 Admin admin, String snapshotName, TableName tableName)
132 throws IOException {
133
134 List<SnapshotDescription> snapshots = admin.listSnapshots();
135
136 assertEquals("Should only have 1 snapshot", 1, snapshots.size());
137 assertEquals(snapshotName, snapshots.get(0).getName());
138 assertEquals(tableName, TableName.valueOf(snapshots.get(0).getTable()));
139
140 return snapshots;
141 }
142
143
144
145
146
147 public static List<SnapshotDescription> assertOneSnapshotThatMatches(
148 Admin admin, byte[] snapshot, TableName tableName) throws IOException {
149 return assertOneSnapshotThatMatches(admin, Bytes.toString(snapshot),
150 tableName);
151 }
152
153
154
155
156
157 public static void confirmSnapshotValid(
158 SnapshotDescription snapshotDescriptor, TableName tableName,
159 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
160 throws IOException {
161 ArrayList nonEmptyTestFamilies = new ArrayList(1);
162 nonEmptyTestFamilies.add(testFamily);
163 confirmSnapshotValid(snapshotDescriptor, tableName,
164 nonEmptyTestFamilies, null, rootDir, admin, fs);
165 }
166
167
168
169
170 public static void confirmEmptySnapshotValid(
171 SnapshotDescription snapshotDescriptor, TableName tableName,
172 byte[] testFamily, Path rootDir, Admin admin, FileSystem fs)
173 throws IOException {
174 ArrayList emptyTestFamilies = new ArrayList(1);
175 emptyTestFamilies.add(testFamily);
176 confirmSnapshotValid(snapshotDescriptor, tableName,
177 null, emptyTestFamilies, rootDir, admin, fs);
178 }
179
180
181
182
183
184
185
186 public static void confirmSnapshotValid(
187 SnapshotDescription snapshotDescriptor, TableName tableName,
188 List<byte[]> nonEmptyTestFamilies, List<byte[]> emptyTestFamilies,
189 Path rootDir, Admin admin, FileSystem fs) throws IOException {
190 final Configuration conf = admin.getConfiguration();
191
192
193 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
194 snapshotDescriptor, rootDir);
195 assertTrue(fs.exists(snapshotDir));
196
197 SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
198
199
200 final Set<byte[]> snapshotFamilies = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
201
202 SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, desc);
203 Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
204 for (SnapshotRegionManifest regionManifest: regionManifests.values()) {
205 SnapshotReferenceUtil.visitRegionStoreFiles(regionManifest,
206 new SnapshotReferenceUtil.StoreFileVisitor() {
207 @Override
208 public void storeFile(final HRegionInfo regionInfo, final String family,
209 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
210 snapshotFamilies.add(Bytes.toBytes(family));
211 }
212 });
213 }
214
215
216 if (nonEmptyTestFamilies != null) {
217 for (final byte[] familyName: nonEmptyTestFamilies) {
218 assertTrue(snapshotFamilies.contains(familyName));
219 }
220 }
221
222
223 if (emptyTestFamilies != null) {
224 for (final byte[] familyName: emptyTestFamilies) {
225 assertFalse(snapshotFamilies.contains(familyName));
226 }
227 }
228
229
230 List<HRegionInfo> regions = admin.getTableRegions(tableName);
231
232 RegionReplicaUtil.removeNonDefaultRegions(regions);
233 assertEquals(regions.size(), regionManifests.size());
234
235
236 for (HRegionInfo info : regions) {
237 String regionName = info.getEncodedName();
238 assertTrue(regionManifests.containsKey(regionName));
239 }
240 }
241
242
243
244
245
246
247
248
249
250
251 public static void waitForSnapshotToComplete(HMaster master,
252 SnapshotDescription snapshot, long sleep) throws ServiceException {
253 final IsSnapshotDoneRequest request = IsSnapshotDoneRequest.newBuilder()
254 .setSnapshot(snapshot).build();
255 IsSnapshotDoneResponse done = IsSnapshotDoneResponse.newBuilder()
256 .buildPartial();
257 while (!done.getDone()) {
258 done = master.getMasterRpcServices().isSnapshotDone(null, request);
259 try {
260 Thread.sleep(sleep);
261 } catch (InterruptedException e) {
262 throw new ServiceException(e);
263 }
264 }
265 }
266
267
268
269
270
271 public static void snapshot(Admin admin,
272 final String snapshotName, final String tableName,
273 SnapshotDescription.Type type, int numTries) throws IOException {
274 int tries = 0;
275 CorruptedSnapshotException lastEx = null;
276 while (tries++ < numTries) {
277 try {
278 admin.snapshot(snapshotName, TableName.valueOf(tableName), type);
279 return;
280 } catch (CorruptedSnapshotException cse) {
281 LOG.warn("Got CorruptedSnapshotException", cse);
282 lastEx = cse;
283 }
284 }
285 throw lastEx;
286 }
287
288 public static void cleanupSnapshot(Admin admin, byte[] tableName)
289 throws IOException {
290 SnapshotTestingUtils.cleanupSnapshot(admin, Bytes.toString(tableName));
291 }
292
293 public static void cleanupSnapshot(Admin admin, String snapshotName)
294 throws IOException {
295
296 admin.deleteSnapshot(snapshotName);
297 assertNoSnapshots(admin);
298 }
299
300
301
302
303
304
305
306
307
308 public static void expectSnapshotDoneException(HMaster master,
309 IsSnapshotDoneRequest snapshot,
310 Class<? extends HBaseSnapshotException> clazz) {
311 try {
312 master.getMasterRpcServices().isSnapshotDone(null, snapshot);
313 Assert.fail("didn't fail to lookup a snapshot");
314 } catch (ServiceException se) {
315 try {
316 throw ProtobufUtil.getRemoteException(se);
317 } catch (HBaseSnapshotException e) {
318 assertEquals("Threw wrong snapshot exception!", clazz, e.getClass());
319 } catch (Throwable t) {
320 Assert.fail("Threw an unexpected exception:" + t);
321 }
322 }
323 }
324
325
326
327
328
329
330
331
332
333 public static Path[] listHFiles(final FileSystem fs, final Path tableDir)
334 throws IOException {
335 final ArrayList<Path> hfiles = new ArrayList<Path>();
336 FSVisitor.visitTableStoreFiles(fs, tableDir, new FSVisitor.StoreFileVisitor() {
337 @Override
338 public void storeFile(final String region, final String family, final String hfileName)
339 throws IOException {
340 hfiles.add(new Path(tableDir, new Path(region, new Path(family, hfileName))));
341 }
342 });
343 return hfiles.toArray(new Path[hfiles.size()]);
344 }
345
346 public static String[] listHFileNames(final FileSystem fs, final Path tableDir)
347 throws IOException {
348 Path[] files = listHFiles(fs, tableDir);
349 String[] names = new String[files.length];
350 for (int i = 0; i < files.length; ++i) {
351 names[i] = files[i].getName();
352 }
353 Arrays.sort(names);
354 return names;
355 }
356
357
358
359
360
361
362 public static void createSnapshotAndValidate(Admin admin,
363 TableName tableName, String familyName, String snapshotNameString,
364 Path rootDir, FileSystem fs, boolean onlineSnapshot)
365 throws Exception {
366 ArrayList<byte[]> nonEmptyFamilyNames = new ArrayList<byte[]>(1);
367 nonEmptyFamilyNames.add(Bytes.toBytes(familyName));
368 createSnapshotAndValidate(admin, tableName, nonEmptyFamilyNames,
369 snapshotNameString, rootDir, fs, onlineSnapshot);
370 }
371
372
373
374
375
376 public static void createSnapshotAndValidate(Admin admin,
377 TableName tableName, List<byte[]> nonEmptyFamilyNames, List<byte[]> emptyFamilyNames,
378 String snapshotNameString, Path rootDir, FileSystem fs, boolean onlineSnapshot)
379 throws Exception {
380 if (!onlineSnapshot) {
381 try {
382 admin.disableTable(tableName);
383 } catch (TableNotEnabledException tne) {
384 LOG.info("In attempting to disable " + tableName + " it turns out that the this table is " +
385 "already disabled.");
386 }
387 }
388 admin.snapshot(snapshotNameString, tableName);
389
390 List<SnapshotDescription> snapshots = SnapshotTestingUtils.assertExistsMatchingSnapshot(admin,
391 snapshotNameString, tableName);
392 if (snapshots == null || snapshots.size() != 1) {
393 Assert.fail("Incorrect number of snapshots for table " + tableName);
394 }
395
396 SnapshotTestingUtils.confirmSnapshotValid(snapshots.get(0), tableName, nonEmptyFamilyNames,
397 emptyFamilyNames, rootDir, admin, fs);
398 }
399
400
401
402
403
404
405
406
407
408 public static ArrayList corruptSnapshot(final HBaseTestingUtility util, final String snapshotName)
409 throws IOException {
410 final MasterFileSystem mfs = util.getHBaseCluster().getMaster().getMasterFileSystem();
411 final FileSystem fs = mfs.getFileSystem();
412
413 Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName,
414 mfs.getRootDir());
415 SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
416 final TableName table = TableName.valueOf(snapshotDesc.getTable());
417
418 final ArrayList corruptedFiles = new ArrayList();
419 final Configuration conf = util.getConfiguration();
420 SnapshotReferenceUtil.visitTableStoreFiles(conf, fs, snapshotDir, snapshotDesc,
421 new SnapshotReferenceUtil.StoreFileVisitor() {
422 @Override
423 public void storeFile(final HRegionInfo regionInfo, final String family,
424 final SnapshotRegionManifest.StoreFile storeFile) throws IOException {
425 String region = regionInfo.getEncodedName();
426 String hfile = storeFile.getName();
427 HFileLink link = HFileLink.create(conf, table, region, family, hfile);
428 if (corruptedFiles.size() % 2 == 0) {
429 fs.delete(link.getAvailablePath(fs), true);
430 corruptedFiles.add(hfile);
431 }
432 }
433 });
434
435 assertTrue(corruptedFiles.size() > 0);
436 return corruptedFiles;
437 }
438
439
440
441
442 public static class SnapshotMock {
443 private final static String TEST_FAMILY = "cf";
444 public final static int TEST_NUM_REGIONS = 4;
445
446 private final Configuration conf;
447 private final FileSystem fs;
448 private final Path rootDir;
449
450 static class RegionData {
451 public HRegionInfo hri;
452 public Path tableDir;
453 public Path[] files;
454
455 public RegionData(final Path tableDir, final HRegionInfo hri, final int nfiles) {
456 this.tableDir = tableDir;
457 this.hri = hri;
458 this.files = new Path[nfiles];
459 }
460 }
461
462 public static class SnapshotBuilder {
463 private final RegionData[] tableRegions;
464 private final SnapshotDescription desc;
465 private final HTableDescriptor htd;
466 private final Configuration conf;
467 private final FileSystem fs;
468 private final Path rootDir;
469 private Path snapshotDir;
470 private int snapshotted = 0;
471
472 public SnapshotBuilder(final Configuration conf, final FileSystem fs,
473 final Path rootDir, final HTableDescriptor htd,
474 final SnapshotDescription desc, final RegionData[] tableRegions)
475 throws IOException {
476 this.fs = fs;
477 this.conf = conf;
478 this.rootDir = rootDir;
479 this.htd = htd;
480 this.desc = desc;
481 this.tableRegions = tableRegions;
482 this.snapshotDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
483 new FSTableDescriptors(conf)
484 .createTableDescriptorForTableDirectory(snapshotDir, htd, false);
485 }
486
487 public HTableDescriptor getTableDescriptor() {
488 return this.htd;
489 }
490
491 public SnapshotDescription getSnapshotDescription() {
492 return this.desc;
493 }
494
495 public Path getSnapshotsDir() {
496 return this.snapshotDir;
497 }
498
499 public Path[] addRegion() throws IOException {
500 return addRegion(desc);
501 }
502
503 public Path[] addRegionV1() throws IOException {
504 return addRegion(desc.toBuilder()
505 .setVersion(SnapshotManifestV1.DESCRIPTOR_VERSION)
506 .build());
507 }
508
509 public Path[] addRegionV2() throws IOException {
510 return addRegion(desc.toBuilder()
511 .setVersion(SnapshotManifestV2.DESCRIPTOR_VERSION)
512 .build());
513 }
514
515 private Path[] addRegion(final SnapshotDescription desc) throws IOException {
516 if (this.snapshotted == tableRegions.length) {
517 throw new UnsupportedOperationException("No more regions in the table");
518 }
519
520 RegionData regionData = tableRegions[this.snapshotted++];
521 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
522 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
523 manifest.addRegion(regionData.tableDir, regionData.hri);
524 return regionData.files;
525 }
526
527 public Path commit() throws IOException {
528 ForeignExceptionDispatcher monitor = new ForeignExceptionDispatcher(desc.getName());
529 SnapshotManifest manifest = SnapshotManifest.create(conf, fs, snapshotDir, desc, monitor);
530 manifest.addTableDescriptor(htd);
531 manifest.consolidate();
532 SnapshotDescriptionUtils.completeSnapshot(desc, rootDir, snapshotDir, fs);
533 snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(desc, rootDir);
534 return snapshotDir;
535 }
536 }
537
538 public SnapshotMock(final Configuration conf, final FileSystem fs, final Path rootDir) {
539 this.fs = fs;
540 this.conf = conf;
541 this.rootDir = rootDir;
542 }
543
544 public SnapshotBuilder createSnapshotV1(final String snapshotName) throws IOException {
545 return createSnapshot(snapshotName, SnapshotManifestV1.DESCRIPTOR_VERSION);
546 }
547
548 public SnapshotBuilder createSnapshotV2(final String snapshotName) throws IOException {
549 return createSnapshot(snapshotName, SnapshotManifestV2.DESCRIPTOR_VERSION);
550 }
551
552 private SnapshotBuilder createSnapshot(final String snapshotName, final int version)
553 throws IOException {
554 HTableDescriptor htd = createHtd(snapshotName);
555
556 RegionData[] regions = createTable(htd, TEST_NUM_REGIONS);
557
558 SnapshotDescription desc = SnapshotDescription.newBuilder()
559 .setTable(htd.getNameAsString())
560 .setName(snapshotName)
561 .setVersion(version)
562 .build();
563
564 Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
565 SnapshotDescriptionUtils.writeSnapshotInfo(desc, workingDir, fs);
566 return new SnapshotBuilder(conf, fs, rootDir, htd, desc, regions);
567 }
568
569 public HTableDescriptor createHtd(final String tableName) {
570 HTableDescriptor htd = new HTableDescriptor(tableName);
571 htd.addFamily(new HColumnDescriptor(TEST_FAMILY));
572 return htd;
573 }
574
575 private RegionData[] createTable(final HTableDescriptor htd, final int nregions)
576 throws IOException {
577 Path tableDir = FSUtils.getTableDir(rootDir, htd.getTableName());
578 new FSTableDescriptors(conf).createTableDescriptorForTableDirectory(tableDir, htd, false);
579
580 assertTrue(nregions % 2 == 0);
581 RegionData[] regions = new RegionData[nregions];
582 for (int i = 0; i < regions.length; i += 2) {
583 byte[] startKey = Bytes.toBytes(0 + i * 2);
584 byte[] endKey = Bytes.toBytes(1 + i * 2);
585
586
587 HRegionInfo hri = new HRegionInfo(htd.getTableName(), startKey, endKey);
588 HRegionFileSystem rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
589 regions[i] = new RegionData(tableDir, hri, 3);
590 for (int j = 0; j < regions[i].files.length; ++j) {
591 Path storeFile = createStoreFile(rfs.createTempName());
592 regions[i].files[j] = rfs.commitStoreFile(TEST_FAMILY, storeFile);
593 }
594
595
596
597 startKey = Bytes.toBytes(2 + i * 2);
598 endKey = Bytes.toBytes(3 + i * 2);
599 hri = new HRegionInfo(htd.getTableName());
600 rfs = HRegionFileSystem.createRegionOnFileSystem(conf, fs, tableDir, hri);
601 regions[i+1] = new RegionData(tableDir, hri, regions[i].files.length);
602 for (int j = 0; j < regions[i].files.length; ++j) {
603 String refName = regions[i].files[j].getName() + '.' + regions[i].hri.getEncodedName();
604 Path refFile = createStoreFile(new Path(rootDir, refName));
605 regions[i+1].files[j] = rfs.commitStoreFile(TEST_FAMILY, refFile);
606 }
607 }
608 return regions;
609 }
610
611 private Path createStoreFile(final Path storeFile)
612 throws IOException {
613 FSDataOutputStream out = fs.create(storeFile);
614 try {
615 out.write(Bytes.toBytes(storeFile.toString()));
616 } finally {
617 out.close();
618 }
619 return storeFile;
620 }
621 }
622
623
624
625
626 public static void waitForTableToBeOnline(final HBaseTestingUtility util,
627 final TableName tableName)
628 throws IOException, InterruptedException {
629 HRegionServer rs = util.getRSForFirstRegionInTable(tableName);
630 List<HRegion> onlineRegions = rs.getOnlineRegions(tableName);
631 for (HRegion region : onlineRegions) {
632 region.waitForFlushesAndCompactions();
633 }
634
635 final HBaseAdmin hBaseAdmin = util.getHBaseAdmin();
636 util.waitFor(60000, new Waiter.Predicate<IOException>() {
637 @Override
638 public boolean evaluate() throws IOException {
639 return hBaseAdmin.isTableAvailable(tableName);
640 }
641 });
642 }
643
644 public static void createTable(final HBaseTestingUtility util, final TableName tableName,
645 int regionReplication, final byte[]... families) throws IOException, InterruptedException {
646 HTableDescriptor htd = new HTableDescriptor(tableName);
647 htd.setRegionReplication(regionReplication);
648 for (byte[] family : families) {
649 HColumnDescriptor hcd = new HColumnDescriptor(family);
650 htd.addFamily(hcd);
651 }
652 byte[][] splitKeys = getSplitKeys();
653 util.createTable(htd, splitKeys);
654 assertEquals((splitKeys.length + 1) * regionReplication,
655 util.getHBaseAdmin().getTableRegions(tableName).size());
656 }
657
658 public static byte[][] getSplitKeys() {
659 byte[][] splitKeys = new byte[KEYS.length-2][];
660 for (int i = 0; i < splitKeys.length; ++i) {
661 splitKeys[i] = new byte[] { KEYS[i+1] };
662 }
663 return splitKeys;
664 }
665
666 public static void createTable(final HBaseTestingUtility util, final TableName tableName,
667 final byte[]... families) throws IOException, InterruptedException {
668 createTable(util, tableName, 1, families);
669 }
670
671 public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
672 byte[]... families) throws IOException, InterruptedException {
673 try (BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName)) {
674 loadData(util, mutator, rows, families);
675 }
676 }
677
678 public static void loadData(final HBaseTestingUtility util, final BufferedMutator mutator, int rows,
679 byte[]... families) throws IOException, InterruptedException {
680
681 assertTrue(rows >= KEYS.length);
682 for (byte k0: KEYS) {
683 byte[] k = new byte[] { k0 };
684 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), k);
685 byte[] key = Bytes.add(k, Bytes.toBytes(MD5Hash.getMD5AsHex(value)));
686 final byte[][] families1 = families;
687 final byte[] key1 = key;
688 final byte[] value1 = value;
689 mutator.mutate(createPut(families1, key1, value1));
690 rows--;
691 }
692
693
694 while (rows-- > 0) {
695 byte[] value = Bytes.add(Bytes.toBytes(System.currentTimeMillis()), Bytes.toBytes(rows));
696 byte[] key = Bytes.toBytes(MD5Hash.getMD5AsHex(value));
697 final byte[][] families1 = families;
698 final byte[] key1 = key;
699 final byte[] value1 = value;
700 mutator.mutate(createPut(families1, key1, value1));
701 }
702 mutator.flush();
703
704 waitForTableToBeOnline(util, mutator.getName());
705 }
706
707 private static Put createPut(final byte[][] families, final byte[] key, final byte[] value) {
708 byte[] q = Bytes.toBytes("q");
709 Put put = new Put(key);
710 put.setDurability(Durability.SKIP_WAL);
711 for (byte[] family: families) {
712 put.add(family, q, value);
713 }
714 return put;
715 }
716
717 public static void deleteAllSnapshots(final Admin admin)
718 throws IOException {
719
720 for (SnapshotDescription snapshot: admin.listSnapshots()) {
721 admin.deleteSnapshot(snapshot.getName());
722 }
723 SnapshotTestingUtils.assertNoSnapshots(admin);
724 }
725
726 public static void deleteArchiveDirectory(final HBaseTestingUtility util)
727 throws IOException {
728
729 MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
730 Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
731 mfs.getFileSystem().delete(archiveDir, true);
732 }
733
734 public static void verifyRowCount(final HBaseTestingUtility util, final TableName tableName,
735 long expectedRows) throws IOException {
736 Table table = new HTable(util.getConfiguration(), tableName);
737 try {
738 assertEquals(expectedRows, util.countRows(table));
739 } finally {
740 table.close();
741 }
742 }
743
744 public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
745 int regionReplication) throws IOException {
746 List<HRegionInfo> regions = admin.getTableRegions(tableName);
747 HashSet<HRegionInfo> set = new HashSet<HRegionInfo>();
748 for (HRegionInfo hri : regions) {
749 set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
750 for (int i = 0; i < regionReplication; i++) {
751 HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
752 if (!regions.contains(replica)) {
753 Assert.fail(replica + " is not contained in the list of online regions");
754 }
755 }
756 }
757 assert(set.size() == getSplitKeys().length + 1);
758 }
759 }