1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.mapreduce;
19
20 import static org.junit.Assert.assertEquals;
21 import static org.junit.Assert.assertTrue;
22 import static org.junit.Assert.fail;
23
24 import java.io.IOException;
25 import java.nio.ByteBuffer;
26 import java.util.Collection;
27 import java.util.Deque;
28 import java.util.List;
29 import java.util.NavigableMap;
30 import java.util.concurrent.ExecutorService;
31 import java.util.concurrent.atomic.AtomicInteger;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.hadoop.conf.Configuration;
36 import org.apache.hadoop.fs.FileSystem;
37 import org.apache.hadoop.fs.Path;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HConstants;
41 import org.apache.hadoop.hbase.HRegionInfo;
42 import org.apache.hadoop.hbase.HRegionLocation;
43 import org.apache.hadoop.hbase.HTableDescriptor;
44 import org.apache.hadoop.hbase.LargeTests;
45 import org.apache.hadoop.hbase.TableExistsException;
46 import org.apache.hadoop.hbase.catalog.CatalogTracker;
47 import org.apache.hadoop.hbase.catalog.MetaEditor;
48 import org.apache.hadoop.hbase.catalog.MetaReader;
49 import org.apache.hadoop.hbase.client.HConnection;
50 import org.apache.hadoop.hbase.client.HTable;
51 import org.apache.hadoop.hbase.client.Result;
52 import org.apache.hadoop.hbase.client.ResultScanner;
53 import org.apache.hadoop.hbase.client.Scan;
54 import org.apache.hadoop.hbase.ipc.HRegionInterface;
55 import org.apache.hadoop.hbase.regionserver.HRegionServer;
56 import org.apache.hadoop.hbase.regionserver.TestHRegionServerBulkLoad;
57 import org.apache.hadoop.hbase.util.Bytes;
58 import org.apache.hadoop.hbase.util.Pair;
59 import org.junit.AfterClass;
60 import org.junit.BeforeClass;
61 import org.junit.Test;
62 import org.junit.experimental.categories.Category;
63 import org.mockito.Mockito;
64
65 import com.google.common.collect.Multimap;
66
67
68
69
70 @Category(LargeTests.class)
71 public class TestLoadIncrementalHFilesSplitRecovery {
72 final static Log LOG = LogFactory.getLog(TestHRegionServerBulkLoad.class);
73
74 static HBaseTestingUtility util;
75
76 final static int NUM_CFS = 10;
77 final static byte[] QUAL = Bytes.toBytes("qual");
78 final static int ROWCOUNT = 100;
79
80 protected static boolean useSecureHBaseOverride = false;
81
82 private final static byte[][] families = new byte[NUM_CFS][];
83 static {
84 for (int i = 0; i < NUM_CFS; i++) {
85 families[i] = Bytes.toBytes(family(i));
86 }
87 }
88
89 static byte[] rowkey(int i) {
90 return Bytes.toBytes(String.format("row_%08d", i));
91 }
92
93 static String family(int i) {
94 return String.format("family_%04d", i);
95 }
96
97 static byte[] value(int i) {
98 return Bytes.toBytes(String.format("%010d", i));
99 }
100
101 public static void buildHFiles(FileSystem fs, Path dir, int value)
102 throws IOException {
103 byte[] val = value(value);
104 for (int i = 0; i < NUM_CFS; i++) {
105 Path testIn = new Path(dir, family(i));
106
107 TestHRegionServerBulkLoad.createHFile(fs, new Path(testIn, "hfile_" + i),
108 Bytes.toBytes(family(i)), QUAL, val, ROWCOUNT);
109 }
110 }
111
112
113
114
115
116 private void setupTable(String table, int cfs) throws IOException {
117 try {
118 LOG.info("Creating table " + table);
119 HTableDescriptor htd = new HTableDescriptor(table);
120 for (int i = 0; i < 10; i++) {
121 htd.addFamily(new HColumnDescriptor(family(i)));
122 }
123
124 util.getHBaseAdmin().createTable(htd);
125 } catch (TableExistsException tee) {
126 LOG.info("Table " + table + " already exists");
127 }
128 }
129
130
131
132
133
134
135
136
137 private void setupTableWithSplitkeys(String table, int cfs, byte[][] SPLIT_KEYS)
138 throws IOException {
139 try {
140 LOG.info("Creating table " + table);
141 HTableDescriptor htd = new HTableDescriptor(table);
142 for (int i = 0; i < cfs; i++) {
143 htd.addFamily(new HColumnDescriptor(family(i)));
144 }
145
146 util.getHBaseAdmin().createTable(htd, SPLIT_KEYS);
147 } catch (TableExistsException tee) {
148 LOG.info("Table " + table + " already exists");
149 }
150 }
151
152
153 private Path buildBulkFiles(String table, int value) throws Exception {
154 Path dir = util.getDataTestDir(table);
155 Path bulk1 = new Path(dir, table+value);
156 FileSystem fs = util.getTestFileSystem();
157 buildHFiles(fs, bulk1, value);
158 return bulk1;
159 }
160
161
162
163
164 private void populateTable(String table, int value) throws Exception {
165
166 LoadIncrementalHFiles lih =
167 new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride);
168 Path bulk1 = buildBulkFiles(table, value);
169 HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
170 lih.doBulkLoad(bulk1, t);
171 }
172
173
174
175
176 private void forceSplit(String table) {
177 try {
178
179 HRegionServer hrs = util.getRSForFirstRegionInTable(Bytes
180 .toBytes(table));
181
182 for (HRegionInfo hri : hrs.getOnlineRegions()) {
183 if (Bytes.equals(hri.getTableName(), Bytes.toBytes(table))) {
184
185 hrs.splitRegion(hri, rowkey(ROWCOUNT / 2));
186 }
187 }
188
189
190 int regions;
191 do {
192 regions = 0;
193 for (HRegionInfo hri : hrs.getOnlineRegions()) {
194 if (Bytes.equals(hri.getTableName(), Bytes.toBytes(table))) {
195 regions++;
196 }
197 }
198 if (regions != 2) {
199 LOG.info("Taking some time to complete split...");
200 Thread.sleep(250);
201 }
202 } while (regions != 2);
203 } catch (IOException e) {
204 e.printStackTrace();
205 } catch (InterruptedException e) {
206 e.printStackTrace();
207 }
208 }
209
210 @BeforeClass
211 public static void setupCluster() throws Exception {
212 util = new HBaseTestingUtility();
213 util.startMiniCluster(1);
214 }
215
216 @AfterClass
217 public static void teardownCluster() throws Exception {
218 util.shutdownMiniCluster();
219 }
220
221
222
223
224
225 void assertExpectedTable(String table, int count, int value) {
226 try {
227 assertEquals(util.getHBaseAdmin().listTables(table).length, 1);
228
229 HTable t = new HTable(util.getConfiguration(), table);
230 Scan s = new Scan();
231 ResultScanner sr = t.getScanner(s);
232 int i = 0;
233 for (Result r : sr) {
234 i++;
235 for (NavigableMap<byte[], byte[]> nm : r.getNoVersionMap().values()) {
236 for (byte[] val : nm.values()) {
237 assertTrue(Bytes.equals(val, value(value)));
238 }
239 }
240 }
241 assertEquals(count, i);
242 } catch (IOException e) {
243 fail("Failed due to exception");
244 }
245 }
246
247
248
249
250
251 @Test(expected=IOException.class)
252 public void testBulkLoadPhaseFailure() throws Exception {
253 String table = "bulkLoadPhaseFailure";
254 setupTable(table, 10);
255
256 final AtomicInteger attmptedCalls = new AtomicInteger();
257 final AtomicInteger failedCalls = new AtomicInteger();
258 LoadIncrementalHFiles lih = new LoadIncrementalHFiles(
259 util.getConfiguration(), useSecureHBaseOverride) {
260
261 protected List<LoadQueueItem> tryAtomicRegionLoad(final HConnection conn,
262 byte[] tableName, final byte[] first, Collection<LoadQueueItem> lqis)
263 throws IOException {
264 int i = attmptedCalls.incrementAndGet();
265 if (i == 1) {
266 HConnection errConn = null;
267 try {
268 errConn = getMockedConnection(util.getConfiguration());
269 } catch (Exception e) {
270 LOG.fatal("mocking cruft, should never happen", e);
271 throw new RuntimeException("mocking cruft, should never happen");
272 }
273 failedCalls.incrementAndGet();
274 return super.tryAtomicRegionLoad(errConn, tableName, first, lqis);
275 }
276
277 return super.tryAtomicRegionLoad(conn, tableName, first, lqis);
278 }
279 };
280
281
282 Path dir = buildBulkFiles(table, 1);
283 HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
284 lih.doBulkLoad(dir, t);
285
286 fail("doBulkLoad should have thrown an exception");
287 }
288
289 private HConnection getMockedConnection(final Configuration conf)
290 throws IOException {
291 HConnection c = Mockito.mock(HConnection.class);
292 Mockito.when(c.getConfiguration()).thenReturn(conf);
293 Mockito.doNothing().when(c).close();
294
295 final HRegionLocation loc = new HRegionLocation(HRegionInfo.FIRST_META_REGIONINFO,
296 "example.org", 1234);
297 Mockito.when(c.getRegionLocation((byte[]) Mockito.any(),
298 (byte[]) Mockito.any(), Mockito.anyBoolean())).
299 thenReturn(loc);
300 Mockito.when(c.locateRegion((byte[]) Mockito.any(), (byte[]) Mockito.any())).
301 thenReturn(loc);
302 HRegionInterface hri = Mockito.mock(HRegionInterface.class);
303 Mockito.when(hri.bulkLoadHFiles(Mockito.anyList(), (byte [])Mockito.any(),
304 Mockito.anyBoolean())).thenThrow(new IOException("injecting bulk load error"));
305 Mockito.when(c.getHRegionConnection(Mockito.anyString(), Mockito.anyInt())).
306 thenReturn(hri);
307 return c;
308 }
309
310
311
312
313
314
315
316 @Test
317 public void testSplitWhileBulkLoadPhase() throws Exception {
318 final String table = "splitWhileBulkloadPhase";
319 setupTable(table, 10);
320 populateTable(table,1);
321 assertExpectedTable(table, ROWCOUNT, 1);
322
323
324
325 final AtomicInteger attemptedCalls = new AtomicInteger();
326 LoadIncrementalHFiles lih2 =
327 new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
328
329 protected void bulkLoadPhase(final HTable htable, final HConnection conn,
330 ExecutorService pool, Deque<LoadQueueItem> queue,
331 final Multimap<ByteBuffer, LoadQueueItem> regionGroups) throws IOException {
332 int i = attemptedCalls.incrementAndGet();
333 if (i == 1) {
334
335 forceSplit(table);
336 }
337
338 super.bulkLoadPhase(htable, conn, pool, queue, regionGroups);
339 }
340 };
341
342
343 HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
344 Path bulk = buildBulkFiles(table, 2);
345 lih2.doBulkLoad(bulk, t);
346
347
348
349
350 assertEquals(attemptedCalls.get(), 3);
351 assertExpectedTable(table, ROWCOUNT, 2);
352 }
353
354
355
356
357
358 @Test
359 public void testGroupOrSplitPresplit() throws Exception {
360 final String table = "groupOrSplitPresplit";
361 setupTable(table, 10);
362 populateTable(table, 1);
363 assertExpectedTable(table, ROWCOUNT, 1);
364 forceSplit(table);
365
366 final AtomicInteger countedLqis= new AtomicInteger();
367 LoadIncrementalHFiles lih =
368 new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
369 protected List<LoadQueueItem> groupOrSplit(
370 Multimap<ByteBuffer, LoadQueueItem> regionGroups,
371 final LoadQueueItem item, final HTable htable,
372 final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
373 List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
374 if (lqis != null) {
375 countedLqis.addAndGet(lqis.size());
376 }
377 return lqis;
378 }
379 };
380
381
382 Path bulk = buildBulkFiles(table, 2);
383 HTable ht = new HTable(util.getConfiguration(), Bytes.toBytes(table));
384 lih.doBulkLoad(bulk, ht);
385
386 assertExpectedTable(table, ROWCOUNT, 2);
387 assertEquals(20, countedLqis.get());
388 }
389
390
391
392
393
394 @Test(expected = IOException.class)
395 public void testGroupOrSplitFailure() throws Exception {
396 String table = "groupOrSplitFailure";
397 setupTable(table, 10);
398
399 LoadIncrementalHFiles lih =
400 new LoadIncrementalHFiles(util.getConfiguration(), useSecureHBaseOverride) {
401 int i = 0;
402
403 protected List<LoadQueueItem> groupOrSplit(
404 Multimap<ByteBuffer, LoadQueueItem> regionGroups,
405 final LoadQueueItem item, final HTable table,
406 final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
407 i++;
408
409 if (i == 5) {
410 throw new IOException("failure");
411 }
412 return super.groupOrSplit(regionGroups, item, table, startEndKeys);
413 }
414 };
415
416
417 Path dir = buildBulkFiles(table,1);
418 HTable t = new HTable(util.getConfiguration(), Bytes.toBytes(table));
419 lih.doBulkLoad(dir, t);
420
421 fail("doBulkLoad should have thrown an exception");
422 }
423
424 @Test
425 public void testGroupOrSplitWhenRegionHoleExistsInMeta() throws Exception {
426 String tableName = "testGroupOrSplitWhenRegionHoleExistsInMeta";
427 byte[][] SPLIT_KEYS = new byte[][] { Bytes.toBytes("row_00000100") };
428
429 setupTableWithSplitkeys(tableName, 10, SPLIT_KEYS);
430 HTable table = new HTable(util.getConfiguration(), Bytes.toBytes(tableName));
431 Path dir = buildBulkFiles(tableName, 2);
432
433 final AtomicInteger countedLqis = new AtomicInteger();
434 LoadIncrementalHFiles loader = new LoadIncrementalHFiles(
435 util.getConfiguration()) {
436
437 protected List<LoadQueueItem> groupOrSplit(
438 Multimap<ByteBuffer, LoadQueueItem> regionGroups,
439 final LoadQueueItem item, final HTable htable,
440 final Pair<byte[][], byte[][]> startEndKeys) throws IOException {
441 List<LoadQueueItem> lqis = super.groupOrSplit(regionGroups, item, htable, startEndKeys);
442 if (lqis != null) {
443 countedLqis.addAndGet(lqis.size());
444 }
445 return lqis;
446 }
447 };
448
449
450 try {
451 loader.doBulkLoad(dir, table);
452 } catch (Exception e) {
453 LOG.error("exeception=", e);
454 }
455
456 this.assertExpectedTable(tableName, ROWCOUNT, 2);
457
458 dir = buildBulkFiles(tableName, 3);
459
460
461 CatalogTracker ct = new CatalogTracker(util.getConfiguration());
462 List<HRegionInfo> regionInfos = MetaReader.getTableRegions(ct, Bytes.toBytes(tableName));
463 for (HRegionInfo regionInfo : regionInfos) {
464 if (Bytes.equals(regionInfo.getStartKey(), HConstants.EMPTY_BYTE_ARRAY)) {
465 MetaEditor.deleteRegion(ct, regionInfo);
466 break;
467 }
468 }
469
470 try {
471 loader.doBulkLoad(dir, table);
472 } catch (Exception e) {
473 LOG.error("exeception=", e);
474 assertTrue("IOException expected", e instanceof IOException);
475 }
476
477 table.close();
478
479 this.assertExpectedTable(tableName, ROWCOUNT, 2);
480 }
481
482 @org.junit.Rule
483 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
484 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
485 }
486