1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.lang.reflect.Field;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.ThreadPoolExecutor;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.hbase.util.JVMClusterUtil;
33 import org.junit.AfterClass;
34 import org.junit.Assert;
35 import org.junit.Before;
36 import org.junit.BeforeClass;
37 import org.junit.Test;
38 import org.junit.experimental.categories.Category;
39
40 import static org.junit.Assert.*;
41
42 @Category(MediumTests.class)
43 public class TestMultiParallel {
44 private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
45 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
46 private static final byte[] VALUE = Bytes.toBytes("value");
47 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
48 private static final String FAMILY = "family";
49 private static final String TEST_TABLE = "multi_test_table";
50 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
51 private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
52 private static final byte [][] KEYS = makeKeys();
53
54 private static final int slaves = 2;
55
56 @BeforeClass public static void beforeClass() throws Exception {
57 UTIL.startMiniCluster(slaves);
58 HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
59 UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
60 UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000);
61 t.close();
62 }
63
64 @AfterClass public static void afterClass() throws Exception {
65 UTIL.shutdownMiniCluster();
66 }
67
68 @Before public void before() throws IOException {
69 LOG.info("before");
70 if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
71
72 UTIL.getMiniHBaseCluster().getMaster().balance();
73 }
74 LOG.info("before done");
75 }
76
77 private static byte[][] makeKeys() {
78 byte [][] starterKeys = HBaseTestingUtility.KEYS;
79
80
81
82
83
84 int numKeys = (int) ((float) starterKeys.length * 10.33F);
85
86 List<byte[]> keys = new ArrayList<byte[]>();
87 for (int i = 0; i < numKeys; i++) {
88 int kIdx = i % starterKeys.length;
89 byte[] k = starterKeys[kIdx];
90 byte[] cp = new byte[k.length + 1];
91 System.arraycopy(k, 0, cp, 0, k.length);
92 cp[k.length] = new Integer(i % 256).byteValue();
93 keys.add(cp);
94 }
95
96
97
98
99
100 for (int i = 0; i < 100; i++) {
101 int kIdx = i % starterKeys.length;
102 byte[] k = starterKeys[kIdx];
103 byte[] cp = new byte[k.length + 1];
104 System.arraycopy(k, 0, cp, 0, k.length);
105 cp[k.length] = new Integer(i % 256).byteValue();
106 keys.add(cp);
107 }
108 return keys.toArray(new byte [][] {new byte [] {}});
109 }
110
111
112
113
114
115
116
117
118
119
120
121
122 @Test(timeout=300000)
123 public void testActiveThreadsCount() throws Exception{
124 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
125 List<Row> puts = constructPutRequests();
126 table.batch(puts);
127 Field poolField = table.getClass().getDeclaredField("pool");
128 poolField.setAccessible(true);
129 ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
130 assertEquals(slaves, tExecutor.getLargestPoolSize());
131 table.close();
132 }
133
134 @Test(timeout=300000)
135 public void testBatchWithGet() throws Exception {
136 LOG.info("test=testBatchWithGet");
137 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
138
139
140 List<Row> puts = constructPutRequests();
141 table.batch(puts);
142
143
144 List<Row> gets = new ArrayList<Row>();
145 for (byte[] k : KEYS) {
146 Get get = new Get(k);
147 get.addColumn(BYTES_FAMILY, QUALIFIER);
148 gets.add(get);
149 }
150 Result[] multiRes = new Result[gets.size()];
151 table.batch(gets, multiRes);
152
153
154 List<Result> singleRes = new ArrayList<Result>();
155 for (Row get : gets) {
156 singleRes.add(table.get((Get) get));
157 }
158
159
160 Assert.assertEquals(singleRes.size(), multiRes.length);
161 for (int i = 0; i < singleRes.size(); i++) {
162 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
163 KeyValue[] singleKvs = singleRes.get(i).raw();
164 KeyValue[] multiKvs = multiRes[i].raw();
165 for (int j = 0; j < singleKvs.length; j++) {
166 Assert.assertEquals(singleKvs[j], multiKvs[j]);
167 Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
168 .getValue()));
169 }
170 }
171 table.close();
172 }
173
174 @Test
175 public void testBadFam() throws Exception {
176 LOG.info("test=testBadFam");
177 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
178
179 List<Row> actions = new ArrayList<Row>();
180 Put p = new Put(Bytes.toBytes("row1"));
181 p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
182 actions.add(p);
183 p = new Put(Bytes.toBytes("row2"));
184 p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
185 actions.add(p);
186
187
188
189 Object [] r = new Object[actions.size()];
190 try {
191 table.batch(actions, r);
192 fail();
193 } catch (RetriesExhaustedWithDetailsException ex) {
194 LOG.debug(ex);
195
196 assertFalse(ex.mayHaveClusterIssues());
197 }
198 assertEquals(2, r.length);
199 assertTrue(r[0] instanceof Throwable);
200 assertTrue(r[1] instanceof Result);
201 table.close();
202 }
203
204
205
206
207
208
209
210 @Test (timeout=300000)
211 public void testFlushCommitsWithAbort() throws Exception {
212 LOG.info("test=testFlushCommitsWithAbort");
213 doTestFlushCommits(true);
214 }
215
216 @Test (timeout=300000)
217 public void testFlushCommitsNoAbort() throws Exception {
218 LOG.info("test=testFlushCommitsNoAbort");
219 doTestFlushCommits(false);
220 }
221
222
223
224
225
226
227 private void doTestFlushCommits(boolean doAbort) throws Exception {
228
229 LOG.info("get new table");
230 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
231 table.setAutoFlush(false);
232 table.setWriteBufferSize(10 * 1024 * 1024);
233
234 LOG.info("constructPutRequests");
235 List<Row> puts = constructPutRequests();
236 for (Row put : puts) {
237 table.put((Put) put);
238 }
239 LOG.info("puts");
240 table.flushCommits();
241 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
242 .size();
243 assert liveRScount > 0;
244 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
245 .getLiveRegionServerThreads().get(0);
246 if (doAbort) {
247 LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
248
249
250
251
252
253 while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
254 Thread.sleep(100);
255 }
256 while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) {
257 Thread.sleep(100);
258 }
259
260
261
262 puts = constructPutRequests();
263 for (Row put : puts) {
264 table.put((Put) put);
265 }
266
267 table.flushCommits();
268 }
269
270 LOG.info("validating loaded data");
271 validateLoadedData(table);
272
273
274 List<JVMClusterUtil.RegionServerThread> liveRSs =
275 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
276 int count = 0;
277 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
278 count++;
279 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
280 }
281 LOG.info("Count=" + count);
282 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
283 (doAbort ? (liveRScount - 1) : liveRScount), count);
284 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
285 int regions = t.getRegionServer().getOnlineRegions().size();
286 Assert.assertTrue("Count of regions=" + regions, regions > 10);
287 }
288 table.close();
289 LOG.info("done");
290 }
291
292 @Test (timeout=300000)
293 public void testBatchWithPut() throws Exception {
294 LOG.info("test=testBatchWithPut");
295 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
296
297
298 List<Row> puts = constructPutRequests();
299
300 Object[] results = table.batch(puts);
301 validateSizeAndEmpty(results, KEYS.length);
302
303 if (true) {
304 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
305 .size();
306 assert liveRScount > 0;
307 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
308 .getLiveRegionServerThreads().get(0);
309 liveRS.getRegionServer().abort("Aborting for tests",
310 new Exception("testBatchWithPut"));
311
312 puts = constructPutRequests();
313 results = table.batch(puts);
314 validateSizeAndEmpty(results, KEYS.length);
315 }
316
317 validateLoadedData(table);
318 table.close();
319 }
320
321 @Test(timeout=300000)
322 public void testBatchWithDelete() throws Exception {
323 LOG.info("test=testBatchWithDelete");
324 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
325
326
327 List<Row> puts = constructPutRequests();
328 Object[] results = table.batch(puts);
329 validateSizeAndEmpty(results, KEYS.length);
330
331
332 List<Row> deletes = new ArrayList<Row>();
333 for (int i = 0; i < KEYS.length; i++) {
334 Delete delete = new Delete(KEYS[i]);
335 delete.deleteFamily(BYTES_FAMILY);
336 deletes.add(delete);
337 }
338 results = table.batch(deletes);
339 validateSizeAndEmpty(results, KEYS.length);
340
341
342 for (byte[] k : KEYS) {
343 Get get = new Get(k);
344 get.addColumn(BYTES_FAMILY, QUALIFIER);
345 Assert.assertFalse(table.exists(get));
346 }
347 table.close();
348 }
349
350 @Test(timeout=300000)
351 public void testHTableDeleteWithList() throws Exception {
352 LOG.info("test=testHTableDeleteWithList");
353 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
354
355
356 List<Row> puts = constructPutRequests();
357 Object[] results = table.batch(puts);
358 validateSizeAndEmpty(results, KEYS.length);
359
360
361 ArrayList<Delete> deletes = new ArrayList<Delete>();
362 for (int i = 0; i < KEYS.length; i++) {
363 Delete delete = new Delete(KEYS[i]);
364 delete.deleteFamily(BYTES_FAMILY);
365 deletes.add(delete);
366 }
367 table.delete(deletes);
368 Assert.assertTrue(deletes.isEmpty());
369
370
371 for (byte[] k : KEYS) {
372 Get get = new Get(k);
373 get.addColumn(BYTES_FAMILY, QUALIFIER);
374 Assert.assertFalse(table.exists(get));
375 }
376 table.close();
377 }
378
379 @Test(timeout=300000)
380 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
381 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
382 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
383
384 List<Row> puts = new ArrayList<Row>();
385 for (int i = 0; i < 100; i++) {
386 Put put = new Put(ONE_ROW);
387 byte[] qual = Bytes.toBytes("column" + i);
388 put.add(BYTES_FAMILY, qual, VALUE);
389 puts.add(put);
390 }
391 Object[] results = table.batch(puts);
392
393
394 validateSizeAndEmpty(results, 100);
395
396
397 List<Row> gets = new ArrayList<Row>();
398 for (int i = 0; i < 100; i++) {
399 Get get = new Get(ONE_ROW);
400 byte[] qual = Bytes.toBytes("column" + i);
401 get.addColumn(BYTES_FAMILY, qual);
402 gets.add(get);
403 }
404
405 Object[] multiRes = table.batch(gets);
406
407 int idx = 0;
408 for (Object r : multiRes) {
409 byte[] qual = Bytes.toBytes("column" + idx);
410 validateResult(r, qual, VALUE);
411 idx++;
412 }
413 table.close();
414 }
415
416 @Test(timeout=300000)
417 public void testBatchWithIncrementAndAppend() throws Exception {
418 LOG.info("test=testBatchWithIncrementAndAppend");
419 final byte[] QUAL1 = Bytes.toBytes("qual1");
420 final byte[] QUAL2 = Bytes.toBytes("qual2");
421 final byte[] QUAL3 = Bytes.toBytes("qual3");
422 final byte[] QUAL4 = Bytes.toBytes("qual4");
423 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
424 Delete d = new Delete(ONE_ROW);
425 table.delete(d);
426 Put put = new Put(ONE_ROW);
427 put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
428 put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
429 table.put(put);
430
431 Increment inc = new Increment(ONE_ROW);
432 inc.addColumn(BYTES_FAMILY, QUAL2, 1);
433 inc.addColumn(BYTES_FAMILY, QUAL3, 1);
434
435 Append a = new Append(ONE_ROW);
436 a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
437 a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
438 List<Row> actions = new ArrayList<Row>();
439 actions.add(inc);
440 actions.add(a);
441
442 Object[] multiRes = table.batch(actions);
443 validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
444 validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
445 validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
446 validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
447 table.close();
448 }
449
450 @Test(timeout=300000)
451 public void testBatchWithMixedActions() throws Exception {
452 LOG.info("test=testBatchWithMixedActions");
453 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
454
455
456 Object[] results = table.batch(constructPutRequests());
457 validateSizeAndEmpty(results, KEYS.length);
458
459
460
461 List<Row> actions = new ArrayList<Row>();
462
463 byte[] qual2 = Bytes.toBytes("qual2");
464 byte[] val2 = Bytes.toBytes("putvalue2");
465
466
467 Get get = new Get(KEYS[10]);
468 get.addColumn(BYTES_FAMILY, QUALIFIER);
469 actions.add(get);
470
471
472 get = new Get(KEYS[11]);
473 get.addColumn(BYTES_FAMILY, QUALIFIER);
474 actions.add(get);
475
476
477 Put put = new Put(KEYS[10]);
478 put.add(BYTES_FAMILY, qual2, val2);
479 actions.add(put);
480
481
482 Delete delete = new Delete(KEYS[20]);
483 delete.deleteFamily(BYTES_FAMILY);
484 actions.add(delete);
485
486
487 get = new Get(KEYS[30]);
488 get.addColumn(BYTES_FAMILY, QUALIFIER);
489 actions.add(get);
490
491
492
493
494
495
496 put = new Put(KEYS[40]);
497 put.add(BYTES_FAMILY, qual2, val2);
498 actions.add(put);
499
500 results = table.batch(actions);
501
502
503
504 validateResult(results[0]);
505 validateResult(results[1]);
506 validateEmpty(results[2]);
507 validateEmpty(results[3]);
508 validateResult(results[4]);
509 validateEmpty(results[5]);
510
511
512 get = new Get(KEYS[40]);
513 get.addColumn(BYTES_FAMILY, qual2);
514 Result r = table.get(get);
515 validateResult(r, qual2, val2);
516
517 table.close();
518 }
519
520
521
522 private void validateResult(Object r) {
523 validateResult(r, QUALIFIER, VALUE);
524 }
525
526 private void validateResult(Object r1, byte[] qual, byte[] val) {
527
528 Result r = (Result)r1;
529 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
530 Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
531 }
532
533 private List<Row> constructPutRequests() {
534 List<Row> puts = new ArrayList<Row>();
535 for (byte[] k : KEYS) {
536 Put put = new Put(k);
537 put.add(BYTES_FAMILY, QUALIFIER, VALUE);
538 puts.add(put);
539 }
540 return puts;
541 }
542
543 private void validateLoadedData(HTable table) throws IOException {
544
545 for (byte[] k : KEYS) {
546 Get get = new Get(k);
547 get.addColumn(BYTES_FAMILY, QUALIFIER);
548 Result r = table.get(get);
549 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
550 Assert.assertEquals(0, Bytes.compareTo(VALUE, r
551 .getValue(BYTES_FAMILY, QUALIFIER)));
552 }
553 }
554
555 private void validateEmpty(Object r1) {
556 Result result = (Result)r1;
557 Assert.assertTrue(result != null);
558 Assert.assertTrue(result.getRow() == null);
559 Assert.assertEquals(0, result.raw().length);
560 }
561
562 private void validateSizeAndEmpty(Object[] results, int expectedSize) {
563
564 Assert.assertEquals(expectedSize, results.length);
565 for (Object result : results) {
566 validateEmpty(result);
567 }
568 }
569
570 @org.junit.Rule
571 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
572 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
573 }
574