1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24 import static org.junit.Assert.fail;
25
26 import java.io.IOException;
27 import java.lang.reflect.Field;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.concurrent.CountDownLatch;
31 import java.util.concurrent.ThreadPoolExecutor;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35 import org.apache.commons.logging.impl.Log4JLogger;
36 import org.apache.hadoop.hbase.Cell;
37 import org.apache.hadoop.hbase.CellUtil;
38 import org.apache.hadoop.hbase.HBaseTestingUtility;
39 import org.apache.hadoop.hbase.MediumTests;
40 import org.apache.hadoop.hbase.Waiter;
41 import org.apache.hadoop.hbase.exceptions.OperationConflictException;
42 import org.apache.hadoop.hbase.ipc.RpcClient;
43 import org.apache.hadoop.hbase.ipc.RpcServer;
44 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
45 import org.apache.hadoop.hbase.util.Bytes;
46 import org.apache.hadoop.hbase.util.JVMClusterUtil;
47 import org.apache.hadoop.hbase.util.Threads;
48 import org.apache.log4j.Level;
49 import org.junit.AfterClass;
50 import org.junit.Assert;
51 import org.junit.Before;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56 @Category(MediumTests.class)
57 public class TestMultiParallel {
58 private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
59
60 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61 private static final byte[] VALUE = Bytes.toBytes("value");
62 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
63 private static final String FAMILY = "family";
64 private static final String TEST_TABLE = "multi_test_table";
65 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
66 private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
67 private static final byte [][] KEYS = makeKeys();
68
69 private static final int slaves = 5;
70
71 @BeforeClass public static void beforeClass() throws Exception {
72
73
74
75
76
77 UTIL.startMiniCluster(slaves);
78 HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
79 UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
80 UTIL.waitTableEnabled(Bytes.toBytes(TEST_TABLE));
81 t.close();
82 }
83
84 @AfterClass public static void afterClass() throws Exception {
85 UTIL.shutdownMiniCluster();
86 }
87
88 @Before public void before() throws Exception {
89 LOG.info("before");
90 if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
91
92 UTIL.getMiniHBaseCluster().getMaster().balance();
93
94
95 UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
96 }
97 HConnection conn = HConnectionManager.getConnection(UTIL.getConfiguration());
98 conn.clearRegionCache();
99 conn.close();
100 LOG.info("before done");
101 }
102
103 private static byte[][] makeKeys() {
104 byte [][] starterKeys = HBaseTestingUtility.KEYS;
105
106
107
108
109
110 int numKeys = (int) ((float) starterKeys.length * 10.33F);
111
112 List<byte[]> keys = new ArrayList<byte[]>();
113 for (int i = 0; i < numKeys; i++) {
114 int kIdx = i % starterKeys.length;
115 byte[] k = starterKeys[kIdx];
116 byte[] cp = new byte[k.length + 1];
117 System.arraycopy(k, 0, cp, 0, k.length);
118 cp[k.length] = new Integer(i % 256).byteValue();
119 keys.add(cp);
120 }
121
122
123
124
125
126 for (int i = 0; i < 100; i++) {
127 int kIdx = i % starterKeys.length;
128 byte[] k = starterKeys[kIdx];
129 byte[] cp = new byte[k.length + 1];
130 System.arraycopy(k, 0, cp, 0, k.length);
131 cp[k.length] = new Integer(i % 256).byteValue();
132 keys.add(cp);
133 }
134 return keys.toArray(new byte [][] {new byte [] {}});
135 }
136
137
138
139
140
141
142
143
144
145
146
147
148 @Test(timeout=300000)
149 public void testActiveThreadsCount() throws Exception{
150 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
151 List<Row> puts = constructPutRequests();
152 table.batch(puts);
153 Field poolField = table.getClass().getDeclaredField("pool");
154 poolField.setAccessible(true);
155 ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
156 assertEquals(slaves, tExecutor.getLargestPoolSize());
157 table.close();
158 }
159
160 @Test(timeout=300000)
161 public void testBatchWithGet() throws Exception {
162 LOG.info("test=testBatchWithGet");
163 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
164
165
166 List<Row> puts = constructPutRequests();
167 table.batch(puts);
168
169
170 List<Row> gets = new ArrayList<Row>();
171 for (byte[] k : KEYS) {
172 Get get = new Get(k);
173 get.addColumn(BYTES_FAMILY, QUALIFIER);
174 gets.add(get);
175 }
176 Result[] multiRes = new Result[gets.size()];
177 table.batch(gets, multiRes);
178
179
180 List<Result> singleRes = new ArrayList<Result>();
181 for (Row get : gets) {
182 singleRes.add(table.get((Get) get));
183 }
184
185 Assert.assertEquals(singleRes.size(), multiRes.length);
186 for (int i = 0; i < singleRes.size(); i++) {
187 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
188 Cell[] singleKvs = singleRes.get(i).rawCells();
189 Cell[] multiKvs = multiRes[i].rawCells();
190 for (int j = 0; j < singleKvs.length; j++) {
191 Assert.assertEquals(singleKvs[j], multiKvs[j]);
192 Assert.assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(singleKvs[j]),
193 CellUtil.cloneValue(multiKvs[j])));
194 }
195 }
196 table.close();
197 }
198
199 @Test
200 public void testBadFam() throws Exception {
201 LOG.info("test=testBadFam");
202 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
203
204 List<Row> actions = new ArrayList<Row>();
205 Put p = new Put(Bytes.toBytes("row1"));
206 p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
207 actions.add(p);
208 p = new Put(Bytes.toBytes("row2"));
209 p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
210 actions.add(p);
211
212
213
214 Object [] r = new Object[actions.size()];
215 try {
216 table.batch(actions, r);
217 fail();
218 } catch (RetriesExhaustedWithDetailsException ex) {
219 LOG.debug(ex);
220
221 assertFalse(ex.mayHaveClusterIssues());
222 }
223 assertEquals(2, r.length);
224 assertTrue(r[0] instanceof Throwable);
225 assertTrue(r[1] instanceof Result);
226 table.close();
227 }
228
229 @Test (timeout=300000)
230 public void testFlushCommitsNoAbort() throws Exception {
231 LOG.info("test=testFlushCommitsNoAbort");
232 doTestFlushCommits(false);
233 }
234
235
236
237
238
239
240
241 @Test (timeout=300000)
242 public void testFlushCommitsWithAbort() throws Exception {
243 LOG.info("test=testFlushCommitsWithAbort");
244 doTestFlushCommits(true);
245 }
246
247
248
249
250
251
252 private void doTestFlushCommits(boolean doAbort) throws Exception {
253
254 LOG.info("get new table");
255 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
256 table.setAutoFlush(false, true);
257 table.setWriteBufferSize(10 * 1024 * 1024);
258
259 LOG.info("constructPutRequests");
260 List<Row> puts = constructPutRequests();
261 for (Row put : puts) {
262 table.put((Put) put);
263 }
264 LOG.info("puts");
265 table.flushCommits();
266 final int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
267 .size();
268 assert liveRScount > 0;
269 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
270 .getLiveRegionServerThreads().get(0);
271 if (doAbort) {
272 liveRS.getRegionServer().abort("Aborting for tests",
273 new Exception("doTestFlushCommits"));
274
275
276
277
278 while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
279 Thread.sleep(10);
280 }
281
282
283 puts = constructPutRequests();
284 for (Row put : puts) {
285 table.put((Put) put);
286 }
287
288 table.flushCommits();
289 }
290
291 LOG.info("validating loaded data");
292 validateLoadedData(table);
293
294
295 List<JVMClusterUtil.RegionServerThread> liveRSs = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
296 int count = 0;
297 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
298 count++;
299 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
300 }
301 LOG.info("Count=" + count);
302 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
303 (doAbort ? (liveRScount - 1) : liveRScount), count);
304 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
305 int regions = ProtobufUtil.getOnlineRegions(t.getRegionServer()).size();
306
307 }
308 if (doAbort) {
309 UTIL.getMiniHBaseCluster().waitOnRegionServer(0);
310 UTIL.waitFor(15 * 1000, new Waiter.Predicate<Exception>() {
311 @Override
312 public boolean evaluate() throws Exception {
313 return UTIL.getMiniHBaseCluster().getMaster()
314 .getClusterStatus().getServersSize() == (liveRScount - 1);
315 }
316 });
317 UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
318 }
319
320 table.close();
321 LOG.info("done");
322 }
323
324 @Test (timeout=300000)
325 public void testBatchWithPut() throws Exception {
326 LOG.info("test=testBatchWithPut");
327 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
328
329
330 List<Row> puts = constructPutRequests();
331
332 Object[] results = table.batch(puts);
333 validateSizeAndEmpty(results, KEYS.length);
334
335 if (true) {
336 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
337 assert liveRScount > 0;
338 JVMClusterUtil.RegionServerThread liveRS =
339 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0);
340 liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut"));
341 puts = constructPutRequests();
342 try {
343 results = table.batch(puts);
344 } catch (RetriesExhaustedWithDetailsException ree) {
345 LOG.info(ree.getExhaustiveDescription());
346 throw ree;
347 } finally {
348 table.close();
349 }
350 validateSizeAndEmpty(results, KEYS.length);
351 }
352
353 validateLoadedData(table);
354 table.close();
355 }
356
357 @Test(timeout=300000)
358 public void testBatchWithDelete() throws Exception {
359 LOG.info("test=testBatchWithDelete");
360 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
361
362
363 List<Row> puts = constructPutRequests();
364 Object[] results = table.batch(puts);
365 validateSizeAndEmpty(results, KEYS.length);
366
367
368 List<Row> deletes = new ArrayList<Row>();
369 for (int i = 0; i < KEYS.length; i++) {
370 Delete delete = new Delete(KEYS[i]);
371 delete.deleteFamily(BYTES_FAMILY);
372 deletes.add(delete);
373 }
374 results = table.batch(deletes);
375 validateSizeAndEmpty(results, KEYS.length);
376
377
378 for (byte[] k : KEYS) {
379 Get get = new Get(k);
380 get.addColumn(BYTES_FAMILY, QUALIFIER);
381 Assert.assertFalse(table.exists(get));
382 }
383 table.close();
384 }
385
386 @Test(timeout=300000)
387 public void testHTableDeleteWithList() throws Exception {
388 LOG.info("test=testHTableDeleteWithList");
389 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
390
391
392 List<Row> puts = constructPutRequests();
393 Object[] results = table.batch(puts);
394 validateSizeAndEmpty(results, KEYS.length);
395
396
397 ArrayList<Delete> deletes = new ArrayList<Delete>();
398 for (int i = 0; i < KEYS.length; i++) {
399 Delete delete = new Delete(KEYS[i]);
400 delete.deleteFamily(BYTES_FAMILY);
401 deletes.add(delete);
402 }
403 table.delete(deletes);
404 Assert.assertTrue(deletes.isEmpty());
405
406
407 for (byte[] k : KEYS) {
408 Get get = new Get(k);
409 get.addColumn(BYTES_FAMILY, QUALIFIER);
410 Assert.assertFalse(table.exists(get));
411 }
412 table.close();
413 }
414
415 @Test(timeout=300000)
416 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
417 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
418 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
419
420 List<Row> puts = new ArrayList<Row>();
421 for (int i = 0; i < 100; i++) {
422 Put put = new Put(ONE_ROW);
423 byte[] qual = Bytes.toBytes("column" + i);
424 put.add(BYTES_FAMILY, qual, VALUE);
425 puts.add(put);
426 }
427 Object[] results = table.batch(puts);
428
429
430 validateSizeAndEmpty(results, 100);
431
432
433 List<Row> gets = new ArrayList<Row>();
434 for (int i = 0; i < 100; i++) {
435 Get get = new Get(ONE_ROW);
436 byte[] qual = Bytes.toBytes("column" + i);
437 get.addColumn(BYTES_FAMILY, qual);
438 gets.add(get);
439 }
440
441 Object[] multiRes = table.batch(gets);
442
443 int idx = 0;
444 for (Object r : multiRes) {
445 byte[] qual = Bytes.toBytes("column" + idx);
446 validateResult(r, qual, VALUE);
447 idx++;
448 }
449 table.close();
450 }
451
452 @Test(timeout=300000)
453 public void testBatchWithIncrementAndAppend() throws Exception {
454 LOG.info("test=testBatchWithIncrementAndAppend");
455 final byte[] QUAL1 = Bytes.toBytes("qual1");
456 final byte[] QUAL2 = Bytes.toBytes("qual2");
457 final byte[] QUAL3 = Bytes.toBytes("qual3");
458 final byte[] QUAL4 = Bytes.toBytes("qual4");
459 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
460 Delete d = new Delete(ONE_ROW);
461 table.delete(d);
462 Put put = new Put(ONE_ROW);
463 put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
464 put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
465 table.put(put);
466
467 Increment inc = new Increment(ONE_ROW);
468 inc.addColumn(BYTES_FAMILY, QUAL2, 1);
469 inc.addColumn(BYTES_FAMILY, QUAL3, 1);
470
471 Append a = new Append(ONE_ROW);
472 a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
473 a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
474 List<Row> actions = new ArrayList<Row>();
475 actions.add(inc);
476 actions.add(a);
477
478 Object[] multiRes = table.batch(actions);
479 validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
480 validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
481 validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
482 validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
483 table.close();
484 }
485
486 @Test(timeout=300000)
487 public void testNonceCollision() throws Exception {
488 LOG.info("test=testNonceCollision");
489 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
490 Put put = new Put(ONE_ROW);
491 put.add(BYTES_FAMILY, QUALIFIER, Bytes.toBytes(0L));
492
493
494 NonceGenerator cnm = new PerClientRandomNonceGenerator() {
495 long lastNonce = -1;
496 @Override
497 public synchronized long newNonce() {
498 long nonce = 0;
499 if (lastNonce == -1) {
500 lastNonce = nonce = super.newNonce();
501 } else {
502 nonce = lastNonce;
503 lastNonce = -1L;
504 }
505 return nonce;
506 }
507 };
508 NonceGenerator oldCnm =
509 HConnectionManager.injectNonceGeneratorForTesting(table.getConnection(), cnm);
510
511
512 try {
513 Increment inc = new Increment(ONE_ROW);
514 inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
515 table.increment(inc);
516 inc = new Increment(ONE_ROW);
517 inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
518 try {
519 table.increment(inc);
520 fail("Should have thrown an exception");
521 } catch (OperationConflictException ex) {
522 }
523 Get get = new Get(ONE_ROW);
524 get.addColumn(BYTES_FAMILY, QUALIFIER);
525 Result result = table.get(get);
526 validateResult(result, QUALIFIER, Bytes.toBytes(1L));
527
528
529 int numRequests = 40;
530 final CountDownLatch startedLatch = new CountDownLatch(numRequests);
531 final CountDownLatch startLatch = new CountDownLatch(1);
532 final CountDownLatch doneLatch = new CountDownLatch(numRequests);
533 for (int i = 0; i < numRequests; ++i) {
534 Runnable r = new Runnable() {
535 @Override
536 public void run() {
537 HTable table = null;
538 try {
539 table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
540 } catch (IOException e) {
541 fail("Not expected");
542 }
543 Increment inc = new Increment(ONE_ROW);
544 inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
545 startedLatch.countDown();
546 try {
547 startLatch.await();
548 } catch (InterruptedException e) {
549 fail("Not expected");
550 }
551 try {
552 table.increment(inc);
553 } catch (OperationConflictException ex) {
554 } catch (IOException ioEx) {
555 fail("Not expected");
556 }
557 doneLatch.countDown();
558 }
559 };
560 Threads.setDaemonThreadRunning(new Thread(r));
561 }
562 startedLatch.await();
563 startLatch.countDown();
564 doneLatch.await();
565
566 get = new Get(ONE_ROW);
567 get.addColumn(BYTES_FAMILY, QUALIFIER);
568 result = table.get(get);
569 validateResult(result, QUALIFIER, Bytes.toBytes((numRequests / 2) + 1L));
570 table.close();
571 } finally {
572 HConnectionManager.injectNonceGeneratorForTesting(table.getConnection(), oldCnm);
573 }
574 }
575
576 @Test(timeout=300000)
577 public void testBatchWithMixedActions() throws Exception {
578 LOG.info("test=testBatchWithMixedActions");
579 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
580
581
582 Object[] results = table.batch(constructPutRequests());
583 validateSizeAndEmpty(results, KEYS.length);
584
585
586
587 List<Row> actions = new ArrayList<Row>();
588
589 byte[] qual2 = Bytes.toBytes("qual2");
590 byte[] val2 = Bytes.toBytes("putvalue2");
591
592
593 Get get = new Get(KEYS[10]);
594 get.addColumn(BYTES_FAMILY, QUALIFIER);
595 actions.add(get);
596
597
598 get = new Get(KEYS[11]);
599 get.addColumn(BYTES_FAMILY, QUALIFIER);
600 actions.add(get);
601
602
603 Put put = new Put(KEYS[10]);
604 put.add(BYTES_FAMILY, qual2, val2);
605 actions.add(put);
606
607
608 Delete delete = new Delete(KEYS[20]);
609 delete.deleteFamily(BYTES_FAMILY);
610 actions.add(delete);
611
612
613 get = new Get(KEYS[30]);
614 get.addColumn(BYTES_FAMILY, QUALIFIER);
615 actions.add(get);
616
617
618
619
620
621
622 put = new Put(KEYS[40]);
623 put.add(BYTES_FAMILY, qual2, val2);
624 actions.add(put);
625
626 results = table.batch(actions);
627
628
629
630 validateResult(results[0]);
631 validateResult(results[1]);
632 validateEmpty(results[2]);
633 validateEmpty(results[3]);
634 validateResult(results[4]);
635 validateEmpty(results[5]);
636
637
638 get = new Get(KEYS[40]);
639 get.addColumn(BYTES_FAMILY, qual2);
640 Result r = table.get(get);
641 validateResult(r, qual2, val2);
642
643 table.close();
644 }
645
646
647
648 private void validateResult(Object r) {
649 validateResult(r, QUALIFIER, VALUE);
650 }
651
652 private void validateResult(Object r1, byte[] qual, byte[] val) {
653 Result r = (Result)r1;
654 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
655 byte[] value = r.getValue(BYTES_FAMILY, qual);
656 if (0 != Bytes.compareTo(val, value)) {
657 fail("Expected [" + Bytes.toStringBinary(val)
658 + "] but got [" + Bytes.toStringBinary(value) + "]");
659 }
660 }
661
662 private List<Row> constructPutRequests() {
663 List<Row> puts = new ArrayList<Row>();
664 for (byte[] k : KEYS) {
665 Put put = new Put(k);
666 put.add(BYTES_FAMILY, QUALIFIER, VALUE);
667 puts.add(put);
668 }
669 return puts;
670 }
671
672 private void validateLoadedData(HTable table) throws IOException {
673
674 for (byte[] k : KEYS) {
675 Get get = new Get(k);
676 get.addColumn(BYTES_FAMILY, QUALIFIER);
677 Result r = table.get(get);
678 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
679 Assert.assertEquals(0, Bytes.compareTo(VALUE, r
680 .getValue(BYTES_FAMILY, QUALIFIER)));
681 }
682 }
683
684 private void validateEmpty(Object r1) {
685 Result result = (Result)r1;
686 Assert.assertTrue(result != null);
687 Assert.assertTrue(result.getRow() == null);
688 Assert.assertEquals(0, result.rawCells().length);
689 }
690
691 private void validateSizeAndEmpty(Object[] results, int expectedSize) {
692
693 Assert.assertEquals(expectedSize, results.length);
694 for (Object result : results) {
695 validateEmpty(result);
696 }
697 }
698 }