1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.client;
21
22 import java.io.IOException;
23 import java.lang.reflect.Field;
24 import java.util.ArrayList;
25 import java.util.List;
26 import java.util.concurrent.ThreadPoolExecutor;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.util.Bytes;
32 import org.apache.hadoop.hbase.util.JVMClusterUtil;
33 import org.junit.AfterClass;
34 import org.junit.Assert;
35 import org.junit.Before;
36 import org.junit.BeforeClass;
37 import org.junit.FixMethodOrder;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40 import org.junit.runners.MethodSorters;
41
42 import static org.junit.Assert.*;
43
44 @FixMethodOrder(MethodSorters.NAME_ASCENDING)
45 @Category(MediumTests.class)
46 public class TestMultiParallel {
47 private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
48 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49 private static final byte[] VALUE = Bytes.toBytes("value");
50 private static final byte[] QUALIFIER = Bytes.toBytes("qual");
51 private static final String FAMILY = "family";
52 private static final String TEST_TABLE = "multi_test_table";
53 private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
54 private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
55 private static final byte [][] KEYS = makeKeys();
56
57 private static final int slaves = 2;
58
59 @BeforeClass public static void beforeClass() throws Exception {
60 UTIL.startMiniCluster(slaves);
61 HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
62 UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
63 UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000);
64 t.close();
65 }
66
67 @AfterClass public static void afterClass() throws Exception {
68 UTIL.shutdownMiniCluster();
69 }
70
71 @Before public void before() throws IOException {
72 LOG.info("before");
73 if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
74
75 UTIL.getMiniHBaseCluster().getMaster().balance();
76 }
77 LOG.info("before done");
78 }
79
80 private static byte[][] makeKeys() {
81 byte [][] starterKeys = HBaseTestingUtility.KEYS;
82
83
84
85
86
87 int numKeys = (int) ((float) starterKeys.length * 10.33F);
88
89 List<byte[]> keys = new ArrayList<byte[]>();
90 for (int i = 0; i < numKeys; i++) {
91 int kIdx = i % starterKeys.length;
92 byte[] k = starterKeys[kIdx];
93 byte[] cp = new byte[k.length + 1];
94 System.arraycopy(k, 0, cp, 0, k.length);
95 cp[k.length] = new Integer(i % 256).byteValue();
96 keys.add(cp);
97 }
98
99
100
101
102
103 for (int i = 0; i < 100; i++) {
104 int kIdx = i % starterKeys.length;
105 byte[] k = starterKeys[kIdx];
106 byte[] cp = new byte[k.length + 1];
107 System.arraycopy(k, 0, cp, 0, k.length);
108 cp[k.length] = new Integer(i % 256).byteValue();
109 keys.add(cp);
110 }
111 return keys.toArray(new byte [][] {new byte [] {}});
112 }
113
114
115
116
117
118
119
120
121
122
123
124
125 @Test(timeout=300000)
126 public void testActiveThreadsCount() throws Exception{
127 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
128 List<Row> puts = constructPutRequests();
129 table.batch(puts);
130 Field poolField = table.getClass().getDeclaredField("pool");
131 poolField.setAccessible(true);
132 ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
133 assertEquals(slaves, tExecutor.getLargestPoolSize());
134 table.close();
135 }
136
137 @Test(timeout=300000)
138 public void testBatchWithGet() throws Exception {
139 LOG.info("test=testBatchWithGet");
140 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
141
142
143 List<Row> puts = constructPutRequests();
144 table.batch(puts);
145
146
147 List<Row> gets = new ArrayList<Row>();
148 for (byte[] k : KEYS) {
149 Get get = new Get(k);
150 get.addColumn(BYTES_FAMILY, QUALIFIER);
151 gets.add(get);
152 }
153 Result[] multiRes = new Result[gets.size()];
154 table.batch(gets, multiRes);
155
156
157 List<Result> singleRes = new ArrayList<Result>();
158 for (Row get : gets) {
159 singleRes.add(table.get((Get) get));
160 }
161
162
163 Assert.assertEquals(singleRes.size(), multiRes.length);
164 for (int i = 0; i < singleRes.size(); i++) {
165 Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
166 KeyValue[] singleKvs = singleRes.get(i).raw();
167 KeyValue[] multiKvs = multiRes[i].raw();
168 for (int j = 0; j < singleKvs.length; j++) {
169 Assert.assertEquals(singleKvs[j], multiKvs[j]);
170 Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
171 .getValue()));
172 }
173 }
174 table.close();
175 }
176
177 @Test
178 public void testBadFam() throws Exception {
179 LOG.info("test=testBadFam");
180 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
181
182 List<Row> actions = new ArrayList<Row>();
183 Put p = new Put(Bytes.toBytes("row1"));
184 p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
185 actions.add(p);
186 p = new Put(Bytes.toBytes("row2"));
187 p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
188 actions.add(p);
189
190
191
192 Object [] r = new Object[actions.size()];
193 try {
194 table.batch(actions, r);
195 fail();
196 } catch (RetriesExhaustedWithDetailsException ex) {
197 LOG.debug(ex);
198
199 assertFalse(ex.mayHaveClusterIssues());
200 }
201 assertEquals(2, r.length);
202 assertTrue(r[0] instanceof Throwable);
203 assertTrue(r[1] instanceof Result);
204 table.close();
205 }
206
207
208
209
210
211
212
213 @Test (timeout=300000)
214 public void testFlushCommitsWithAbort() throws Exception {
215 LOG.info("test=testFlushCommitsWithAbort");
216 doTestFlushCommits(true);
217 }
218
219 @Test (timeout=300000)
220 public void testFlushCommitsNoAbort() throws Exception {
221 LOG.info("test=testFlushCommitsNoAbort");
222 doTestFlushCommits(false);
223 }
224
225
226
227
228
229
230 private void doTestFlushCommits(boolean doAbort) throws Exception {
231
232 LOG.info("get new table");
233 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
234 table.setAutoFlush(false);
235 table.setWriteBufferSize(10 * 1024 * 1024);
236
237 LOG.info("constructPutRequests");
238 List<Row> puts = constructPutRequests();
239 for (Row put : puts) {
240 table.put((Put) put);
241 }
242 LOG.info("puts");
243 table.flushCommits();
244 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
245 .size();
246 assert liveRScount > 0;
247 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
248 .getLiveRegionServerThreads().get(0);
249 if (doAbort) {
250 LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
251
252
253
254
255
256 while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
257 Thread.sleep(100);
258 }
259 while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) {
260 Thread.sleep(100);
261 }
262
263
264
265 puts = constructPutRequests();
266 for (Row put : puts) {
267 table.put((Put) put);
268 }
269
270 table.flushCommits();
271 }
272
273 LOG.info("validating loaded data");
274 validateLoadedData(table);
275
276
277 List<JVMClusterUtil.RegionServerThread> liveRSs =
278 UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
279 int count = 0;
280 int regionCount = 0;
281 for (JVMClusterUtil.RegionServerThread t: liveRSs) {
282 count++;
283 regionCount += t.getRegionServer().getOnlineRegions().size();
284 LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
285 }
286 LOG.info("Count=" + count);
287 Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
288 (doAbort ? (liveRScount - 1) : liveRScount), count);
289 Assert.assertTrue("Count of regions=" + regionCount, regionCount >= 25);
290 table.close();
291 LOG.info("done");
292 }
293
294 @Test (timeout=300000)
295
296
297 public void testZBatchWithPut() throws Exception {
298 LOG.info("test=testBatchWithPut");
299 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
300
301
302 List<Row> puts = constructPutRequests();
303
304 Object[] results = table.batch(puts);
305 validateSizeAndEmpty(results, KEYS.length);
306
307 if (true) {
308 int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
309 .size();
310 assert liveRScount > 0;
311 JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
312 .getLiveRegionServerThreads().get(0);
313 liveRS.getRegionServer().abort("Aborting for tests",
314 new Exception("testBatchWithPut"));
315
316 puts = constructPutRequests();
317 results = table.batch(puts);
318 validateSizeAndEmpty(results, KEYS.length);
319 }
320
321 validateLoadedData(table);
322 table.close();
323 }
324
325 @Test(timeout=300000)
326 public void testBatchWithDelete() throws Exception {
327 LOG.info("test=testBatchWithDelete");
328 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
329
330
331 List<Row> puts = constructPutRequests();
332 Object[] results = table.batch(puts);
333 validateSizeAndEmpty(results, KEYS.length);
334
335
336 List<Row> deletes = new ArrayList<Row>();
337 for (int i = 0; i < KEYS.length; i++) {
338 Delete delete = new Delete(KEYS[i]);
339 delete.deleteFamily(BYTES_FAMILY);
340 deletes.add(delete);
341 }
342 results = table.batch(deletes);
343 validateSizeAndEmpty(results, KEYS.length);
344
345
346 for (byte[] k : KEYS) {
347 Get get = new Get(k);
348 get.addColumn(BYTES_FAMILY, QUALIFIER);
349 Assert.assertFalse(table.exists(get));
350 }
351 table.close();
352 }
353
354 @Test(timeout=300000)
355 public void testHTableDeleteWithList() throws Exception {
356 LOG.info("test=testHTableDeleteWithList");
357 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
358
359
360 List<Row> puts = constructPutRequests();
361 Object[] results = table.batch(puts);
362 validateSizeAndEmpty(results, KEYS.length);
363
364
365 ArrayList<Delete> deletes = new ArrayList<Delete>();
366 for (int i = 0; i < KEYS.length; i++) {
367 Delete delete = new Delete(KEYS[i]);
368 delete.deleteFamily(BYTES_FAMILY);
369 deletes.add(delete);
370 }
371 table.delete(deletes);
372 Assert.assertTrue(deletes.isEmpty());
373
374
375 for (byte[] k : KEYS) {
376 Get get = new Get(k);
377 get.addColumn(BYTES_FAMILY, QUALIFIER);
378 Assert.assertFalse(table.exists(get));
379 }
380 table.close();
381 }
382
383 @Test(timeout=300000)
384 public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
385 LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
386 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
387
388 List<Row> puts = new ArrayList<Row>();
389 for (int i = 0; i < 100; i++) {
390 Put put = new Put(ONE_ROW);
391 byte[] qual = Bytes.toBytes("column" + i);
392 put.add(BYTES_FAMILY, qual, VALUE);
393 puts.add(put);
394 }
395 Object[] results = table.batch(puts);
396
397
398 validateSizeAndEmpty(results, 100);
399
400
401 List<Row> gets = new ArrayList<Row>();
402 for (int i = 0; i < 100; i++) {
403 Get get = new Get(ONE_ROW);
404 byte[] qual = Bytes.toBytes("column" + i);
405 get.addColumn(BYTES_FAMILY, qual);
406 gets.add(get);
407 }
408
409 Object[] multiRes = table.batch(gets);
410
411 int idx = 0;
412 for (Object r : multiRes) {
413 byte[] qual = Bytes.toBytes("column" + idx);
414 validateResult(r, qual, VALUE);
415 idx++;
416 }
417 table.close();
418 }
419
420 @Test(timeout=300000)
421 public void testBatchWithIncrementAndAppend() throws Exception {
422 LOG.info("test=testBatchWithIncrementAndAppend");
423 final byte[] QUAL1 = Bytes.toBytes("qual1");
424 final byte[] QUAL2 = Bytes.toBytes("qual2");
425 final byte[] QUAL3 = Bytes.toBytes("qual3");
426 final byte[] QUAL4 = Bytes.toBytes("qual4");
427 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
428 Delete d = new Delete(ONE_ROW);
429 table.delete(d);
430 Put put = new Put(ONE_ROW);
431 put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
432 put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
433 table.put(put);
434
435 Increment inc = new Increment(ONE_ROW);
436 inc.addColumn(BYTES_FAMILY, QUAL2, 1);
437 inc.addColumn(BYTES_FAMILY, QUAL3, 1);
438
439 Append a = new Append(ONE_ROW);
440 a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
441 a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
442 List<Row> actions = new ArrayList<Row>();
443 actions.add(inc);
444 actions.add(a);
445
446 Object[] multiRes = table.batch(actions);
447 validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
448 validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
449 validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
450 validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
451 table.close();
452 }
453
454 @Test(timeout=300000)
455 public void testBatchWithMixedActions() throws Exception {
456 LOG.info("test=testBatchWithMixedActions");
457 HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
458
459
460 Object[] results = table.batch(constructPutRequests());
461 validateSizeAndEmpty(results, KEYS.length);
462
463
464
465 List<Row> actions = new ArrayList<Row>();
466
467 byte[] qual2 = Bytes.toBytes("qual2");
468 byte[] val2 = Bytes.toBytes("putvalue2");
469
470
471 Get get = new Get(KEYS[10]);
472 get.addColumn(BYTES_FAMILY, QUALIFIER);
473 actions.add(get);
474
475
476 get = new Get(KEYS[11]);
477 get.addColumn(BYTES_FAMILY, QUALIFIER);
478 actions.add(get);
479
480
481 Put put = new Put(KEYS[10]);
482 put.add(BYTES_FAMILY, qual2, val2);
483 actions.add(put);
484
485
486 Delete delete = new Delete(KEYS[20]);
487 delete.deleteFamily(BYTES_FAMILY);
488 actions.add(delete);
489
490
491 get = new Get(KEYS[30]);
492 get.addColumn(BYTES_FAMILY, QUALIFIER);
493 actions.add(get);
494
495
496
497
498
499
500 put = new Put(KEYS[40]);
501 put.add(BYTES_FAMILY, qual2, val2);
502 actions.add(put);
503
504 results = table.batch(actions);
505
506
507
508 validateResult(results[0]);
509 validateResult(results[1]);
510 validateEmpty(results[2]);
511 validateEmpty(results[3]);
512 validateResult(results[4]);
513 validateEmpty(results[5]);
514
515
516 get = new Get(KEYS[40]);
517 get.addColumn(BYTES_FAMILY, qual2);
518 Result r = table.get(get);
519 validateResult(r, qual2, val2);
520
521 table.close();
522 }
523
524
525
526 private void validateResult(Object r) {
527 validateResult(r, QUALIFIER, VALUE);
528 }
529
530 private void validateResult(Object r1, byte[] qual, byte[] val) {
531
532 Result r = (Result)r1;
533 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
534 Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
535 }
536
537 private List<Row> constructPutRequests() {
538 List<Row> puts = new ArrayList<Row>();
539 for (byte[] k : KEYS) {
540 Put put = new Put(k);
541 put.add(BYTES_FAMILY, QUALIFIER, VALUE);
542 puts.add(put);
543 }
544 return puts;
545 }
546
547 private void validateLoadedData(HTable table) throws IOException {
548
549 for (byte[] k : KEYS) {
550 Get get = new Get(k);
551 get.addColumn(BYTES_FAMILY, QUALIFIER);
552 Result r = table.get(get);
553 Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
554 Assert.assertEquals(0, Bytes.compareTo(VALUE, r
555 .getValue(BYTES_FAMILY, QUALIFIER)));
556 }
557 }
558
559 private void validateEmpty(Object r1) {
560 Result result = (Result)r1;
561 Assert.assertTrue(result != null);
562 Assert.assertTrue(result.getRow() == null);
563 Assert.assertEquals(0, result.raw().length);
564 }
565
566 private void validateSizeAndEmpty(Object[] results, int expectedSize) {
567
568 Assert.assertEquals(expectedSize, results.length);
569 for (Object result : results) {
570 validateEmpty(result);
571 }
572 }
573
574 @org.junit.Rule
575 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
576 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
577 }
578