1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import java.io.IOException;
23  import java.lang.reflect.Field;
24  import java.util.ArrayList;
25  import java.util.List;
26  import java.util.concurrent.ThreadPoolExecutor;
27  
28  import org.apache.commons.logging.Log;
29  import org.apache.commons.logging.LogFactory;
30  import org.apache.hadoop.hbase.*;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.JVMClusterUtil;
33  import org.junit.AfterClass;
34  import org.junit.Assert;
35  import org.junit.Before;
36  import org.junit.BeforeClass;
37  import org.junit.FixMethodOrder;
38  import org.junit.Test;
39  import org.junit.experimental.categories.Category;
40  import org.junit.runners.MethodSorters;
41  
42  import static org.junit.Assert.*;
43  
44  @FixMethodOrder(MethodSorters.NAME_ASCENDING)
45  @Category(MediumTests.class)
46  public class TestMultiParallel {
47    private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
48    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
49    private static final byte[] VALUE = Bytes.toBytes("value");
50    private static final byte[] QUALIFIER = Bytes.toBytes("qual");
51    private static final String FAMILY = "family";
52    private static final String TEST_TABLE = "multi_test_table";
53    private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
54    private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
55    private static final byte [][] KEYS = makeKeys();
56  
57    private static final int slaves = 2; // also used for testing HTable pool size
58  
59    @BeforeClass public static void beforeClass() throws Exception {
60      UTIL.startMiniCluster(slaves);
61      HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
62      UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
63      UTIL.waitTableAvailable(Bytes.toBytes(TEST_TABLE), 15 * 1000);
64      t.close();
65    }
66  
67    @AfterClass public static void afterClass() throws Exception {
68      UTIL.shutdownMiniCluster();
69    }
70  
71    @Before public void before() throws IOException {
72      LOG.info("before");
73      if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
74        // Distribute regions
75        UTIL.getMiniHBaseCluster().getMaster().balance();
76      }
77      LOG.info("before done");
78    }
79  
80    private static byte[][] makeKeys() {
81      byte [][] starterKeys = HBaseTestingUtility.KEYS;
82      // Create a "non-uniform" test set with the following characteristics:
83      // a) Unequal number of keys per region
84  
85      // Don't use integer as a multiple, so that we have a number of keys that is
86      // not a multiple of the number of regions
87      int numKeys = (int) ((float) starterKeys.length * 10.33F);
88  
89      List<byte[]> keys = new ArrayList<byte[]>();
90      for (int i = 0; i < numKeys; i++) {
91        int kIdx = i % starterKeys.length;
92        byte[] k = starterKeys[kIdx];
93        byte[] cp = new byte[k.length + 1];
94        System.arraycopy(k, 0, cp, 0, k.length);
95        cp[k.length] = new Integer(i % 256).byteValue();
96        keys.add(cp);
97      }
98  
99      // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which
100     // should work)
101     // c) keys are not in sorted order (within a region), to ensure that the
102     // sorting code and index mapping doesn't break the functionality
103     for (int i = 0; i < 100; i++) {
104       int kIdx = i % starterKeys.length;
105       byte[] k = starterKeys[kIdx];
106       byte[] cp = new byte[k.length + 1];
107       System.arraycopy(k, 0, cp, 0, k.length);
108       cp[k.length] = new Integer(i % 256).byteValue();
109       keys.add(cp);
110     }
111     return keys.toArray(new byte [][] {new byte [] {}});
112   }
113 
114 
115   /**
116    * This is for testing the active number of threads that were used while
117    * doing a batch operation. It inserts one row per region via the batch
118    * operation, and then checks the number of active threads.
119    * For HBASE-3553
120    * @throws IOException
121    * @throws InterruptedException
122    * @throws NoSuchFieldException
123    * @throws SecurityException
124    */
125   @Test(timeout=300000) 
126   public void testActiveThreadsCount() throws Exception{
127     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
128     List<Row> puts = constructPutRequests(); // creates a Put for every region
129     table.batch(puts);
130     Field poolField = table.getClass().getDeclaredField("pool");
131     poolField.setAccessible(true);
132     ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
133     assertEquals(slaves, tExecutor.getLargestPoolSize());
134     table.close();
135   }
136 
137   @Test(timeout=300000) 
138   public void testBatchWithGet() throws Exception {
139     LOG.info("test=testBatchWithGet");
140     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
141 
142     // load test data
143     List<Row> puts = constructPutRequests();
144     table.batch(puts);
145 
146     // create a list of gets and run it
147     List<Row> gets = new ArrayList<Row>();
148     for (byte[] k : KEYS) {
149       Get get = new Get(k);
150       get.addColumn(BYTES_FAMILY, QUALIFIER);
151       gets.add(get);
152     }
153     Result[] multiRes = new Result[gets.size()];
154     table.batch(gets, multiRes);
155 
156     // Same gets using individual call API
157     List<Result> singleRes = new ArrayList<Result>();
158     for (Row get : gets) {
159       singleRes.add(table.get((Get) get));
160     }
161 
162     // Compare results
163     Assert.assertEquals(singleRes.size(), multiRes.length);
164     for (int i = 0; i < singleRes.size(); i++) {
165       Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
166       KeyValue[] singleKvs = singleRes.get(i).raw();
167       KeyValue[] multiKvs = multiRes[i].raw();
168       for (int j = 0; j < singleKvs.length; j++) {
169         Assert.assertEquals(singleKvs[j], multiKvs[j]);
170         Assert.assertEquals(0, Bytes.compareTo(singleKvs[j].getValue(), multiKvs[j]
171             .getValue()));
172       }
173     }
174     table.close();
175   }
176 
177   @Test
178   public void testBadFam() throws Exception {
179     LOG.info("test=testBadFam");
180     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
181 
182     List<Row> actions = new ArrayList<Row>();
183     Put p = new Put(Bytes.toBytes("row1"));
184     p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
185     actions.add(p);
186     p = new Put(Bytes.toBytes("row2"));
187     p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
188     actions.add(p);
189 
190     // row1 and row2 should be in the same region.
191 
192     Object [] r = new Object[actions.size()];
193     try {
194       table.batch(actions, r);
195       fail();
196     } catch (RetriesExhaustedWithDetailsException ex) {
197       LOG.debug(ex);
198       // good!
199       assertFalse(ex.mayHaveClusterIssues());
200     }
201     assertEquals(2, r.length);
202     assertTrue(r[0] instanceof Throwable);
203     assertTrue(r[1] instanceof Result);
204     table.close();
205   }
206 
207   /**
208    * Only run one Multi test with a forced RegionServer abort. Otherwise, the
209    * unit tests will take an unnecessarily long time to run.
210    *
211    * @throws Exception
212    */
213   @Test (timeout=300000) 
214   public void testFlushCommitsWithAbort() throws Exception {
215     LOG.info("test=testFlushCommitsWithAbort");
216     doTestFlushCommits(true);
217   }
218 
219   @Test (timeout=300000)
220   public void testFlushCommitsNoAbort() throws Exception {
221     LOG.info("test=testFlushCommitsNoAbort");
222     doTestFlushCommits(false);
223   }
224 
225   /**
226    * Set table auto flush to false and test flushing commits
227    * @param doAbort true if abort one regionserver in the testing
228    * @throws Exception
229    */
230   private void doTestFlushCommits(boolean doAbort) throws Exception {
231     // Load the data
232     LOG.info("get new table");
233     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
234     table.setAutoFlush(false);
235     table.setWriteBufferSize(10 * 1024 * 1024);
236 
237     LOG.info("constructPutRequests");
238     List<Row> puts = constructPutRequests();
239     for (Row put : puts) {
240       table.put((Put) put);
241     }
242     LOG.info("puts");
243     table.flushCommits();
244     int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
245         .size();
246     assert liveRScount > 0;
247     JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
248         .getLiveRegionServerThreads().get(0);
249     if (doAbort) {
250       LOG.info("Aborted=" + UTIL.getMiniHBaseCluster().abortRegionServer(0));
251 
252       // If we waiting for no regions being online after we abort the server, we
253       // could ensure the master has re-assigned the regions on killed server
254       // after putting keys successfully, it means the server we abort is dead
255       // and detected by matser
256       while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
257         Thread.sleep(100);
258       }
259       while (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == liveRScount) {
260         Thread.sleep(100);
261       }
262 
263       // try putting more keys after the abort. same key/qual... just validating
264       // no exceptions thrown
265       puts = constructPutRequests();
266       for (Row put : puts) {
267         table.put((Put) put);
268       }
269 
270       table.flushCommits();
271     }
272 
273     LOG.info("validating loaded data");
274     validateLoadedData(table);
275 
276     // Validate server and region count
277     List<JVMClusterUtil.RegionServerThread> liveRSs =
278       UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
279     int count = 0;
280     int regionCount = 0;
281     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
282       count++;
283       regionCount += t.getRegionServer().getOnlineRegions().size();
284       LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
285     }
286     LOG.info("Count=" + count);
287     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
288         (doAbort ? (liveRScount - 1) : liveRScount), count);
289     Assert.assertTrue("Count of regions=" + regionCount, regionCount >= 25);
290     table.close();
291     LOG.info("done");
292   }
293 
294   @Test (timeout=300000)
295   // FIXME: sort test lexicographically to the end.
296   // otherwise clashed with testFlushCommitsWithAbort
297   public void testZBatchWithPut() throws Exception {
298     LOG.info("test=testBatchWithPut");
299     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
300 
301     // put multiple rows using a batch
302     List<Row> puts = constructPutRequests();
303 
304     Object[] results = table.batch(puts);
305     validateSizeAndEmpty(results, KEYS.length);
306 
307     if (true) {
308       int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
309           .size();
310       assert liveRScount > 0;
311       JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
312           .getLiveRegionServerThreads().get(0);
313       liveRS.getRegionServer().abort("Aborting for tests",
314           new Exception("testBatchWithPut"));
315 
316       puts = constructPutRequests();
317       results = table.batch(puts);
318       validateSizeAndEmpty(results, KEYS.length);
319     }
320 
321     validateLoadedData(table);
322     table.close();
323   }
324 
325   @Test(timeout=300000) 
326   public void testBatchWithDelete() throws Exception {
327     LOG.info("test=testBatchWithDelete");
328     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
329 
330     // Load some data
331     List<Row> puts = constructPutRequests();
332     Object[] results = table.batch(puts);
333     validateSizeAndEmpty(results, KEYS.length);
334 
335     // Deletes
336     List<Row> deletes = new ArrayList<Row>();
337     for (int i = 0; i < KEYS.length; i++) {
338       Delete delete = new Delete(KEYS[i]);
339       delete.deleteFamily(BYTES_FAMILY);
340       deletes.add(delete);
341     }
342     results = table.batch(deletes);
343     validateSizeAndEmpty(results, KEYS.length);
344 
345     // Get to make sure ...
346     for (byte[] k : KEYS) {
347       Get get = new Get(k);
348       get.addColumn(BYTES_FAMILY, QUALIFIER);
349       Assert.assertFalse(table.exists(get));
350     }
351     table.close();
352   }
353 
354   @Test(timeout=300000)
355   public void testHTableDeleteWithList() throws Exception {
356     LOG.info("test=testHTableDeleteWithList");
357     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
358 
359     // Load some data
360     List<Row> puts = constructPutRequests();
361     Object[] results = table.batch(puts);
362     validateSizeAndEmpty(results, KEYS.length);
363 
364     // Deletes
365     ArrayList<Delete> deletes = new ArrayList<Delete>();
366     for (int i = 0; i < KEYS.length; i++) {
367       Delete delete = new Delete(KEYS[i]);
368       delete.deleteFamily(BYTES_FAMILY);
369       deletes.add(delete);
370     }
371     table.delete(deletes);
372     Assert.assertTrue(deletes.isEmpty());
373 
374     // Get to make sure ...
375     for (byte[] k : KEYS) {
376       Get get = new Get(k);
377       get.addColumn(BYTES_FAMILY, QUALIFIER);
378       Assert.assertFalse(table.exists(get));
379     }
380     table.close();
381   }
382 
383   @Test(timeout=300000)
384   public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
385     LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
386     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
387 
388     List<Row> puts = new ArrayList<Row>();
389     for (int i = 0; i < 100; i++) {
390       Put put = new Put(ONE_ROW);
391       byte[] qual = Bytes.toBytes("column" + i);
392       put.add(BYTES_FAMILY, qual, VALUE);
393       puts.add(put);
394     }
395     Object[] results = table.batch(puts);
396 
397     // validate
398     validateSizeAndEmpty(results, 100);
399 
400     // get the data back and validate that it is correct
401     List<Row> gets = new ArrayList<Row>();
402     for (int i = 0; i < 100; i++) {
403       Get get = new Get(ONE_ROW);
404       byte[] qual = Bytes.toBytes("column" + i);
405       get.addColumn(BYTES_FAMILY, qual);
406       gets.add(get);
407     }
408 
409     Object[] multiRes = table.batch(gets);
410 
411     int idx = 0;
412     for (Object r : multiRes) {
413       byte[] qual = Bytes.toBytes("column" + idx);
414       validateResult(r, qual, VALUE);
415       idx++;
416     }
417     table.close();
418   }
419 
420   @Test(timeout=300000)
421   public void testBatchWithIncrementAndAppend() throws Exception {
422     LOG.info("test=testBatchWithIncrementAndAppend");
423     final byte[] QUAL1 = Bytes.toBytes("qual1");
424     final byte[] QUAL2 = Bytes.toBytes("qual2");
425     final byte[] QUAL3 = Bytes.toBytes("qual3");
426     final byte[] QUAL4 = Bytes.toBytes("qual4");
427     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
428     Delete d = new Delete(ONE_ROW);
429     table.delete(d);
430     Put put = new Put(ONE_ROW);
431     put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
432     put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
433     table.put(put);
434 
435     Increment inc = new Increment(ONE_ROW);
436     inc.addColumn(BYTES_FAMILY, QUAL2, 1);
437     inc.addColumn(BYTES_FAMILY, QUAL3, 1);
438 
439     Append a = new Append(ONE_ROW);
440     a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
441     a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
442     List<Row> actions = new ArrayList<Row>();
443     actions.add(inc);
444     actions.add(a);
445 
446     Object[] multiRes = table.batch(actions);
447     validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
448     validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
449     validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
450     validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
451     table.close();
452   }
453 
454   @Test(timeout=300000)
455   public void testBatchWithMixedActions() throws Exception {
456     LOG.info("test=testBatchWithMixedActions");
457     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
458 
459     // Load some data to start
460     Object[] results = table.batch(constructPutRequests());
461     validateSizeAndEmpty(results, KEYS.length);
462 
463     // Batch: get, get, put(new col), delete, get, get of put, get of deleted,
464     // put
465     List<Row> actions = new ArrayList<Row>();
466 
467     byte[] qual2 = Bytes.toBytes("qual2");
468     byte[] val2 = Bytes.toBytes("putvalue2");
469 
470     // 0 get
471     Get get = new Get(KEYS[10]);
472     get.addColumn(BYTES_FAMILY, QUALIFIER);
473     actions.add(get);
474 
475     // 1 get
476     get = new Get(KEYS[11]);
477     get.addColumn(BYTES_FAMILY, QUALIFIER);
478     actions.add(get);
479 
480     // 2 put of new column
481     Put put = new Put(KEYS[10]);
482     put.add(BYTES_FAMILY, qual2, val2);
483     actions.add(put);
484 
485     // 3 delete
486     Delete delete = new Delete(KEYS[20]);
487     delete.deleteFamily(BYTES_FAMILY);
488     actions.add(delete);
489 
490     // 4 get
491     get = new Get(KEYS[30]);
492     get.addColumn(BYTES_FAMILY, QUALIFIER);
493     actions.add(get);
494 
495     // There used to be a 'get' of a previous put here, but removed
496     // since this API really cannot guarantee order in terms of mixed
497     // get/puts.
498 
499     // 5 put of new column
500     put = new Put(KEYS[40]);
501     put.add(BYTES_FAMILY, qual2, val2);
502     actions.add(put);
503 
504     results = table.batch(actions);
505 
506     // Validation
507 
508     validateResult(results[0]);
509     validateResult(results[1]);
510     validateEmpty(results[2]);
511     validateEmpty(results[3]);
512     validateResult(results[4]);
513     validateEmpty(results[5]);
514 
515     // validate last put, externally from the batch
516     get = new Get(KEYS[40]);
517     get.addColumn(BYTES_FAMILY, qual2);
518     Result r = table.get(get);
519     validateResult(r, qual2, val2);
520 
521     table.close();
522   }
523 
524   // // Helper methods ////
525 
526   private void validateResult(Object r) {
527     validateResult(r, QUALIFIER, VALUE);
528   }
529 
530   private void validateResult(Object r1, byte[] qual, byte[] val) {
531     // TODO provide nice assert here or something.
532     Result r = (Result)r1;
533     Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
534     Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
535   }
536 
537   private List<Row> constructPutRequests() {
538     List<Row> puts = new ArrayList<Row>();
539     for (byte[] k : KEYS) {
540       Put put = new Put(k);
541       put.add(BYTES_FAMILY, QUALIFIER, VALUE);
542       puts.add(put);
543     }
544     return puts;
545   }
546 
547   private void validateLoadedData(HTable table) throws IOException {
548     // get the data back and validate that it is correct
549     for (byte[] k : KEYS) {
550       Get get = new Get(k);
551       get.addColumn(BYTES_FAMILY, QUALIFIER);
552       Result r = table.get(get);
553       Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
554       Assert.assertEquals(0, Bytes.compareTo(VALUE, r
555           .getValue(BYTES_FAMILY, QUALIFIER)));
556     }
557   }
558 
559   private void validateEmpty(Object r1) {
560     Result result = (Result)r1;
561     Assert.assertTrue(result != null);
562     Assert.assertTrue(result.getRow() == null);
563     Assert.assertEquals(0, result.raw().length);
564   }
565 
566   private void validateSizeAndEmpty(Object[] results, int expectedSize) {
567     // Validate got back the same number of Result objects, all empty
568     Assert.assertEquals(expectedSize, results.length);
569     for (Object result : results) {
570       validateEmpty(result);
571     }
572   }
573 
574   @org.junit.Rule
575   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
576     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
577 }
578