View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.lang.reflect.Field;
28  import java.util.ArrayList;
29  import java.util.List;
30  import java.util.concurrent.ThreadPoolExecutor;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.commons.logging.impl.Log4JLogger;
35  import org.apache.hadoop.hbase.Cell;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.MediumTests;
39  import org.apache.hadoop.hbase.Waiter;
40  import org.apache.hadoop.hbase.ipc.RpcClient;
41  import org.apache.hadoop.hbase.ipc.RpcServer;
42  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
43  import org.apache.hadoop.hbase.util.Bytes;
44  import org.apache.hadoop.hbase.util.JVMClusterUtil;
45  import org.apache.log4j.Level;
46  import org.junit.AfterClass;
47  import org.junit.Assert;
48  import org.junit.Before;
49  import org.junit.BeforeClass;
50  import org.junit.Test;
51  import org.junit.experimental.categories.Category;
52  
53  @Category(MediumTests.class)
54  public class TestMultiParallel {
55    private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
56    {
57      ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
58      ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
59    }
60    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
61    private static final byte[] VALUE = Bytes.toBytes("value");
62    private static final byte[] QUALIFIER = Bytes.toBytes("qual");
63    private static final String FAMILY = "family";
64    private static final String TEST_TABLE = "multi_test_table";
65    private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
66    private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
67    private static final byte [][] KEYS = makeKeys();
68  
69    private static final int slaves = 2; // also used for testing HTable pool size
70  
71    @BeforeClass public static void beforeClass() throws Exception {
72      ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
73      ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
74      ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
75      UTIL.startMiniCluster(slaves);
76      HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
77      UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
78      UTIL.waitTableEnabled(Bytes.toBytes(TEST_TABLE));
79      t.close();
80    }
81  
82    @AfterClass public static void afterClass() throws Exception {
83      UTIL.shutdownMiniCluster();
84    }
85  
86    @Before public void before() throws Exception {
87      LOG.info("before");
88      if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
89        // Distribute regions
90        UTIL.getMiniHBaseCluster().getMaster().balance();
91  
92        // Wait until completing balance
93        UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
94      }
95      HConnection conn = HConnectionManager.getConnection(UTIL.getConfiguration());
96      conn.clearRegionCache();
97      conn.close();
98      LOG.info("before done");
99    }
100 
101   private static byte[][] makeKeys() {
102     byte [][] starterKeys = HBaseTestingUtility.KEYS;
103     // Create a "non-uniform" test set with the following characteristics:
104     // a) Unequal number of keys per region
105 
106     // Don't use integer as a multiple, so that we have a number of keys that is
107     // not a multiple of the number of regions
108     int numKeys = (int) ((float) starterKeys.length * 10.33F);
109 
110     List<byte[]> keys = new ArrayList<byte[]>();
111     for (int i = 0; i < numKeys; i++) {
112       int kIdx = i % starterKeys.length;
113       byte[] k = starterKeys[kIdx];
114       byte[] cp = new byte[k.length + 1];
115       System.arraycopy(k, 0, cp, 0, k.length);
116       cp[k.length] = new Integer(i % 256).byteValue();
117       keys.add(cp);
118     }
119 
120     // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which
121     // should work)
122     // c) keys are not in sorted order (within a region), to ensure that the
123     // sorting code and index mapping doesn't break the functionality
124     for (int i = 0; i < 100; i++) {
125       int kIdx = i % starterKeys.length;
126       byte[] k = starterKeys[kIdx];
127       byte[] cp = new byte[k.length + 1];
128       System.arraycopy(k, 0, cp, 0, k.length);
129       cp[k.length] = new Integer(i % 256).byteValue();
130       keys.add(cp);
131     }
132     return keys.toArray(new byte [][] {new byte [] {}});
133   }
134 
135 
136   /**
137    * This is for testing the active number of threads that were used while
138    * doing a batch operation. It inserts one row per region via the batch
139    * operation, and then checks the number of active threads.
140    * For HBASE-3553
141    * @throws IOException
142    * @throws InterruptedException
143    * @throws NoSuchFieldException
144    * @throws SecurityException
145    */
146   @Test(timeout=300000)
147   public void testActiveThreadsCount() throws Exception{
148     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
149     List<Row> puts = constructPutRequests(); // creates a Put for every region
150     table.batch(puts);
151     Field poolField = table.getClass().getDeclaredField("pool");
152     poolField.setAccessible(true);
153     ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
154     assertEquals(slaves, tExecutor.getLargestPoolSize());
155     table.close();
156   }
157 
158   @Test(timeout=300000)
159   public void testBatchWithGet() throws Exception {
160     LOG.info("test=testBatchWithGet");
161     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
162 
163     // load test data
164     List<Row> puts = constructPutRequests();
165     table.batch(puts);
166 
167     // create a list of gets and run it
168     List<Row> gets = new ArrayList<Row>();
169     for (byte[] k : KEYS) {
170       Get get = new Get(k);
171       get.addColumn(BYTES_FAMILY, QUALIFIER);
172       gets.add(get);
173     }
174     Result[] multiRes = new Result[gets.size()];
175     table.batch(gets, multiRes);
176 
177     // Same gets using individual call API
178     List<Result> singleRes = new ArrayList<Result>();
179     for (Row get : gets) {
180       singleRes.add(table.get((Get) get));
181     }
182     // Compare results
183     Assert.assertEquals(singleRes.size(), multiRes.length);
184     for (int i = 0; i < singleRes.size(); i++) {
185       Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
186       Cell[] singleKvs = singleRes.get(i).rawCells();
187       Cell[] multiKvs = multiRes[i].rawCells();
188       for (int j = 0; j < singleKvs.length; j++) {
189         Assert.assertEquals(singleKvs[j], multiKvs[j]);
190         Assert.assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(singleKvs[j]), 
191             CellUtil.cloneValue(multiKvs[j])));
192       }
193     }
194     table.close();
195   }
196 
197   @Test
198   public void testBadFam() throws Exception {
199     LOG.info("test=testBadFam");
200     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
201 
202     List<Row> actions = new ArrayList<Row>();
203     Put p = new Put(Bytes.toBytes("row1"));
204     p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
205     actions.add(p);
206     p = new Put(Bytes.toBytes("row2"));
207     p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
208     actions.add(p);
209 
210     // row1 and row2 should be in the same region.
211 
212     Object [] r = new Object[actions.size()];
213     try {
214       table.batch(actions, r);
215       fail();
216     } catch (RetriesExhaustedWithDetailsException ex) {
217       LOG.debug(ex);
218       // good!
219       assertFalse(ex.mayHaveClusterIssues());
220     }
221     assertEquals(2, r.length);
222     assertTrue(r[0] instanceof Throwable);
223     assertTrue(r[1] instanceof Result);
224     table.close();
225   }
226 
227   @Test (timeout=300000)
228   public void testFlushCommitsNoAbort() throws Exception {
229     LOG.info("test=testFlushCommitsNoAbort");
230     doTestFlushCommits(false);
231   }
232 
233   /**
234    * Only run one Multi test with a forced RegionServer abort. Otherwise, the
235    * unit tests will take an unnecessarily long time to run.
236    *
237    * @throws Exception
238    */
239   @Test (timeout=300000)
240   public void testFlushCommitsWithAbort() throws Exception {
241     LOG.info("test=testFlushCommitsWithAbort");
242     doTestFlushCommits(true);
243   }
244 
245   /**
246    * Set table auto flush to false and test flushing commits
247    * @param doAbort true if abort one regionserver in the testing
248    * @throws Exception
249    */
250   private void doTestFlushCommits(boolean doAbort) throws Exception {
251     // Load the data
252     LOG.info("get new table");
253     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
254     table.setAutoFlush(false, true);
255     table.setWriteBufferSize(10 * 1024 * 1024);
256 
257     LOG.info("constructPutRequests");
258     List<Row> puts = constructPutRequests();
259     for (Row put : puts) {
260       table.put((Put) put);
261     }
262     LOG.info("puts");
263     table.flushCommits();
264     final int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
265         .size();
266     assert liveRScount > 0;
267     JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
268         .getLiveRegionServerThreads().get(0);
269     if (doAbort) {
270       liveRS.getRegionServer().abort("Aborting for tests",
271           new Exception("doTestFlushCommits"));
272       // If we wait for no regions being online after we abort the server, we
273       // could ensure the master has re-assigned the regions on killed server
274       // after writing successfully. It means the server we aborted is dead
275       // and detected by matser
276       while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
277         Thread.sleep(10);
278       }
279       // try putting more keys after the abort. same key/qual... just validating
280       // no exceptions thrown
281       puts = constructPutRequests();
282       for (Row put : puts) {
283         table.put((Put) put);
284       }
285 
286       table.flushCommits();
287     }
288 
289     LOG.info("validating loaded data");
290     validateLoadedData(table);
291 
292     // Validate server and region count
293     List<JVMClusterUtil.RegionServerThread> liveRSs = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
294     int count = 0;
295     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
296       count++;
297       LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
298     }
299     LOG.info("Count=" + count);
300     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
301         (doAbort ? (liveRScount - 1) : liveRScount), count);
302     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
303       int regions = ProtobufUtil.getOnlineRegions(t.getRegionServer()).size();
304       // Assert.assertTrue("Count of regions=" + regions, regions > 10);
305     }
306     if (doAbort) {
307       UTIL.getMiniHBaseCluster().waitOnRegionServer(0);
308       UTIL.waitFor(15 * 1000, new Waiter.Predicate<Exception>() {
309         @Override
310         public boolean evaluate() throws Exception {
311           return UTIL.getMiniHBaseCluster().getMaster()
312               .getClusterStatus().getServersSize() == (liveRScount - 1);
313         }
314       });
315       UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
316     }
317 
318     table.close();
319     LOG.info("done");
320   }
321 
322   @Test (timeout=300000)
323   public void testBatchWithPut() throws Exception {
324     LOG.info("test=testBatchWithPut");
325     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
326 
327     // put multiple rows using a batch
328     List<Row> puts = constructPutRequests();
329 
330     Object[] results = table.batch(puts);
331     validateSizeAndEmpty(results, KEYS.length);
332 
333     if (true) {
334       int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
335       assert liveRScount > 0;
336       JVMClusterUtil.RegionServerThread liveRS =
337         UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0);
338       liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut"));
339       puts = constructPutRequests();
340       try {
341         results = table.batch(puts);
342       } catch (RetriesExhaustedWithDetailsException ree) {
343         LOG.info(ree.getExhaustiveDescription());
344         throw ree;
345       } finally {
346         table.close();
347       }
348       validateSizeAndEmpty(results, KEYS.length);
349     }
350 
351     validateLoadedData(table);
352     table.close();
353   }
354 
355   @Test(timeout=300000)
356   public void testBatchWithDelete() throws Exception {
357     LOG.info("test=testBatchWithDelete");
358     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
359 
360     // Load some data
361     List<Row> puts = constructPutRequests();
362     Object[] results = table.batch(puts);
363     validateSizeAndEmpty(results, KEYS.length);
364 
365     // Deletes
366     List<Row> deletes = new ArrayList<Row>();
367     for (int i = 0; i < KEYS.length; i++) {
368       Delete delete = new Delete(KEYS[i]);
369       delete.deleteFamily(BYTES_FAMILY);
370       deletes.add(delete);
371     }
372     results = table.batch(deletes);
373     validateSizeAndEmpty(results, KEYS.length);
374 
375     // Get to make sure ...
376     for (byte[] k : KEYS) {
377       Get get = new Get(k);
378       get.addColumn(BYTES_FAMILY, QUALIFIER);
379       Assert.assertFalse(table.exists(get));
380     }
381     table.close();
382   }
383 
384   @Test(timeout=300000)
385   public void testHTableDeleteWithList() throws Exception {
386     LOG.info("test=testHTableDeleteWithList");
387     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
388 
389     // Load some data
390     List<Row> puts = constructPutRequests();
391     Object[] results = table.batch(puts);
392     validateSizeAndEmpty(results, KEYS.length);
393 
394     // Deletes
395     ArrayList<Delete> deletes = new ArrayList<Delete>();
396     for (int i = 0; i < KEYS.length; i++) {
397       Delete delete = new Delete(KEYS[i]);
398       delete.deleteFamily(BYTES_FAMILY);
399       deletes.add(delete);
400     }
401     table.delete(deletes);
402     Assert.assertTrue(deletes.isEmpty());
403 
404     // Get to make sure ...
405     for (byte[] k : KEYS) {
406       Get get = new Get(k);
407       get.addColumn(BYTES_FAMILY, QUALIFIER);
408       Assert.assertFalse(table.exists(get));
409     }
410     table.close();
411   }
412 
413   @Test(timeout=300000)
414   public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
415     LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
416     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
417 
418     List<Row> puts = new ArrayList<Row>();
419     for (int i = 0; i < 100; i++) {
420       Put put = new Put(ONE_ROW);
421       byte[] qual = Bytes.toBytes("column" + i);
422       put.add(BYTES_FAMILY, qual, VALUE);
423       puts.add(put);
424     }
425     Object[] results = table.batch(puts);
426 
427     // validate
428     validateSizeAndEmpty(results, 100);
429 
430     // get the data back and validate that it is correct
431     List<Row> gets = new ArrayList<Row>();
432     for (int i = 0; i < 100; i++) {
433       Get get = new Get(ONE_ROW);
434       byte[] qual = Bytes.toBytes("column" + i);
435       get.addColumn(BYTES_FAMILY, qual);
436       gets.add(get);
437     }
438 
439     Object[] multiRes = table.batch(gets);
440 
441     int idx = 0;
442     for (Object r : multiRes) {
443       byte[] qual = Bytes.toBytes("column" + idx);
444       validateResult(r, qual, VALUE);
445       idx++;
446     }
447     table.close();
448   }
449 
450   @Test(timeout=300000)
451   public void testBatchWithIncrementAndAppend() throws Exception {
452     LOG.info("test=testBatchWithIncrementAndAppend");
453     final byte[] QUAL1 = Bytes.toBytes("qual1");
454     final byte[] QUAL2 = Bytes.toBytes("qual2");
455     final byte[] QUAL3 = Bytes.toBytes("qual3");
456     final byte[] QUAL4 = Bytes.toBytes("qual4");
457     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
458     Delete d = new Delete(ONE_ROW);
459     table.delete(d);
460     Put put = new Put(ONE_ROW);
461     put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
462     put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
463     table.put(put);
464 
465     Increment inc = new Increment(ONE_ROW);
466     inc.addColumn(BYTES_FAMILY, QUAL2, 1);
467     inc.addColumn(BYTES_FAMILY, QUAL3, 1);
468 
469     Append a = new Append(ONE_ROW);
470     a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
471     a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
472     List<Row> actions = new ArrayList<Row>();
473     actions.add(inc);
474     actions.add(a);
475 
476     Object[] multiRes = table.batch(actions);
477     validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
478     validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
479     validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
480     validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
481     table.close();
482   }
483 
484   @Test(timeout=300000)
485   public void testBatchWithMixedActions() throws Exception {
486     LOG.info("test=testBatchWithMixedActions");
487     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
488 
489     // Load some data to start
490     Object[] results = table.batch(constructPutRequests());
491     validateSizeAndEmpty(results, KEYS.length);
492 
493     // Batch: get, get, put(new col), delete, get, get of put, get of deleted,
494     // put
495     List<Row> actions = new ArrayList<Row>();
496 
497     byte[] qual2 = Bytes.toBytes("qual2");
498     byte[] val2 = Bytes.toBytes("putvalue2");
499 
500     // 0 get
501     Get get = new Get(KEYS[10]);
502     get.addColumn(BYTES_FAMILY, QUALIFIER);
503     actions.add(get);
504 
505     // 1 get
506     get = new Get(KEYS[11]);
507     get.addColumn(BYTES_FAMILY, QUALIFIER);
508     actions.add(get);
509 
510     // 2 put of new column
511     Put put = new Put(KEYS[10]);
512     put.add(BYTES_FAMILY, qual2, val2);
513     actions.add(put);
514 
515     // 3 delete
516     Delete delete = new Delete(KEYS[20]);
517     delete.deleteFamily(BYTES_FAMILY);
518     actions.add(delete);
519 
520     // 4 get
521     get = new Get(KEYS[30]);
522     get.addColumn(BYTES_FAMILY, QUALIFIER);
523     actions.add(get);
524 
525     // There used to be a 'get' of a previous put here, but removed
526     // since this API really cannot guarantee order in terms of mixed
527     // get/puts.
528 
529     // 5 put of new column
530     put = new Put(KEYS[40]);
531     put.add(BYTES_FAMILY, qual2, val2);
532     actions.add(put);
533 
534     results = table.batch(actions);
535 
536     // Validation
537 
538     validateResult(results[0]);
539     validateResult(results[1]);
540     validateEmpty(results[2]);
541     validateEmpty(results[3]);
542     validateResult(results[4]);
543     validateEmpty(results[5]);
544 
545     // validate last put, externally from the batch
546     get = new Get(KEYS[40]);
547     get.addColumn(BYTES_FAMILY, qual2);
548     Result r = table.get(get);
549     validateResult(r, qual2, val2);
550 
551     table.close();
552   }
553 
554   // // Helper methods ////
555 
556   private void validateResult(Object r) {
557     validateResult(r, QUALIFIER, VALUE);
558   }
559 
560   private void validateResult(Object r1, byte[] qual, byte[] val) {
561     // TODO provide nice assert here or something.
562     Result r = (Result)r1;
563     Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
564     Assert.assertEquals(0, Bytes.compareTo(val, r.getValue(BYTES_FAMILY, qual)));
565   }
566 
567   private List<Row> constructPutRequests() {
568     List<Row> puts = new ArrayList<Row>();
569     for (byte[] k : KEYS) {
570       Put put = new Put(k);
571       put.add(BYTES_FAMILY, QUALIFIER, VALUE);
572       puts.add(put);
573     }
574     return puts;
575   }
576 
577   private void validateLoadedData(HTable table) throws IOException {
578     // get the data back and validate that it is correct
579     for (byte[] k : KEYS) {
580       Get get = new Get(k);
581       get.addColumn(BYTES_FAMILY, QUALIFIER);
582       Result r = table.get(get);
583       Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
584       Assert.assertEquals(0, Bytes.compareTo(VALUE, r
585           .getValue(BYTES_FAMILY, QUALIFIER)));
586     }
587   }
588 
589   private void validateEmpty(Object r1) {
590     Result result = (Result)r1;
591     Assert.assertTrue(result != null);
592     Assert.assertTrue(result.getRow() == null);
593     Assert.assertEquals(0, result.rawCells().length);
594   }
595 
596   private void validateSizeAndEmpty(Object[] results, int expectedSize) {
597     // Validate got back the same number of Result objects, all empty
598     Assert.assertEquals(expectedSize, results.length);
599     for (Object result : results) {
600       validateEmpty(result);
601     }
602   }
603 }