View Javadoc

1   /*
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.client;
20  
21  import static org.junit.Assert.assertEquals;
22  import static org.junit.Assert.assertFalse;
23  import static org.junit.Assert.assertTrue;
24  import static org.junit.Assert.fail;
25  
26  import java.io.IOException;
27  import java.lang.reflect.Field;
28  import java.util.ArrayList;
29  import java.util.List;
30  import java.util.concurrent.CountDownLatch;
31  import java.util.concurrent.ThreadPoolExecutor;
32  
33  import org.apache.commons.logging.Log;
34  import org.apache.commons.logging.LogFactory;
35  import org.apache.hadoop.hbase.Cell;
36  import org.apache.hadoop.hbase.CellUtil;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.testclassification.MediumTests;
40  import org.apache.hadoop.hbase.Waiter;
41  import org.apache.hadoop.hbase.codec.KeyValueCodec;
42  import org.apache.hadoop.hbase.exceptions.OperationConflictException;
43  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.JVMClusterUtil;
46  import org.apache.hadoop.hbase.util.Threads;
47  import org.junit.AfterClass;
48  import org.junit.Assert;
49  import org.junit.Before;
50  import org.junit.BeforeClass;
51  import org.junit.Test;
52  import org.junit.experimental.categories.Category;
53  
54  @Category(MediumTests.class)
55  public class TestMultiParallel {
56    private static final Log LOG = LogFactory.getLog(TestMultiParallel.class);
57  
58    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
59    private static final byte[] VALUE = Bytes.toBytes("value");
60    private static final byte[] QUALIFIER = Bytes.toBytes("qual");
61    private static final String FAMILY = "family";
62    private static final String TEST_TABLE = "multi_test_table";
63    private static final byte[] BYTES_FAMILY = Bytes.toBytes(FAMILY);
64    private static final byte[] ONE_ROW = Bytes.toBytes("xxx");
65    private static final byte [][] KEYS = makeKeys();
66  
67    private static final int slaves = 5; // also used for testing HTable pool size
68  
69    @BeforeClass public static void beforeClass() throws Exception {
70      // Uncomment the following lines if more verbosity is needed for
71      // debugging (see HBASE-12285 for details).
72      //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
73      //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
74      //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
75      UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
76          KeyValueCodec.class.getCanonicalName());
77      UTIL.startMiniCluster(slaves);
78      HTable t = UTIL.createTable(Bytes.toBytes(TEST_TABLE), Bytes.toBytes(FAMILY));
79      UTIL.createMultiRegions(t, Bytes.toBytes(FAMILY));
80      UTIL.waitTableEnabled(Bytes.toBytes(TEST_TABLE));
81      t.close();
82    }
83  
84    @AfterClass public static void afterClass() throws Exception {
85      UTIL.shutdownMiniCluster();
86    }
87  
88    @Before public void before() throws Exception {
89      LOG.info("before");
90      if (UTIL.ensureSomeRegionServersAvailable(slaves)) {
91        // Distribute regions
92        UTIL.getMiniHBaseCluster().getMaster().balance();
93  
94        // Wait until completing balance
95        UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
96      }
97      HConnection conn = HConnectionManager.getConnection(UTIL.getConfiguration());
98      conn.clearRegionCache();
99      conn.close();
100     LOG.info("before done");
101   }
102 
103   private static byte[][] makeKeys() {
104     byte [][] starterKeys = HBaseTestingUtility.KEYS;
105     // Create a "non-uniform" test set with the following characteristics:
106     // a) Unequal number of keys per region
107 
108     // Don't use integer as a multiple, so that we have a number of keys that is
109     // not a multiple of the number of regions
110     int numKeys = (int) ((float) starterKeys.length * 10.33F);
111 
112     List<byte[]> keys = new ArrayList<byte[]>();
113     for (int i = 0; i < numKeys; i++) {
114       int kIdx = i % starterKeys.length;
115       byte[] k = starterKeys[kIdx];
116       byte[] cp = new byte[k.length + 1];
117       System.arraycopy(k, 0, cp, 0, k.length);
118       cp[k.length] = new Integer(i % 256).byteValue();
119       keys.add(cp);
120     }
121 
122     // b) Same duplicate keys (showing multiple Gets/Puts to the same row, which
123     // should work)
124     // c) keys are not in sorted order (within a region), to ensure that the
125     // sorting code and index mapping doesn't break the functionality
126     for (int i = 0; i < 100; i++) {
127       int kIdx = i % starterKeys.length;
128       byte[] k = starterKeys[kIdx];
129       byte[] cp = new byte[k.length + 1];
130       System.arraycopy(k, 0, cp, 0, k.length);
131       cp[k.length] = new Integer(i % 256).byteValue();
132       keys.add(cp);
133     }
134     return keys.toArray(new byte [][] {new byte [] {}});
135   }
136 
137 
138   /**
139    * This is for testing the active number of threads that were used while
140    * doing a batch operation. It inserts one row per region via the batch
141    * operation, and then checks the number of active threads.
142    * For HBASE-3553
143    * @throws IOException
144    * @throws InterruptedException
145    * @throws NoSuchFieldException
146    * @throws SecurityException
147    */
148   @Test(timeout=300000)
149   public void testActiveThreadsCount() throws Exception{
150     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
151     List<Row> puts = constructPutRequests(); // creates a Put for every region
152     table.batch(puts);
153     Field poolField = table.getClass().getDeclaredField("pool");
154     poolField.setAccessible(true);
155     ThreadPoolExecutor tExecutor = (ThreadPoolExecutor) poolField.get(table);
156     assertEquals(slaves, tExecutor.getLargestPoolSize());
157     table.close();
158   }
159 
160   @Test(timeout=300000)
161   public void testBatchWithGet() throws Exception {
162     LOG.info("test=testBatchWithGet");
163     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
164 
165     // load test data
166     List<Row> puts = constructPutRequests();
167     table.batch(puts);
168 
169     // create a list of gets and run it
170     List<Row> gets = new ArrayList<Row>();
171     for (byte[] k : KEYS) {
172       Get get = new Get(k);
173       get.addColumn(BYTES_FAMILY, QUALIFIER);
174       gets.add(get);
175     }
176     Result[] multiRes = new Result[gets.size()];
177     table.batch(gets, multiRes);
178 
179     // Same gets using individual call API
180     List<Result> singleRes = new ArrayList<Result>();
181     for (Row get : gets) {
182       singleRes.add(table.get((Get) get));
183     }
184     // Compare results
185     Assert.assertEquals(singleRes.size(), multiRes.length);
186     for (int i = 0; i < singleRes.size(); i++) {
187       Assert.assertTrue(singleRes.get(i).containsColumn(BYTES_FAMILY, QUALIFIER));
188       Cell[] singleKvs = singleRes.get(i).rawCells();
189       Cell[] multiKvs = multiRes[i].rawCells();
190       for (int j = 0; j < singleKvs.length; j++) {
191         Assert.assertEquals(singleKvs[j], multiKvs[j]);
192         Assert.assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(singleKvs[j]), 
193             CellUtil.cloneValue(multiKvs[j])));
194       }
195     }
196     table.close();
197   }
198 
199   @Test
200   public void testBadFam() throws Exception {
201     LOG.info("test=testBadFam");
202     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
203 
204     List<Row> actions = new ArrayList<Row>();
205     Put p = new Put(Bytes.toBytes("row1"));
206     p.add(Bytes.toBytes("bad_family"), Bytes.toBytes("qual"), Bytes.toBytes("value"));
207     actions.add(p);
208     p = new Put(Bytes.toBytes("row2"));
209     p.add(BYTES_FAMILY, Bytes.toBytes("qual"), Bytes.toBytes("value"));
210     actions.add(p);
211 
212     // row1 and row2 should be in the same region.
213 
214     Object [] r = new Object[actions.size()];
215     try {
216       table.batch(actions, r);
217       fail();
218     } catch (RetriesExhaustedWithDetailsException ex) {
219       LOG.debug(ex);
220       // good!
221       assertFalse(ex.mayHaveClusterIssues());
222     }
223     assertEquals(2, r.length);
224     assertTrue(r[0] instanceof Throwable);
225     assertTrue(r[1] instanceof Result);
226     table.close();
227   }
228 
229   @Test (timeout=300000)
230   public void testFlushCommitsNoAbort() throws Exception {
231     LOG.info("test=testFlushCommitsNoAbort");
232     doTestFlushCommits(false);
233   }
234 
235   /**
236    * Only run one Multi test with a forced RegionServer abort. Otherwise, the
237    * unit tests will take an unnecessarily long time to run.
238    *
239    * @throws Exception
240    */
241   @Test (timeout=300000)
242   public void testFlushCommitsWithAbort() throws Exception {
243     LOG.info("test=testFlushCommitsWithAbort");
244     doTestFlushCommits(true);
245   }
246 
247   /**
248    * Set table auto flush to false and test flushing commits
249    * @param doAbort true if abort one regionserver in the testing
250    * @throws Exception
251    */
252   private void doTestFlushCommits(boolean doAbort) throws Exception {
253     // Load the data
254     LOG.info("get new table");
255     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
256     table.setAutoFlush(false, true);
257     table.setWriteBufferSize(10 * 1024 * 1024);
258 
259     LOG.info("constructPutRequests");
260     List<Row> puts = constructPutRequests();
261     for (Row put : puts) {
262       table.put((Put) put);
263     }
264     LOG.info("puts");
265     table.flushCommits();
266     final int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads()
267         .size();
268     assert liveRScount > 0;
269     JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster()
270         .getLiveRegionServerThreads().get(0);
271     if (doAbort) {
272       liveRS.getRegionServer().abort("Aborting for tests",
273           new Exception("doTestFlushCommits"));
274       // If we wait for no regions being online after we abort the server, we
275       // could ensure the master has re-assigned the regions on killed server
276       // after writing successfully. It means the server we aborted is dead
277       // and detected by matser
278       while (liveRS.getRegionServer().getNumberOfOnlineRegions() != 0) {
279         Thread.sleep(10);
280       }
281       // try putting more keys after the abort. same key/qual... just validating
282       // no exceptions thrown
283       puts = constructPutRequests();
284       for (Row put : puts) {
285         table.put((Put) put);
286       }
287 
288       table.flushCommits();
289     }
290 
291     LOG.info("validating loaded data");
292     validateLoadedData(table);
293 
294     // Validate server and region count
295     List<JVMClusterUtil.RegionServerThread> liveRSs = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads();
296     int count = 0;
297     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
298       count++;
299       LOG.info("Count=" + count + ", Alive=" + t.getRegionServer());
300     }
301     LOG.info("Count=" + count);
302     Assert.assertEquals("Server count=" + count + ", abort=" + doAbort,
303         (doAbort ? (liveRScount - 1) : liveRScount), count);
304     for (JVMClusterUtil.RegionServerThread t: liveRSs) {
305       int regions = ProtobufUtil.getOnlineRegions(t.getRegionServer()).size();
306       // Assert.assertTrue("Count of regions=" + regions, regions > 10);
307     }
308     if (doAbort) {
309       UTIL.getMiniHBaseCluster().waitOnRegionServer(0);
310       UTIL.waitFor(15 * 1000, new Waiter.Predicate<Exception>() {
311         @Override
312         public boolean evaluate() throws Exception {
313           return UTIL.getMiniHBaseCluster().getMaster()
314               .getClusterStatus().getServersSize() == (liveRScount - 1);
315         }
316       });
317       UTIL.waitFor(15 * 1000, UTIL.predicateNoRegionsInTransition());
318     }
319 
320     table.close();
321     LOG.info("done");
322   }
323 
324   @Test (timeout=300000)
325   public void testBatchWithPut() throws Exception {
326     LOG.info("test=testBatchWithPut");
327     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
328 
329     // put multiple rows using a batch
330     List<Row> puts = constructPutRequests();
331 
332     Object[] results = table.batch(puts);
333     validateSizeAndEmpty(results, KEYS.length);
334 
335     if (true) {
336       int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
337       assert liveRScount > 0;
338       JVMClusterUtil.RegionServerThread liveRS =
339         UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0);
340       liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut"));
341       puts = constructPutRequests();
342       try {
343         results = table.batch(puts);
344       } catch (RetriesExhaustedWithDetailsException ree) {
345         LOG.info(ree.getExhaustiveDescription());
346         throw ree;
347       } finally {
348         table.close();
349       }
350       validateSizeAndEmpty(results, KEYS.length);
351     }
352 
353     validateLoadedData(table);
354     table.close();
355   }
356 
357   @Test(timeout=300000)
358   public void testBatchWithDelete() throws Exception {
359     LOG.info("test=testBatchWithDelete");
360     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
361 
362     // Load some data
363     List<Row> puts = constructPutRequests();
364     Object[] results = table.batch(puts);
365     validateSizeAndEmpty(results, KEYS.length);
366 
367     // Deletes
368     List<Row> deletes = new ArrayList<Row>();
369     for (int i = 0; i < KEYS.length; i++) {
370       Delete delete = new Delete(KEYS[i]);
371       delete.deleteFamily(BYTES_FAMILY);
372       deletes.add(delete);
373     }
374     results = table.batch(deletes);
375     validateSizeAndEmpty(results, KEYS.length);
376 
377     // Get to make sure ...
378     for (byte[] k : KEYS) {
379       Get get = new Get(k);
380       get.addColumn(BYTES_FAMILY, QUALIFIER);
381       Assert.assertFalse(table.exists(get));
382     }
383     table.close();
384   }
385 
386   @Test(timeout=300000)
387   public void testHTableDeleteWithList() throws Exception {
388     LOG.info("test=testHTableDeleteWithList");
389     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
390 
391     // Load some data
392     List<Row> puts = constructPutRequests();
393     Object[] results = table.batch(puts);
394     validateSizeAndEmpty(results, KEYS.length);
395 
396     // Deletes
397     ArrayList<Delete> deletes = new ArrayList<Delete>();
398     for (int i = 0; i < KEYS.length; i++) {
399       Delete delete = new Delete(KEYS[i]);
400       delete.deleteFamily(BYTES_FAMILY);
401       deletes.add(delete);
402     }
403     table.delete(deletes);
404     Assert.assertTrue(deletes.isEmpty());
405 
406     // Get to make sure ...
407     for (byte[] k : KEYS) {
408       Get get = new Get(k);
409       get.addColumn(BYTES_FAMILY, QUALIFIER);
410       Assert.assertFalse(table.exists(get));
411     }
412     table.close();
413   }
414 
415   @Test(timeout=300000)
416   public void testBatchWithManyColsInOneRowGetAndPut() throws Exception {
417     LOG.info("test=testBatchWithManyColsInOneRowGetAndPut");
418     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
419 
420     List<Row> puts = new ArrayList<Row>();
421     for (int i = 0; i < 100; i++) {
422       Put put = new Put(ONE_ROW);
423       byte[] qual = Bytes.toBytes("column" + i);
424       put.add(BYTES_FAMILY, qual, VALUE);
425       puts.add(put);
426     }
427     Object[] results = table.batch(puts);
428 
429     // validate
430     validateSizeAndEmpty(results, 100);
431 
432     // get the data back and validate that it is correct
433     List<Row> gets = new ArrayList<Row>();
434     for (int i = 0; i < 100; i++) {
435       Get get = new Get(ONE_ROW);
436       byte[] qual = Bytes.toBytes("column" + i);
437       get.addColumn(BYTES_FAMILY, qual);
438       gets.add(get);
439     }
440 
441     Object[] multiRes = table.batch(gets);
442 
443     int idx = 0;
444     for (Object r : multiRes) {
445       byte[] qual = Bytes.toBytes("column" + idx);
446       validateResult(r, qual, VALUE);
447       idx++;
448     }
449     table.close();
450   }
451 
452   @Test(timeout=300000)
453   public void testBatchWithIncrementAndAppend() throws Exception {
454     LOG.info("test=testBatchWithIncrementAndAppend");
455     final byte[] QUAL1 = Bytes.toBytes("qual1");
456     final byte[] QUAL2 = Bytes.toBytes("qual2");
457     final byte[] QUAL3 = Bytes.toBytes("qual3");
458     final byte[] QUAL4 = Bytes.toBytes("qual4");
459     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
460     Delete d = new Delete(ONE_ROW);
461     table.delete(d);
462     Put put = new Put(ONE_ROW);
463     put.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("abc"));
464     put.add(BYTES_FAMILY, QUAL2, Bytes.toBytes(1L));
465     table.put(put);
466 
467     Increment inc = new Increment(ONE_ROW);
468     inc.addColumn(BYTES_FAMILY, QUAL2, 1);
469     inc.addColumn(BYTES_FAMILY, QUAL3, 1);
470 
471     Append a = new Append(ONE_ROW);
472     a.add(BYTES_FAMILY, QUAL1, Bytes.toBytes("def"));
473     a.add(BYTES_FAMILY, QUAL4, Bytes.toBytes("xyz"));
474     List<Row> actions = new ArrayList<Row>();
475     actions.add(inc);
476     actions.add(a);
477 
478     Object[] multiRes = table.batch(actions);
479     validateResult(multiRes[1], QUAL1, Bytes.toBytes("abcdef"));
480     validateResult(multiRes[1], QUAL4, Bytes.toBytes("xyz"));
481     validateResult(multiRes[0], QUAL2, Bytes.toBytes(2L));
482     validateResult(multiRes[0], QUAL3, Bytes.toBytes(1L));
483     table.close();
484   }
485 
486   @Test(timeout=300000)
487   public void testNonceCollision() throws Exception {
488     LOG.info("test=testNonceCollision");
489     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
490     Put put = new Put(ONE_ROW);
491     put.add(BYTES_FAMILY, QUALIFIER, Bytes.toBytes(0L));
492 
493     // Replace nonce manager with the one that returns each nonce twice.
494     NonceGenerator cnm = new PerClientRandomNonceGenerator() {
495       long lastNonce = -1;
496       @Override
497       public synchronized long newNonce() {
498         long nonce = 0;
499         if (lastNonce == -1) {
500           lastNonce = nonce = super.newNonce();
501         } else {
502           nonce = lastNonce;
503           lastNonce = -1L;
504         }
505         return nonce;
506       }
507     };
508     NonceGenerator oldCnm =
509         HConnectionManager.injectNonceGeneratorForTesting(table.getConnection(), cnm);
510 
511     // First test sequential requests.
512     try {
513       Increment inc = new Increment(ONE_ROW);
514       inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
515       table.increment(inc);
516       inc = new Increment(ONE_ROW);
517       inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
518       try {
519         table.increment(inc);
520         fail("Should have thrown an exception");
521       } catch (OperationConflictException ex) {
522       }
523       Get get = new Get(ONE_ROW);
524       get.addColumn(BYTES_FAMILY, QUALIFIER);
525       Result result = table.get(get);
526       validateResult(result, QUALIFIER, Bytes.toBytes(1L));
527 
528       // Now run a bunch of requests in parallel, exactly half should succeed.
529       int numRequests = 40;
530       final CountDownLatch startedLatch = new CountDownLatch(numRequests);
531       final CountDownLatch startLatch = new CountDownLatch(1);
532       final CountDownLatch doneLatch = new CountDownLatch(numRequests);
533       for (int i = 0; i < numRequests; ++i) {
534         Runnable r = new Runnable() {
535           @Override
536           public void run() {
537             HTable table = null;
538             try {
539               table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
540             } catch (IOException e) {
541               fail("Not expected");
542             }
543             Increment inc = new Increment(ONE_ROW);
544             inc.addColumn(BYTES_FAMILY, QUALIFIER, 1L);
545             startedLatch.countDown();
546             try {
547               startLatch.await();
548             } catch (InterruptedException e) {
549               fail("Not expected");
550             }
551             try {
552               table.increment(inc);
553             } catch (OperationConflictException ex) { // Some threads are expected to fail.
554             } catch (IOException ioEx) {
555               fail("Not expected");
556             }
557             doneLatch.countDown();
558           }
559         };
560         Threads.setDaemonThreadRunning(new Thread(r));
561       }
562       startedLatch.await(); // Wait until all threads are ready...
563       startLatch.countDown(); // ...and unleash the herd!
564       doneLatch.await();
565       // Now verify
566       get = new Get(ONE_ROW);
567       get.addColumn(BYTES_FAMILY, QUALIFIER);
568       result = table.get(get);
569       validateResult(result, QUALIFIER, Bytes.toBytes((numRequests / 2) + 1L));
570       table.close();
571     } finally {
572       HConnectionManager.injectNonceGeneratorForTesting(table.getConnection(), oldCnm);
573     }
574   }
575 
576   @Test(timeout=300000)
577   public void testBatchWithMixedActions() throws Exception {
578     LOG.info("test=testBatchWithMixedActions");
579     HTable table = new HTable(UTIL.getConfiguration(), TEST_TABLE);
580 
581     // Load some data to start
582     Object[] results = table.batch(constructPutRequests());
583     validateSizeAndEmpty(results, KEYS.length);
584 
585     // Batch: get, get, put(new col), delete, get, get of put, get of deleted,
586     // put
587     List<Row> actions = new ArrayList<Row>();
588 
589     byte[] qual2 = Bytes.toBytes("qual2");
590     byte[] val2 = Bytes.toBytes("putvalue2");
591 
592     // 0 get
593     Get get = new Get(KEYS[10]);
594     get.addColumn(BYTES_FAMILY, QUALIFIER);
595     actions.add(get);
596 
597     // 1 get
598     get = new Get(KEYS[11]);
599     get.addColumn(BYTES_FAMILY, QUALIFIER);
600     actions.add(get);
601 
602     // 2 put of new column
603     Put put = new Put(KEYS[10]);
604     put.add(BYTES_FAMILY, qual2, val2);
605     actions.add(put);
606 
607     // 3 delete
608     Delete delete = new Delete(KEYS[20]);
609     delete.deleteFamily(BYTES_FAMILY);
610     actions.add(delete);
611 
612     // 4 get
613     get = new Get(KEYS[30]);
614     get.addColumn(BYTES_FAMILY, QUALIFIER);
615     actions.add(get);
616 
617     // There used to be a 'get' of a previous put here, but removed
618     // since this API really cannot guarantee order in terms of mixed
619     // get/puts.
620 
621     // 5 put of new column
622     put = new Put(KEYS[40]);
623     put.add(BYTES_FAMILY, qual2, val2);
624     actions.add(put);
625 
626     results = table.batch(actions);
627 
628     // Validation
629 
630     validateResult(results[0]);
631     validateResult(results[1]);
632     validateEmpty(results[2]);
633     validateEmpty(results[3]);
634     validateResult(results[4]);
635     validateEmpty(results[5]);
636 
637     // validate last put, externally from the batch
638     get = new Get(KEYS[40]);
639     get.addColumn(BYTES_FAMILY, qual2);
640     Result r = table.get(get);
641     validateResult(r, qual2, val2);
642 
643     table.close();
644   }
645 
646   // // Helper methods ////
647 
648   private void validateResult(Object r) {
649     validateResult(r, QUALIFIER, VALUE);
650   }
651 
652   private void validateResult(Object r1, byte[] qual, byte[] val) {
653     Result r = (Result)r1;
654     Assert.assertTrue(r.containsColumn(BYTES_FAMILY, qual));
655     byte[] value = r.getValue(BYTES_FAMILY, qual);
656     if (0 != Bytes.compareTo(val, value)) {
657       fail("Expected [" + Bytes.toStringBinary(val)
658           + "] but got [" + Bytes.toStringBinary(value) + "]");
659     }
660   }
661 
662   private List<Row> constructPutRequests() {
663     List<Row> puts = new ArrayList<Row>();
664     for (byte[] k : KEYS) {
665       Put put = new Put(k);
666       put.add(BYTES_FAMILY, QUALIFIER, VALUE);
667       puts.add(put);
668     }
669     return puts;
670   }
671 
672   private void validateLoadedData(HTable table) throws IOException {
673     // get the data back and validate that it is correct
674     for (byte[] k : KEYS) {
675       Get get = new Get(k);
676       get.addColumn(BYTES_FAMILY, QUALIFIER);
677       Result r = table.get(get);
678       Assert.assertTrue(r.containsColumn(BYTES_FAMILY, QUALIFIER));
679       Assert.assertEquals(0, Bytes.compareTo(VALUE, r
680           .getValue(BYTES_FAMILY, QUALIFIER)));
681     }
682   }
683 
684   private void validateEmpty(Object r1) {
685     Result result = (Result)r1;
686     Assert.assertTrue(result != null);
687     Assert.assertTrue(result.getRow() == null);
688     Assert.assertEquals(0, result.rawCells().length);
689   }
690 
691   private void validateSizeAndEmpty(Object[] results, int expectedSize) {
692     // Validate got back the same number of Result objects, all empty
693     Assert.assertEquals(expectedSize, results.length);
694     for (Object result : results) {
695       validateEmpty(result);
696     }
697   }
698 }