View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Arrays;
24  import java.util.List;
25  import java.util.Random;
26  import java.util.concurrent.CountDownLatch;
27  import java.util.concurrent.atomic.AtomicInteger;
28  import java.util.concurrent.atomic.AtomicLong;
29  
30  import org.apache.commons.logging.Log;
31  import org.apache.commons.logging.LogFactory;
32  import org.apache.hadoop.conf.Configuration;
33  import org.apache.hadoop.fs.FileSystem;
34  import org.apache.hadoop.fs.Path;
35  import org.apache.hadoop.hbase.HBaseConfiguration;
36  import org.apache.hadoop.hbase.HBaseTestCase;
37  import org.apache.hadoop.hbase.HBaseTestingUtility;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HTableDescriptor;
42  import org.apache.hadoop.hbase.KeyValue;
43  import org.apache.hadoop.hbase.MediumTests;
44  import org.apache.hadoop.hbase.MultithreadedTestUtil;
45  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
46  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
47  import org.apache.hadoop.hbase.TableName;
48  import org.apache.hadoop.hbase.client.Append;
49  import org.apache.hadoop.hbase.client.Delete;
50  import org.apache.hadoop.hbase.client.Get;
51  import org.apache.hadoop.hbase.client.Increment;
52  import org.apache.hadoop.hbase.client.Mutation;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.RowMutations;
56  import org.apache.hadoop.hbase.client.Scan;
57  import org.apache.hadoop.hbase.filter.BinaryComparator;
58  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
59  import org.apache.hadoop.hbase.io.HeapSize;
60  import org.apache.hadoop.hbase.regionserver.wal.HLog;
61  import org.apache.hadoop.hbase.util.Bytes;
62  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
63  import org.junit.experimental.categories.Category;
64  
65  
66  /**
67   * Testing of HRegion.incrementColumnValue, HRegion.increment,
68   * and HRegion.append
69   */
70  @Category(MediumTests.class) // Starts 100 threads
71  public class TestAtomicOperation extends HBaseTestCase {
72    static final Log LOG = LogFactory.getLog(TestAtomicOperation.class);
73  
74    HRegion region = null;
75    private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
76    private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
77  
78  
79    // Test names
80    static final byte[] tableName = Bytes.toBytes("testtable");;
81    static final byte[] qual1 = Bytes.toBytes("qual1");
82    static final byte[] qual2 = Bytes.toBytes("qual2");
83    static final byte[] qual3 = Bytes.toBytes("qual3");
84    static final byte[] value1 = Bytes.toBytes("value1");
85    static final byte[] value2 = Bytes.toBytes("value2");
86    static final byte [] row = Bytes.toBytes("rowA");
87    static final byte [] row2 = Bytes.toBytes("rowB");
88  
89    /**
90     * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
91     */
92    @Override
93    protected void setUp() throws Exception {
94      super.setUp();
95    }
96  
97    @Override
98    protected void tearDown() throws Exception {
99      super.tearDown();
100     EnvironmentEdgeManagerTestHelper.reset();
101   }
102 
103   //////////////////////////////////////////////////////////////////////////////
104   // New tests that doesn't spin up a mini cluster but rather just test the
105   // individual code pieces in the HRegion. 
106   //////////////////////////////////////////////////////////////////////////////
107 
108   /**
109    * Test basic append operation.
110    * More tests in
111    * @see org.apache.hadoop.hbase.client.TestFromClientSide#testAppend()
112    */
113   public void testAppend() throws IOException {
114     initHRegion(tableName, getName(), fam1);
115     String v1 = "Ultimate Answer to the Ultimate Question of Life,"+
116     " The Universe, and Everything";
117     String v2 = " is... 42.";
118     Append a = new Append(row);
119     a.setReturnResults(false);
120     a.add(fam1, qual1, Bytes.toBytes(v1));
121     a.add(fam1, qual2, Bytes.toBytes(v2));
122     assertNull(region.append(a));
123     a = new Append(row);
124     a.add(fam1, qual1, Bytes.toBytes(v2));
125     a.add(fam1, qual2, Bytes.toBytes(v1));
126     Result result = region.append(a);
127     assertEquals(0, Bytes.compareTo(Bytes.toBytes(v1+v2), result.getValue(fam1, qual1)));
128     assertEquals(0, Bytes.compareTo(Bytes.toBytes(v2+v1), result.getValue(fam1, qual2)));
129   }
130 
131   /**
132    * Test multi-threaded increments.
133    */
134   public void testIncrementMultiThreads() throws IOException {
135 
136     LOG.info("Starting test testIncrementMultiThreads");
137     // run a with mixed column families (1 and 3 versions)
138     initHRegion(tableName, getName(), new int[] {1,3}, fam1, fam2);
139 
140     // create 100 threads, each will increment by its own quantity
141     int numThreads = 100;
142     int incrementsPerThread = 1000;
143     Incrementer[] all = new Incrementer[numThreads];
144     int expectedTotal = 0;
145 
146     // create all threads
147     for (int i = 0; i < numThreads; i++) {
148       all[i] = new Incrementer(region, i, i, incrementsPerThread);
149       expectedTotal += (i * incrementsPerThread);
150     }
151 
152     // run all threads
153     for (int i = 0; i < numThreads; i++) {
154       all[i].start();
155     }
156 
157     // wait for all threads to finish
158     for (int i = 0; i < numThreads; i++) {
159       try {
160         all[i].join();
161       } catch (InterruptedException e) {
162       }
163     }
164     assertICV(row, fam1, qual1, expectedTotal);
165     assertICV(row, fam1, qual2, expectedTotal*2);
166     assertICV(row, fam2, qual3, expectedTotal*3);
167     LOG.info("testIncrementMultiThreads successfully verified that total is " +
168              expectedTotal);
169   }
170 
171 
172   private void assertICV(byte [] row,
173                          byte [] familiy,
174                          byte[] qualifier,
175                          long amount) throws IOException {
176     // run a get and see?
177     Get get = new Get(row);
178     get.addColumn(familiy, qualifier);
179     Result result = region.get(get);
180     assertEquals(1, result.size());
181 
182     KeyValue kv = result.raw()[0];
183     long r = Bytes.toLong(kv.getValue());
184     assertEquals(amount, r);
185   }
186 
187   private void initHRegion (byte [] tableName, String callingMethod,
188       byte[] ... families)
189     throws IOException {
190     initHRegion(tableName, callingMethod, null, families);
191   }
192 
193   private void initHRegion (byte [] tableName, String callingMethod, int [] maxVersions,
194     byte[] ... families)
195   throws IOException {
196     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
197     int i=0;
198     for(byte [] family : families) {
199       HColumnDescriptor hcd = new HColumnDescriptor(family);
200       hcd.setMaxVersions(maxVersions != null ? maxVersions[i++] : 1);
201       htd.addFamily(hcd);
202     }
203     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
204     Path path = new Path(DIR + callingMethod);
205     if (fs.exists(path)) {
206       if (!fs.delete(path, true)) {
207         throw new IOException("Failed delete of " + path);
208       }
209     }
210     region = HRegion.createHRegion(info, path, HBaseConfiguration.create(), htd);
211   }
212 
213   /**
214    * A thread that makes a few increment calls
215    */
216   public static class Incrementer extends Thread {
217 
218     private final HRegion region;
219     private final int numIncrements;
220     private final int amount;
221 
222 
223     public Incrementer(HRegion region,
224         int threadNumber, int amount, int numIncrements) {
225       this.region = region;
226       this.numIncrements = numIncrements;
227       this.amount = amount;
228       setDaemon(true);
229     }
230 
231     @Override
232     public void run() {
233       for (int i=0; i<numIncrements; i++) {
234         try {
235           Increment inc = new Increment(row);
236           inc.addColumn(fam1, qual1, amount);
237           inc.addColumn(fam1, qual2, amount*2);
238           inc.addColumn(fam2, qual3, amount*3);
239           region.increment(inc);
240 
241           // verify: Make sure we only see completed increments
242           Get g = new Get(row);
243           Result result = region.get(g);
244           assertEquals(Bytes.toLong(result.getValue(fam1, qual1))*2, Bytes.toLong(result.getValue(fam1, qual2))); 
245           assertEquals(Bytes.toLong(result.getValue(fam1, qual1))*3, Bytes.toLong(result.getValue(fam2, qual3)));
246         } catch (IOException e) {
247           e.printStackTrace();
248         }
249       }
250     }
251   }
252 
253   public void testAppendMultiThreads() throws IOException {
254     LOG.info("Starting test testAppendMultiThreads");
255     // run a with mixed column families (1 and 3 versions)
256     initHRegion(tableName, getName(), new int[] {1,3}, fam1, fam2);
257 
258     int numThreads = 100;
259     int opsPerThread = 100;
260     AtomicOperation[] all = new AtomicOperation[numThreads];
261     final byte[] val = new byte[]{1};
262 
263     AtomicInteger failures = new AtomicInteger(0);
264     // create all threads
265     for (int i = 0; i < numThreads; i++) {
266       all[i] = new AtomicOperation(region, opsPerThread, null, failures) {
267         @Override
268         public void run() {
269           for (int i=0; i<numOps; i++) {
270             try {
271               Append a = new Append(row);
272               a.add(fam1, qual1, val);
273               a.add(fam1, qual2, val);
274               a.add(fam2, qual3, val);
275               region.append(a);
276 
277               Get g = new Get(row);
278               Result result = region.get(g);
279               assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam1, qual2).length); 
280               assertEquals(result.getValue(fam1, qual1).length, result.getValue(fam2, qual3).length); 
281             } catch (IOException e) {
282               e.printStackTrace();
283               failures.incrementAndGet();
284               fail();
285             }
286           }
287         }
288       };
289     }
290 
291     // run all threads
292     for (int i = 0; i < numThreads; i++) {
293       all[i].start();
294     }
295 
296     // wait for all threads to finish
297     for (int i = 0; i < numThreads; i++) {
298       try {
299         all[i].join();
300       } catch (InterruptedException e) {
301       }
302     }
303     assertEquals(0, failures.get());
304     Get g = new Get(row);
305     Result result = region.get(g);
306     assertEquals(result.getValue(fam1, qual1).length, 10000);
307     assertEquals(result.getValue(fam1, qual2).length, 10000);
308     assertEquals(result.getValue(fam2, qual3).length, 10000);
309   }
310   /**
311    * Test multi-threaded row mutations.
312    */
313   public void testRowMutationMultiThreads() throws IOException {
314 
315     LOG.info("Starting test testRowMutationMultiThreads");
316     initHRegion(tableName, getName(), fam1);
317 
318     // create 10 threads, each will alternate between adding and
319     // removing a column
320     int numThreads = 10;
321     int opsPerThread = 500;
322     AtomicOperation[] all = new AtomicOperation[numThreads];
323 
324     AtomicLong timeStamps = new AtomicLong(0);
325     AtomicInteger failures = new AtomicInteger(0);
326     // create all threads
327     for (int i = 0; i < numThreads; i++) {
328       all[i] = new AtomicOperation(region, opsPerThread, timeStamps, failures) {
329         @Override
330         public void run() {
331           boolean op = true;
332           for (int i=0; i<numOps; i++) {
333             try {
334               // throw in some flushes
335               if (i%10==0) {
336                 synchronized(region) {
337                   LOG.debug("flushing");
338                   region.flushcache();
339                   if (i%100==0) {
340                     region.compactStores();
341                   }
342                 }
343               }
344               long ts = timeStamps.incrementAndGet();
345               RowMutations rm = new RowMutations(row);
346               if (op) {
347                 Put p = new Put(row, ts);
348                 p.add(fam1, qual1, value1);
349                 rm.add(p);
350                 Delete d = new Delete(row);
351                 d.deleteColumns(fam1, qual2, ts);
352                 rm.add(d);
353               } else {
354                 Delete d = new Delete(row);
355                 d.deleteColumns(fam1, qual1, ts);
356                 rm.add(d);
357                 Put p = new Put(row, ts);
358                 p.add(fam1, qual2, value2);
359                 rm.add(p);
360               }
361               region.mutateRow(rm);
362               op ^= true;
363               // check: should always see exactly one column
364               Get g = new Get(row);
365               Result r = region.get(g);
366               if (r.size() != 1) {
367                 LOG.debug(r);
368                 failures.incrementAndGet();
369                 fail();
370               }
371             } catch (IOException e) {
372               e.printStackTrace();
373               failures.incrementAndGet();
374               fail();
375             }
376           }
377         }
378       };
379     }
380 
381     // run all threads
382     for (int i = 0; i < numThreads; i++) {
383       all[i].start();
384     }
385 
386     // wait for all threads to finish
387     for (int i = 0; i < numThreads; i++) {
388       try {
389         all[i].join();
390       } catch (InterruptedException e) {
391       }
392     }
393     assertEquals(0, failures.get());
394   }
395 
396 
397   /**
398    * Test multi-threaded region mutations.
399    */
400   public void testMultiRowMutationMultiThreads() throws IOException {
401 
402     LOG.info("Starting test testMultiRowMutationMultiThreads");
403     initHRegion(tableName, getName(), fam1);
404 
405     // create 10 threads, each will alternate between adding and
406     // removing a column
407     int numThreads = 10;
408     int opsPerThread = 500;
409     AtomicOperation[] all = new AtomicOperation[numThreads];
410 
411     AtomicLong timeStamps = new AtomicLong(0);
412     AtomicInteger failures = new AtomicInteger(0);
413     final List<byte[]> rowsToLock = Arrays.asList(row, row2);
414     // create all threads
415     for (int i = 0; i < numThreads; i++) {
416       all[i] = new AtomicOperation(region, opsPerThread, timeStamps, failures) {
417         @Override
418         public void run() {
419           boolean op = true;
420           for (int i=0; i<numOps; i++) {
421             try {
422               // throw in some flushes
423               if (i%10==0) {
424                 synchronized(region) {
425                   LOG.debug("flushing");
426                   region.flushcache();
427                   if (i%100==0) {
428                     region.compactStores();
429                   }
430                 }
431               }
432               long ts = timeStamps.incrementAndGet();
433               List<Mutation> mrm = new ArrayList<Mutation>();
434               if (op) {
435                 Put p = new Put(row2, ts);
436                 p.add(fam1, qual1, value1);
437                 mrm.add(p);
438                 Delete d = new Delete(row);
439                 d.deleteColumns(fam1, qual1, ts);
440                 mrm.add(d);
441               } else {
442                 Delete d = new Delete(row2);
443                 d.deleteColumns(fam1, qual1, ts);
444                 mrm.add(d);
445                 Put p = new Put(row, ts);
446                 p.add(fam1, qual1, value2);
447                 mrm.add(p);
448               }
449               region.mutateRowsWithLocks(mrm, rowsToLock);
450               op ^= true;
451               // check: should always see exactly one column
452               Scan s = new Scan(row);
453               RegionScanner rs = region.getScanner(s);
454               List<KeyValue> r = new ArrayList<KeyValue>();
455               while(rs.next(r));
456               rs.close();
457               if (r.size() != 1) {
458                 LOG.debug(r);
459                 failures.incrementAndGet();
460                 fail();
461               }
462             } catch (IOException e) {
463               e.printStackTrace();
464               failures.incrementAndGet();
465               fail();
466             }
467           }
468         }
469       };
470     }
471 
472     // run all threads
473     for (int i = 0; i < numThreads; i++) {
474       all[i].start();
475     }
476 
477     // wait for all threads to finish
478     for (int i = 0; i < numThreads; i++) {
479       try {
480         all[i].join();
481       } catch (InterruptedException e) {
482       }
483     }
484     assertEquals(0, failures.get());
485   }
486 
487   public static class AtomicOperation extends Thread {
488     protected final HRegion region;
489     protected final int numOps;
490     protected final AtomicLong timeStamps;
491     protected final AtomicInteger failures;
492     protected final Random r = new Random();
493 
494     public AtomicOperation(HRegion region, int numOps, AtomicLong timeStamps,
495         AtomicInteger failures) {
496       this.region = region;
497       this.numOps = numOps;
498       this.timeStamps = timeStamps;
499       this.failures = failures;
500     }
501   }
502   
503   private static CountDownLatch latch = new CountDownLatch(1);
504   private enum TestStep {
505     INIT,                  // initial put of 10 to set value of the cell
506     PUT_STARTED,           // began doing a put of 50 to cell
507     PUT_COMPLETED,         // put complete (released RowLock, but may not have advanced MVCC).
508     CHECKANDPUT_STARTED,   // began checkAndPut: if 10 -> 11
509     CHECKANDPUT_COMPLETED  // completed checkAndPut
510     // NOTE: at the end of these steps, the value of the cell should be 50, not 11!
511   }
512   private static volatile TestStep testStep = TestStep.INIT;
513   private final String family = "f1";
514      
515   /**
516    * Test written as a verifier for HBASE-7051, CheckAndPut should properly read
517    * MVCC. 
518    * 
519    * Moved into TestAtomicOperation from its original location, TestHBase7051
520    */
521   public void testPutAndCheckAndPutInParallel() throws Exception {
522 
523     final String tableName = "testPutAndCheckAndPut";
524     Configuration conf = HBaseConfiguration.create();
525     conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
526     final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(
527         Bytes.toBytes(tableName), tableName, conf, Bytes.toBytes(family));
528 
529     Put[] puts = new Put[1];
530     Put put = new Put(Bytes.toBytes("r1"));
531     put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
532     puts[0] = put;
533     
534     region.batchMutate(puts);
535     MultithreadedTestUtil.TestContext ctx =
536       new MultithreadedTestUtil.TestContext(conf);
537     ctx.addThread(new PutThread(ctx, region));
538     ctx.addThread(new CheckAndPutThread(ctx, region));
539     ctx.startThreads();
540     while (testStep != TestStep.CHECKANDPUT_COMPLETED) {
541       Thread.sleep(100);
542     }
543     ctx.stop();
544     Scan s = new Scan();
545     RegionScanner scanner = region.getScanner(s);
546     List<KeyValue> results = new ArrayList<KeyValue>();
547     scanner.next(results, 2);
548     for (KeyValue keyValue : results) {
549       assertEquals("50",Bytes.toString(keyValue.getValue()));
550     }
551 
552   }
553 
554   private class PutThread extends TestThread {
555     private MockHRegion region;
556     PutThread(TestContext ctx, MockHRegion region) {
557       super(ctx);
558       this.region = region;
559     }
560 
561     public void doWork() throws Exception {
562       Put[] puts = new Put[1];
563       Put put = new Put(Bytes.toBytes("r1"));
564       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
565       puts[0] = put;
566       testStep = TestStep.PUT_STARTED;
567       region.batchMutate(puts);
568     }
569   }
570 
571   private class CheckAndPutThread extends TestThread {
572     private MockHRegion region;
573     CheckAndPutThread(TestContext ctx, MockHRegion region) {
574       super(ctx);
575       this.region = region;
576    }
577 
578     public void doWork() throws Exception {
579       Put[] puts = new Put[1];
580       Put put = new Put(Bytes.toBytes("r1"));
581       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
582       puts[0] = put;
583       while (testStep != TestStep.PUT_COMPLETED) {
584         Thread.sleep(100);
585       }
586       testStep = TestStep.CHECKANDPUT_STARTED;
587       region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
588         CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, true);
589       testStep = TestStep.CHECKANDPUT_COMPLETED;
590     }
591   }
592 
593   public static class MockHRegion extends HRegion {
594 
595     public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
596         final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
597       super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
598     }
599 
600     @Override
601     public RowLock getRowLock(final byte[] row, boolean waitForLock) throws IOException {
602       if (testStep == TestStep.CHECKANDPUT_STARTED) {
603         latch.countDown();
604       }
605       return new WrappedRowLock(super.getRowLock(row, waitForLock));
606     }
607     
608     public class WrappedRowLock extends RowLock {
609 
610       private WrappedRowLock(RowLock rowLock) {
611         super(rowLock.context);
612       }
613 
614       @Override
615       public void release() {
616         if (testStep == TestStep.INIT) {
617           super.release();
618           return;
619         }
620 
621         if (testStep == TestStep.PUT_STARTED) {
622           try {
623             testStep = TestStep.PUT_COMPLETED;
624             super.release();
625             // put has been written to the memstore and the row lock has been released, but the
626             // MVCC has not been advanced.  Prior to fixing HBASE-7051, the following order of
627             // operations would cause the non-atomicity to show up:
628             // 1) Put releases row lock (where we are now)
629             // 2) CheckAndPut grabs row lock and reads the value prior to the put (10)
630             //    because the MVCC has not advanced
631             // 3) Put advances MVCC
632             // So, in order to recreate this order, we wait for the checkAndPut to grab the rowLock
633             // (see below), and then wait some more to give the checkAndPut time to read the old
634             // value.
635             latch.await();
636             Thread.sleep(1000);
637           } catch (InterruptedException e) {
638             Thread.currentThread().interrupt();
639           }
640         }
641         else if (testStep == TestStep.CHECKANDPUT_STARTED) {
642           super.release();
643         }
644       }
645     }
646   }
647 }