1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import static org.junit.Assert.assertEquals;
21  
22  import java.io.IOException;
23  import java.util.ArrayList;
24  import java.util.List;
25  import java.util.concurrent.CountDownLatch;
26  
27  import org.apache.hadoop.conf.Configuration;
28  import org.apache.hadoop.fs.FileSystem;
29  import org.apache.hadoop.fs.Path;
30  import org.apache.hadoop.hbase.HBaseConfiguration;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.HRegionInfo;
33  import org.apache.hadoop.hbase.HTableDescriptor;
34  import org.apache.hadoop.hbase.KeyValue;
35  import org.apache.hadoop.hbase.SmallTests;
36  import org.apache.hadoop.hbase.MultithreadedTestUtil;
37  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
38  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
39  import org.apache.hadoop.hbase.client.Mutation;
40  import org.apache.hadoop.hbase.client.Put;
41  import org.apache.hadoop.hbase.client.Scan;
42  import org.apache.hadoop.hbase.filter.BinaryComparator;
43  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
44  import org.apache.hadoop.hbase.io.HeapSize;
45  import org.apache.hadoop.hbase.regionserver.wal.HLog;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.HashedBytes;
48  import org.apache.hadoop.hbase.util.Pair;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  import com.google.common.collect.Lists;
53  
54  /**
55   * Test of HBASE-7051; that checkAndPuts and puts behave atomically with respect to each other.
56   * Rather than perform a bunch of trials to verify atomicity, this test recreates a race condition
57   * that causes the test to fail if checkAndPut doesn't wait for outstanding put transactions
58   * to complete.  It does this by invasively overriding HRegion function to affect the timing of
59   * the operations.
60   */
61  @Category(SmallTests.class)
62  public class TestHBase7051 {
63  
64    private static CountDownLatch latch = new CountDownLatch(1);
65    private enum TestStep {
66      INIT,                  // initial put of 10 to set value of the cell
67      PUT_STARTED,           // began doing a put of 50 to cell
68      PUT_COMPLETED,         // put complete (released RowLock, but may not have advanced MVCC).
69      CHECKANDPUT_STARTED,   // began checkAndPut: if 10 -> 11
70      CHECKANDPUT_COMPLETED  // completed checkAndPut
71      // NOTE: at the end of these steps, the value of the cell should be 50, not 11!
72    }
73    private static volatile TestStep testStep = TestStep.INIT;
74    private final String family = "f1";
75    	 
76    @Test
77    public void testPutAndCheckAndPutInParallel() throws Exception {
78  
79      final String tableName = "testPutAndCheckAndPut";
80      Configuration conf = HBaseConfiguration.create();
81      conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
82      final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName),
83          tableName, conf, Bytes.toBytes(family));
84  
85      List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
86      Put[] puts = new Put[1];
87      Put put = new Put(Bytes.toBytes("r1"));
88      put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
89      puts[0] = put;
90      Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
91  
92      putsAndLocks.add(pair);
93  
94      region.batchMutate(putsAndLocks.toArray(new Pair[0]));
95      MultithreadedTestUtil.TestContext ctx =
96        new MultithreadedTestUtil.TestContext(conf);
97      ctx.addThread(new PutThread(ctx, region));
98      ctx.addThread(new CheckAndPutThread(ctx, region));
99      ctx.startThreads();
100     while (testStep != TestStep.CHECKANDPUT_COMPLETED) {
101       Thread.sleep(100);
102     }
103     ctx.stop();
104     Scan s = new Scan();
105     RegionScanner scanner = region.getScanner(s);
106     List<KeyValue> results = new ArrayList<KeyValue>();
107     scanner.next(results, 2);
108     for (KeyValue keyValue : results) {
109       assertEquals("50",Bytes.toString(keyValue.getValue()));
110     }
111 
112   }
113 
114   private class PutThread extends TestThread {
115     private MockHRegion region;
116     PutThread(TestContext ctx, MockHRegion region) {
117       super(ctx);
118       this.region = region;
119     }
120 
121     public void doWork() throws Exception {
122       List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
123       Put[] puts = new Put[1];
124       Put put = new Put(Bytes.toBytes("r1"));
125       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
126       puts[0] = put;
127       Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
128       putsAndLocks.add(pair);
129       testStep = TestStep.PUT_STARTED;
130       region.batchMutate(putsAndLocks.toArray(new Pair[0]));
131     }
132   }
133 
134   private class CheckAndPutThread extends TestThread {
135     private MockHRegion region;
136     CheckAndPutThread(TestContext ctx, MockHRegion region) {
137       super(ctx);
138       this.region = region;
139    }
140 
141     public void doWork() throws Exception {
142       Put[] puts = new Put[1];
143       Put put = new Put(Bytes.toBytes("r1"));
144       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
145       puts[0] = put;
146       while (testStep != TestStep.PUT_COMPLETED) {
147         Thread.sleep(100);
148       }
149       testStep = TestStep.CHECKANDPUT_STARTED;
150       region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
151         CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true);
152       testStep = TestStep.CHECKANDPUT_COMPLETED;
153     }
154   }
155 
156   public static class MockHRegion extends HRegion {
157 
158     public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
159         final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
160       super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
161     }
162 
163     @Override
164     public void releaseRowLock(Integer lockId) {
165       if (testStep == TestStep.INIT) {
166         super.releaseRowLock(lockId);
167         return;
168       }
169 
170       if (testStep == TestStep.PUT_STARTED) {
171         try {
172           testStep = TestStep.PUT_COMPLETED;
173           super.releaseRowLock(lockId);
174           // put has been written to the memstore and the row lock has been released, but the
175           // MVCC has not been advanced.  Prior to fixing HBASE-7051, the following order of
176           // operations would cause the non-atomicity to show up:
177           // 1) Put releases row lock (where we are now)
178           // 2) CheckAndPut grabs row lock and reads the value prior to the put (10)
179           //    because the MVCC has not advanced
180           // 3) Put advances MVCC
181           // So, in order to recreate this order, we wait for the checkAndPut to grab the rowLock
182           // (see below), and then wait some more to give the checkAndPut time to read the old
183           // value.
184           latch.await();
185           Thread.sleep(1000);
186         } catch (InterruptedException e) {
187           Thread.currentThread().interrupt();
188         }
189       }
190       else if (testStep == TestStep.CHECKANDPUT_STARTED) {
191         super.releaseRowLock(lockId);
192       }
193     }
194 
195     @Override
196     public Integer getLock(Integer lockid, HashedBytes row, boolean waitForLock) throws IOException {
197       if (testStep == TestStep.CHECKANDPUT_STARTED) {
198         latch.countDown();
199       }
200       return super.getLock(lockid, row, waitForLock);
201     }
202 
203   }
204 
205 }