View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  
21  import java.io.IOException;
22  import java.util.Random;
23  
24  import org.apache.commons.logging.Log;
25  import org.apache.commons.logging.LogFactory;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.*;
28  import org.apache.hadoop.fs.Path;
29  import org.apache.hadoop.hbase.HConstants.OperationStatusCode;
30  import org.apache.hadoop.hbase.client.Get;
31  import org.apache.hadoop.hbase.client.Put;
32  import org.apache.hadoop.hbase.client.Result;
33  import org.apache.hadoop.hbase.util.Bytes;
34  import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
35  
36  import org.junit.experimental.categories.Category;
37  
38  
39  /**
40   * Testing of multiPut in parallel.
41   *
42   */
43  @Category(MediumTests.class)
44  public class TestParallelPut extends HBaseTestCase {
45    static final Log LOG = LogFactory.getLog(TestParallelPut.class);
46  
47    private static HRegion region = null;
48    private static HBaseTestingUtility hbtu = new HBaseTestingUtility();
49    private static final String DIR = hbtu.getDataTestDir() + "/TestParallelPut/";
50  
51    // Test names
52    static final byte[] tableName = Bytes.toBytes("testtable");;
53    static final byte[] qual1 = Bytes.toBytes("qual1");
54    static final byte[] qual2 = Bytes.toBytes("qual2");
55    static final byte[] qual3 = Bytes.toBytes("qual3");
56    static final byte[] value1 = Bytes.toBytes("value1");
57    static final byte[] value2 = Bytes.toBytes("value2");
58    static final byte [] row = Bytes.toBytes("rowA");
59    static final byte [] row2 = Bytes.toBytes("rowB");
60  
61    /**
62     * @see org.apache.hadoop.hbase.HBaseTestCase#setUp()
63     */
64    @Override
65    protected void setUp() throws Exception {
66      super.setUp();
67    }
68  
69    @Override
70    protected void tearDown() throws Exception {
71      super.tearDown();
72      EnvironmentEdgeManagerTestHelper.reset();
73    }
74  
75    //////////////////////////////////////////////////////////////////////////////
76    // New tests that don't spin up a mini cluster but rather just test the
77    // individual code pieces in the HRegion. 
78    //////////////////////////////////////////////////////////////////////////////
79  
80    /**
81     * Test one put command.
82     */
83    public void testPut() throws IOException {
84      LOG.info("Starting testPut");
85      initHRegion(tableName, getName(), fam1);
86  
87      long value = 1L;
88  
89      Put put = new Put(row);
90      put.add(fam1, qual1, Bytes.toBytes(value));
91      region.put(put);
92  
93      assertGet(row, fam1, qual1, Bytes.toBytes(value));
94    }
95  
96    /**
97     * Test multi-threaded Puts.
98     */
99    public void testParallelPuts() throws IOException {
100 
101     LOG.info("Starting testParallelPuts");
102     initHRegion(tableName, getName(), fam1);
103     int numOps = 1000; // these many operations per thread
104 
105     // create 100 threads, each will do its own puts
106     int numThreads = 100;
107     Putter[] all = new Putter[numThreads];
108 
109     // create all threads
110     for (int i = 0; i < numThreads; i++) {
111       all[i] = new Putter(region, i, numOps);
112     }
113 
114     // run all threads
115     for (int i = 0; i < numThreads; i++) {
116       all[i].start();
117     }
118 
119     // wait for all threads to finish
120     for (int i = 0; i < numThreads; i++) {
121       try {
122         all[i].join();
123       } catch (InterruptedException e) {
124         LOG.warn("testParallelPuts encountered InterruptedException." +
125                  " Ignoring....", e);
126       }
127     }
128     LOG.info("testParallelPuts successfully verified " + 
129              (numOps * numThreads) + " put operations.");
130   }
131 
132 
133   static private void assertGet(byte [] row,
134                          byte [] familiy,
135                          byte[] qualifier,
136                          byte[] value) throws IOException {
137     // run a get and see if the value matches
138     Get get = new Get(row);
139     get.addColumn(familiy, qualifier);
140     Result result = region.get(get);
141     assertEquals(1, result.size());
142 
143     KeyValue kv = result.raw()[0];
144     byte[] r = kv.getValue();
145     assertTrue(Bytes.compareTo(r, value) == 0);
146   }
147 
148   private void initHRegion(byte [] tableName, String callingMethod,
149     byte[] ... families)
150   throws IOException {
151     initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
152   }
153 
154   private void initHRegion(byte [] tableName, String callingMethod,
155     Configuration conf, byte [] ... families)
156   throws IOException{
157     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(tableName));
158     for(byte [] family : families) {
159       htd.addFamily(new HColumnDescriptor(family));
160     }
161     HRegionInfo info = new HRegionInfo(htd.getTableName(), null, null, false);
162     Path path = new Path(DIR + callingMethod);
163     if (fs.exists(path)) {
164       if (!fs.delete(path, true)) {
165         throw new IOException("Failed delete of " + path);
166       }
167     }
168     region = HRegion.createHRegion(info, path, conf, htd);
169   }
170 
171   /**
172    * A thread that makes a few put calls
173    */
174   public static class Putter extends Thread {
175 
176     private final HRegion region;
177     private final int threadNumber;
178     private final int numOps;
179     private final Random rand = new Random();
180     byte [] rowkey = null;
181 
182     public Putter(HRegion region, int threadNumber, int numOps) {
183       this.region = region;
184       this.threadNumber = threadNumber;
185       this.numOps = numOps;
186       this.rowkey = Bytes.toBytes((long)threadNumber); // unique rowid per thread
187       setDaemon(true);
188     }
189 
190     @Override
191     public void run() {
192       byte[] value = new byte[100];
193       Put[]  in = new Put[1];
194 
195       // iterate for the specified number of operations
196       for (int i=0; i<numOps; i++) {
197         // generate random bytes
198         rand.nextBytes(value);  
199 
200         // put the randombytes and verify that we can read it. This is one
201         // way of ensuring that rwcc manipulation in HRegion.put() is fine.
202         Put put = new Put(rowkey);
203         put.add(fam1, qual1, value);
204         in[0] = put;
205         try {
206           OperationStatus[] ret = region.batchMutate(in);
207           assertEquals(1, ret.length);
208           assertEquals(OperationStatusCode.SUCCESS, ret[0].getOperationStatusCode());
209           assertGet(rowkey, fam1, qual1, value);
210         } catch (IOException e) {
211           assertTrue("Thread id " + threadNumber + " operation " + i + " failed.",
212                      false);
213         }
214       }
215     }
216   }
217 
218 }
219