1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.regionserver;
19
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.List;
25 import java.util.Random;
26 import java.util.concurrent.atomic.AtomicInteger;
27 import java.util.concurrent.atomic.AtomicLong;
28
29 import org.apache.commons.logging.Log;
30 import org.apache.commons.logging.LogFactory;
31 import org.apache.hadoop.conf.Configuration;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.*;
34 import org.apache.hadoop.hbase.client.Append;
35 import org.apache.hadoop.hbase.client.Mutation;
36 import org.apache.hadoop.hbase.client.RowMutations;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.Scan;
42 import org.apache.hadoop.hbase.util.Bytes;
43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManagerTestHelper;
44 import org.junit.experimental.categories.Category;
45
46
47
48
49
50
51 @Category(MediumTests.class)
52 public class TestAtomicOperation extends HBaseTestCase {
53 static final Log LOG = LogFactory.getLog(TestAtomicOperation.class);
54
55 HRegion region = null;
56 private HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
57 private final String DIR = TEST_UTIL.getDataTestDir("TestAtomicOperation").toString();
58
59
60
61 static final byte[] tableName = Bytes.toBytes("testtable");;
62 static final byte[] qual1 = Bytes.toBytes("qual1");
63 static final byte[] qual2 = Bytes.toBytes("qual2");
64 static final byte[] qual3 = Bytes.toBytes("qual3");
65 static final byte[] value1 = Bytes.toBytes("value1");
66 static final byte[] value2 = Bytes.toBytes("value2");
67 static final byte [] row = Bytes.toBytes("rowA");
68 static final byte [] row2 = Bytes.toBytes("rowB");
69
70
71
72
73 @Override
74 protected void setUp() throws Exception {
75 super.setUp();
76 }
77
78 @Override
79 protected void tearDown() throws Exception {
80 super.tearDown();
81 EnvironmentEdgeManagerTestHelper.reset();
82 }
83
84
85
86
87
88
89
90
91
92
93
94 public void testAppend() throws IOException {
95 initHRegion(tableName, getName(), fam1);
96 String v1 = "Ultimate Answer to the Ultimate Question of Life,"+
97 " The Universe, and Everything";
98 String v2 = " is... 42.";
99 Append a = new Append(row);
100 a.setReturnResults(false);
101 a.add(fam1, qual1, Bytes.toBytes(v1));
102 a.add(fam1, qual2, Bytes.toBytes(v2));
103 assertNull(region.append(a, null, true));
104 a = new Append(row);
105 a.add(fam1, qual1, Bytes.toBytes(v2));
106 a.add(fam1, qual2, Bytes.toBytes(v1));
107 Result result = region.append(a, null, true);
108 assertEquals(0, Bytes.compareTo(Bytes.toBytes(v1+v2), result.getValue(fam1, qual1)));
109 assertEquals(0, Bytes.compareTo(Bytes.toBytes(v2+v1), result.getValue(fam1, qual2)));
110 }
111
112
113
114
115 public void testIncrementColumnValue() throws IOException {
116 LOG.info("Starting test testIncrementColumnValue");
117 initHRegion(tableName, getName(), fam1);
118
119 long value = 1L;
120 long amount = 3L;
121
122 Put put = new Put(row);
123 put.add(fam1, qual1, Bytes.toBytes(value));
124 region.put(put);
125
126 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
127
128 assertEquals(value+amount, result);
129
130 Store store = region.getStore(fam1);
131
132 assertEquals(1, store.memstore.kvset.size());
133 assertTrue(store.memstore.snapshot.isEmpty());
134
135 assertICV(row, fam1, qual1, value+amount);
136 }
137
138
139
140
141 public void testIncrementMultiThreads() throws IOException {
142
143 LOG.info("Starting test testIncrementMultiThreads");
144 initHRegion(tableName, getName(), fam1);
145
146
147 int numThreads = 100;
148 int incrementsPerThread = 1000;
149 Incrementer[] all = new Incrementer[numThreads];
150 int expectedTotal = 0;
151
152
153 for (int i = 0; i < numThreads; i++) {
154 all[i] = new Incrementer(region, i, i, incrementsPerThread);
155 expectedTotal += (i * incrementsPerThread);
156 }
157
158
159 for (int i = 0; i < numThreads; i++) {
160 all[i].start();
161 }
162
163
164 for (int i = 0; i < numThreads; i++) {
165 try {
166 all[i].join();
167 } catch (InterruptedException e) {
168 }
169 }
170 assertICV(row, fam1, qual1, expectedTotal);
171 LOG.info("testIncrementMultiThreads successfully verified that total is " +
172 expectedTotal);
173 }
174
175
176 private void assertICV(byte [] row,
177 byte [] familiy,
178 byte[] qualifier,
179 long amount) throws IOException {
180
181 Get get = new Get(row);
182 get.addColumn(familiy, qualifier);
183 Result result = region.get(get, null);
184 assertEquals(1, result.size());
185
186 KeyValue kv = result.raw()[0];
187 long r = Bytes.toLong(kv.getValue());
188 assertEquals(amount, r);
189 }
190
191 private void initHRegion (byte [] tableName, String callingMethod,
192 byte[] ... families)
193 throws IOException {
194 initHRegion(tableName, callingMethod, HBaseConfiguration.create(), families);
195 }
196
197 private void initHRegion (byte [] tableName, String callingMethod,
198 Configuration conf, byte [] ... families)
199 throws IOException{
200 HTableDescriptor htd = new HTableDescriptor(tableName);
201 for(byte [] family : families) {
202 htd.addFamily(new HColumnDescriptor(family));
203 }
204 HRegionInfo info = new HRegionInfo(htd.getName(), null, null, false);
205 Path path = new Path(DIR + callingMethod);
206 if (fs.exists(path)) {
207 if (!fs.delete(path, true)) {
208 throw new IOException("Failed delete of " + path);
209 }
210 }
211 region = HRegion.createHRegion(info, path, conf, htd);
212 }
213
214
215
216
217 public static class Incrementer extends Thread {
218
219 private final HRegion region;
220 private final int threadNumber;
221 private final int numIncrements;
222 private final int amount;
223
224 private int count;
225
226 public Incrementer(HRegion region,
227 int threadNumber, int amount, int numIncrements) {
228 this.region = region;
229 this.threadNumber = threadNumber;
230 this.numIncrements = numIncrements;
231 this.count = 0;
232 this.amount = amount;
233 setDaemon(true);
234 }
235
236 @Override
237 public void run() {
238 for (int i=0; i<numIncrements; i++) {
239 try {
240 long result = region.incrementColumnValue(row, fam1, qual1, amount, true);
241
242 } catch (IOException e) {
243 e.printStackTrace();
244 }
245 count++;
246 }
247 }
248 }
249
250
251
252
253 public void testRowMutationMultiThreads() throws IOException {
254
255 LOG.info("Starting test testRowMutationMultiThreads");
256 initHRegion(tableName, getName(), fam1);
257
258
259
260 int numThreads = 10;
261 int opsPerThread = 500;
262 AtomicOperation[] all = new AtomicOperation[numThreads];
263
264 AtomicLong timeStamps = new AtomicLong(0);
265 AtomicInteger failures = new AtomicInteger(0);
266
267 for (int i = 0; i < numThreads; i++) {
268 all[i] = new AtomicOperation(region, opsPerThread, timeStamps, failures) {
269 @Override
270 public void run() {
271 boolean op = true;
272 for (int i=0; i<numOps; i++) {
273 try {
274
275 if (i%10==0) {
276 synchronized(region) {
277 LOG.debug("flushing");
278 region.flushcache();
279 if (i%100==0) {
280 region.compactStores();
281 }
282 }
283 }
284 long ts = timeStamps.incrementAndGet();
285 RowMutations rm = new RowMutations(row);
286 if (op) {
287 Put p = new Put(row, ts);
288 p.add(fam1, qual1, value1);
289 rm.add(p);
290 Delete d = new Delete(row);
291 d.deleteColumns(fam1, qual2, ts);
292 rm.add(d);
293 } else {
294 Delete d = new Delete(row);
295 d.deleteColumns(fam1, qual1, ts);
296 rm.add(d);
297 Put p = new Put(row, ts);
298 p.add(fam1, qual2, value2);
299 rm.add(p);
300 }
301 region.mutateRow(rm);
302 op ^= true;
303
304 Get g = new Get(row);
305 Result r = region.get(g, null);
306 if (r.size() != 1) {
307 LOG.debug(r);
308 failures.incrementAndGet();
309 fail();
310 }
311 } catch (IOException e) {
312 e.printStackTrace();
313 failures.incrementAndGet();
314 fail();
315 }
316 }
317 }
318 };
319 }
320
321
322 for (int i = 0; i < numThreads; i++) {
323 all[i].start();
324 }
325
326
327 for (int i = 0; i < numThreads; i++) {
328 try {
329 all[i].join();
330 } catch (InterruptedException e) {
331 }
332 }
333 assertEquals(0, failures.get());
334 }
335
336
337
338
339
340 public void testMultiRowMutationMultiThreads() throws IOException {
341
342 LOG.info("Starting test testMultiRowMutationMultiThreads");
343 initHRegion(tableName, getName(), fam1);
344
345
346
347 int numThreads = 10;
348 int opsPerThread = 500;
349 AtomicOperation[] all = new AtomicOperation[numThreads];
350
351 AtomicLong timeStamps = new AtomicLong(0);
352 AtomicInteger failures = new AtomicInteger(0);
353 final List<byte[]> rowsToLock = Arrays.asList(row, row2);
354
355 for (int i = 0; i < numThreads; i++) {
356 all[i] = new AtomicOperation(region, opsPerThread, timeStamps, failures) {
357 @Override
358 public void run() {
359 boolean op = true;
360 for (int i=0; i<numOps; i++) {
361 try {
362
363 if (i%10==0) {
364 synchronized(region) {
365 LOG.debug("flushing");
366 region.flushcache();
367 if (i%100==0) {
368 region.compactStores();
369 }
370 }
371 }
372 long ts = timeStamps.incrementAndGet();
373 List<Mutation> mrm = new ArrayList<Mutation>();
374 if (op) {
375 Put p = new Put(row2, ts);
376 p.add(fam1, qual1, value1);
377 mrm.add(p);
378 Delete d = new Delete(row);
379 d.deleteColumns(fam1, qual1, ts);
380 mrm.add(d);
381 } else {
382 Delete d = new Delete(row2);
383 d.deleteColumns(fam1, qual1, ts);
384 mrm.add(d);
385 Put p = new Put(row, ts);
386 p.add(fam1, qual1, value2);
387 mrm.add(p);
388 }
389 region.mutateRowsWithLocks(mrm, rowsToLock);
390 op ^= true;
391
392 Scan s = new Scan(row);
393 RegionScanner rs = region.getScanner(s);
394 List<KeyValue> r = new ArrayList<KeyValue>();
395 while(rs.next(r));
396 rs.close();
397 if (r.size() != 1) {
398 LOG.debug(r);
399 failures.incrementAndGet();
400 fail();
401 }
402 } catch (IOException e) {
403 e.printStackTrace();
404 failures.incrementAndGet();
405 fail();
406 }
407 }
408 }
409 };
410 }
411
412
413 for (int i = 0; i < numThreads; i++) {
414 all[i].start();
415 }
416
417
418 for (int i = 0; i < numThreads; i++) {
419 try {
420 all[i].join();
421 } catch (InterruptedException e) {
422 }
423 }
424 assertEquals(0, failures.get());
425 }
426
427 public static class AtomicOperation extends Thread {
428 protected final HRegion region;
429 protected final int numOps;
430 protected final AtomicLong timeStamps;
431 protected final AtomicInteger failures;
432 protected final Random r = new Random();
433
434 public AtomicOperation(HRegion region, int numOps, AtomicLong timeStamps,
435 AtomicInteger failures) {
436 this.region = region;
437 this.numOps = numOps;
438 this.timeStamps = timeStamps;
439 this.failures = failures;
440 }
441 }
442
443 @org.junit.Rule
444 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
445 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
446 }
447