1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase.thrift;
21
22 import static org.junit.Assert.assertEquals;
23 import static org.junit.Assert.assertFalse;
24 import static org.junit.Assert.assertTrue;
25
26 import java.io.IOException;
27 import java.nio.ByteBuffer;
28 import java.util.ArrayList;
29 import java.util.List;
30 import java.util.Map;
31
32 import org.apache.commons.logging.Log;
33 import org.apache.commons.logging.LogFactory;
34 import org.apache.hadoop.conf.Configuration;
35 import org.apache.hadoop.hbase.HBaseTestingUtility;
36 import org.apache.hadoop.hbase.MediumTests;
37 import org.apache.hadoop.hbase.filter.ParseFilter;
38 import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
39 import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
40 import org.apache.hadoop.hbase.thrift.generated.Hbase;
41 import org.apache.hadoop.hbase.thrift.generated.Mutation;
42 import org.apache.hadoop.hbase.thrift.generated.TCell;
43 import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
44 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
45 import org.apache.hadoop.hbase.thrift.generated.TIncrement;
46 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
47 import org.apache.hadoop.hbase.thrift.generated.TScan;
48 import org.apache.hadoop.hbase.util.Bytes;
49 import org.apache.hadoop.metrics.ContextFactory;
50 import org.apache.hadoop.metrics.MetricsContext;
51 import org.apache.hadoop.metrics.MetricsUtil;
52 import org.apache.hadoop.metrics.spi.NoEmitMetricsContext;
53 import org.apache.hadoop.metrics.spi.OutputRecord;
54 import org.junit.AfterClass;
55 import org.junit.BeforeClass;
56 import org.junit.Test;
57 import org.junit.experimental.categories.Category;
58
59
60
61
62
63 @Category(MediumTests.class)
64 public class TestThriftServer {
65 private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
66 private static final Log LOG = LogFactory.getLog(TestThriftServer.class);
67 protected static final int MAXVERSIONS = 3;
68
69 private static ByteBuffer asByteBuffer(String i) {
70 return ByteBuffer.wrap(Bytes.toBytes(i));
71 }
72 private static ByteBuffer asByteBuffer(long l) {
73 return ByteBuffer.wrap(Bytes.toBytes(l));
74 }
75
76
77 private static ByteBuffer tableAname = asByteBuffer("tableA");
78 private static ByteBuffer tableBname = asByteBuffer("tableB");
79 private static ByteBuffer columnAname = asByteBuffer("columnA:");
80 private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
81 private static ByteBuffer columnBname = asByteBuffer("columnB:");
82 private static ByteBuffer rowAname = asByteBuffer("rowA");
83 private static ByteBuffer rowBname = asByteBuffer("rowB");
84 private static ByteBuffer valueAname = asByteBuffer("valueA");
85 private static ByteBuffer valueBname = asByteBuffer("valueB");
86 private static ByteBuffer valueCname = asByteBuffer("valueC");
87 private static ByteBuffer valueDname = asByteBuffer("valueD");
88 private static ByteBuffer valueEname = asByteBuffer(100l);
89
90 @BeforeClass
91 public static void beforeClass() throws Exception {
92 UTIL.getConfiguration().setBoolean(ThriftServerRunner.COALESCE_INC_KEY, true);
93 UTIL.startMiniCluster();
94 }
95
96 @AfterClass
97 public static void afterClass() throws Exception {
98 UTIL.shutdownMiniCluster();
99 }
100
101
102
103
104
105
106
107
108
109 @Test
110 public void testAll() throws Exception {
111
112 doTestTableCreateDrop();
113 doTestThriftMetrics();
114 doTestTableMutations();
115 doTestTableTimestampsAndColumns();
116 doTestTableScanners();
117 doTestGetTableRegions();
118 doTestFilterRegistration();
119 }
120
121
122
123
124
125
126
127
128 public void doTestTableCreateDrop() throws Exception {
129 ThriftServerRunner.HBaseHandler handler =
130 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
131 doTestTableCreateDrop(handler);
132 }
133 public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
134 createTestTables(handler);
135 dropTestTables(handler);
136 }
137
138
139
140
141 public void doTestThriftMetrics() throws Exception {
142 Configuration conf = UTIL.getConfiguration();
143 ThriftMetrics metrics = getMetrics(conf);
144 Hbase.Iface handler = getHandler(metrics, conf);
145 createTestTables(handler);
146 dropTestTables(handler);
147 verifyMetrics(metrics, "createTable_num_ops", 2);
148 verifyMetrics(metrics, "deleteTable_num_ops", 2);
149 verifyMetrics(metrics, "disableTable_num_ops", 2);
150 }
151
152 private static Hbase.Iface getHandler(ThriftMetrics metrics, Configuration conf)
153 throws Exception {
154 Hbase.Iface handler =
155 new ThriftServerRunner.HBaseHandler(conf);
156 return HbaseHandlerMetricsProxy.newInstance(handler, metrics, conf);
157 }
158
159 private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
160 setupMetricsContext();
161 return new ThriftMetrics(ThriftServerRunner.DEFAULT_LISTEN_PORT, conf, Hbase.Iface.class);
162 }
163
164 private static void setupMetricsContext() throws IOException {
165 ContextFactory factory = ContextFactory.getFactory();
166 factory.setAttribute(ThriftMetrics.CONTEXT_NAME + ".class",
167 NoEmitMetricsContext.class.getName());
168 MetricsUtil.getContext(ThriftMetrics.CONTEXT_NAME)
169 .createRecord(ThriftMetrics.CONTEXT_NAME).remove();
170 }
171
172 private static void verifyMetrics(ThriftMetrics metrics, String name, int expectValue)
173 throws Exception {
174 MetricsContext context = MetricsUtil.getContext(
175 ThriftMetrics.CONTEXT_NAME);
176 metrics.doUpdates(context);
177 OutputRecord record = context.getAllRecords().get(
178 ThriftMetrics.CONTEXT_NAME).iterator().next();
179 assertEquals(expectValue, record.getMetric(name).intValue());
180 }
181
182 public static void createTestTables(Hbase.Iface handler) throws Exception {
183
184 assertEquals(handler.getTableNames().size(), 0);
185 handler.createTable(tableAname, getColumnDescriptors());
186 assertEquals(handler.getTableNames().size(), 1);
187 assertEquals(handler.getColumnDescriptors(tableAname).size(), 2);
188 assertTrue(handler.isTableEnabled(tableAname));
189 handler.createTable(tableBname, new ArrayList<ColumnDescriptor>());
190 assertEquals(handler.getTableNames().size(), 2);
191 }
192
193 public static void dropTestTables(Hbase.Iface handler) throws Exception {
194 handler.disableTable(tableBname);
195 assertFalse(handler.isTableEnabled(tableBname));
196 handler.deleteTable(tableBname);
197 assertEquals(handler.getTableNames().size(), 1);
198 handler.disableTable(tableAname);
199
200
201
202
203
204 handler.deleteTable(tableAname);
205 }
206
207 public void doTestIncrements() throws Exception {
208 ThriftServerRunner.HBaseHandler handler =
209 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
210 createTestTables(handler);
211 doTestIncrements(handler);
212 dropTestTables(handler);
213 }
214
215 public static void doTestIncrements(HBaseHandler handler) throws Exception {
216 List<Mutation> mutations = new ArrayList<Mutation>(1);
217 mutations.add(new Mutation(false, columnAAname, valueEname, true));
218 mutations.add(new Mutation(false, columnAname, valueEname, true));
219 handler.mutateRow(tableAname, rowAname, mutations, null);
220 handler.mutateRow(tableAname, rowBname, mutations, null);
221
222 List<TIncrement> increments = new ArrayList<TIncrement>();
223 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
224 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
225 increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
226
227 int numIncrements = 60000;
228 for (int i = 0; i < numIncrements; i++) {
229 handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
230 handler.incrementRows(increments);
231 }
232
233 Thread.sleep(1000);
234 long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
235 assertEquals((100 + (2 * numIncrements)), lv );
236
237
238 lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
239 assertEquals((100 + (3 * 7 * numIncrements)), lv);
240
241 assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
242
243 }
244
245
246
247
248
249
250
251
252 public void doTestTableMutations() throws Exception {
253 ThriftServerRunner.HBaseHandler handler =
254 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
255 doTestTableMutations(handler);
256 }
257
258 public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
259
260 handler.createTable(tableAname, getColumnDescriptors());
261
262
263
264
265 handler.mutateRow(tableAname, rowAname, getMutations(), null);
266
267
268 assertEquals(valueAname,
269 handler.get(tableAname, rowAname, columnAname, null).get(0).value);
270 TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0);
271 assertEquals(rowAname, rowResult1.row);
272 assertEquals(valueBname,
273 rowResult1.columns.get(columnBname).value);
274
275
276
277
278
279
280
281
282
283 handler.mutateRows(tableAname, getBatchMutations(), null);
284
285
286 List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null);
287 assertFalse(cells.size() > 0);
288 assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value);
289 List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null);
290 assertEquals(valueCname, versions.get(0).value);
291 assertEquals(valueBname, versions.get(1).value);
292
293
294 TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
295 assertEquals(rowBname, rowResult2.row);
296 assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
297 assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
298
299
300 handler.deleteAll(tableAname, rowAname, columnBname, null);
301 handler.deleteAllRow(tableAname, rowBname, null);
302
303
304 int size = handler.get(tableAname, rowAname, columnBname, null).size();
305 assertEquals(0, size);
306 size = handler.getRow(tableAname, rowBname, null).size();
307 assertEquals(0, size);
308
309
310 List<Mutation> mutations = new ArrayList<Mutation>();
311 mutations.add(new Mutation(false, columnAname, null, true));
312 handler.mutateRow(tableAname, rowAname, mutations, null);
313 TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
314 assertEquals(rowAname, rowResult3.row);
315 assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
316
317
318 handler.disableTable(tableAname);
319 handler.deleteTable(tableAname);
320 }
321
322
323
324
325
326
327
328
329 public void doTestTableTimestampsAndColumns() throws Exception {
330
331 ThriftServerRunner.HBaseHandler handler =
332 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
333 handler.createTable(tableAname, getColumnDescriptors());
334
335
336 long time1 = System.currentTimeMillis();
337 handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
338
339 Thread.sleep(1000);
340
341
342 long time2 = System.currentTimeMillis();
343 handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
344
345
346 handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null);
347
348
349 time1 += 1;
350 time2 += 2;
351
352
353 assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
354 MAXVERSIONS, null).size());
355 assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
356 MAXVERSIONS, null).size());
357
358 TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0);
359 TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0);
360
361
362 assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
363 assertEquals(rowResult2.columns.get(columnBname).value, valueCname);
364
365
366 assertFalse(rowResult2.columns.containsKey(columnAname));
367
368 List<ByteBuffer> columns = new ArrayList<ByteBuffer>();
369 columns.add(columnBname);
370
371 rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0);
372 assertEquals(rowResult1.columns.get(columnBname).value, valueCname);
373 assertFalse(rowResult1.columns.containsKey(columnAname));
374
375 rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0);
376 assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
377 assertFalse(rowResult1.columns.containsKey(columnAname));
378
379
380
381
382 handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null);
383 handler.deleteAllRowTs(tableAname, rowBname, time2, null);
384
385
386 int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size();
387 assertEquals(0, size);
388
389 size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size();
390 assertEquals(1, size);
391
392
393 assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname);
394
395 assertEquals(0, handler.getRow(tableAname, rowBname, null).size());
396
397
398 handler.disableTable(tableAname);
399 handler.deleteTable(tableAname);
400 }
401
402
403
404
405
406
407
408 public void doTestTableScanners() throws Exception {
409
410 ThriftServerRunner.HBaseHandler handler =
411 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
412 handler.createTable(tableAname, getColumnDescriptors());
413
414
415 long time1 = System.currentTimeMillis();
416 handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
417
418
419
420 Thread.sleep(1000);
421
422
423 long time2 = System.currentTimeMillis();
424 handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
425
426 time1 += 1;
427
428
429 int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null);
430 TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
431 assertEquals(rowResult1a.row, rowAname);
432
433
434
435 assertEquals(rowResult1a.columns.size(), 1);
436 assertEquals(rowResult1a.columns.get(columnBname).value, valueCname);
437
438 TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
439 assertEquals(rowResult1b.row, rowBname);
440 assertEquals(rowResult1b.columns.size(), 2);
441 assertEquals(rowResult1b.columns.get(columnAname).value, valueCname);
442 assertEquals(rowResult1b.columns.get(columnBname).value, valueDname);
443 closeScanner(scanner1, handler);
444
445
446 int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1, null);
447 TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
448 assertEquals(rowResult2a.columns.size(), 1);
449
450
451 assertEquals(rowResult2a.columns.get(columnBname).value, valueBname);
452 closeScanner(scanner2, handler);
453
454
455 int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
456 getColumnList(true, false), null);
457 closeScanner(scanner3, handler);
458
459
460 int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
461 getColumnList(false, true), time1, null);
462 TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
463 assertEquals(rowResult4a.columns.size(), 1);
464 assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
465
466
467 TScan scanNoSortColumns = new TScan();
468 scanNoSortColumns.setStartRow(rowAname);
469 scanNoSortColumns.setStopRow(rowBname);
470
471 int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null);
472 TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
473 assertEquals(rowResult5.columns.size(), 1);
474 assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
475
476 TScan scanSortColumns = new TScan();
477 scanSortColumns.setStartRow(rowAname);
478 scanSortColumns.setStopRow(rowBname);
479 scanSortColumns = scanSortColumns.setSortColumns(true);
480
481 int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null);
482 TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
483 assertEquals(rowResult6.sortedColumns.size(), 1);
484 assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
485
486 List<Mutation> rowBmutations = new ArrayList<Mutation>();
487 for (int i = 0; i < 20; i++) {
488 rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
489 }
490 ByteBuffer rowC = asByteBuffer("rowC");
491 handler.mutateRow(tableAname, rowC, rowBmutations, null);
492
493 TScan scanSortMultiColumns = new TScan();
494 scanSortMultiColumns.setStartRow(rowC);
495 scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
496 int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
497 TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
498
499 ByteBuffer smallerColumn = asByteBuffer("columnA:");
500 for (int i = 0; i < 20; i++) {
501 ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
502 assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
503 smallerColumn = currentColumn;
504 }
505
506
507 handler.disableTable(tableAname);
508 handler.deleteTable(tableAname);
509 }
510
511
512
513
514
515
516
517 public void doTestGetTableRegions() throws Exception {
518 ThriftServerRunner.HBaseHandler handler =
519 new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration());
520 doTestGetTableRegions(handler);
521 }
522
523 public static void doTestGetTableRegions(Hbase.Iface handler)
524 throws Exception {
525 assertEquals(handler.getTableNames().size(), 0);
526 handler.createTable(tableAname, getColumnDescriptors());
527 assertEquals(handler.getTableNames().size(), 1);
528 List<TRegionInfo> regions = handler.getTableRegions(tableAname);
529 int regionCount = regions.size();
530 assertEquals("empty table should have only 1 region, " +
531 "but found " + regionCount, regionCount, 1);
532 LOG.info("Region found:" + regions.get(0));
533 handler.disableTable(tableAname);
534 handler.deleteTable(tableAname);
535 regionCount = handler.getTableRegions(tableAname).size();
536 assertEquals("non-existing table should have 0 region, " +
537 "but found " + regionCount, regionCount, 0);
538 }
539
540 public void doTestFilterRegistration() throws Exception {
541 Configuration conf = UTIL.getConfiguration();
542
543 conf.set("hbase.thrift.filters", "MyFilter:filterclass");
544
545 ThriftServerRunner.registerFilters(conf);
546
547 Map<String, String> registeredFilters = ParseFilter.getAllFilters();
548
549 assertEquals("filterclass", registeredFilters.get("MyFilter"));
550 }
551
552
553
554
555
556
557 private static List<ColumnDescriptor> getColumnDescriptors() {
558 ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<ColumnDescriptor>();
559
560
561 ColumnDescriptor cDescA = new ColumnDescriptor();
562 cDescA.name = columnAname;
563 cDescriptors.add(cDescA);
564
565
566 ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
567 false, "NONE", 0, 0, false, -1);
568 cDescriptors.add(cDescB);
569
570 return cDescriptors;
571 }
572
573
574
575
576
577
578
579 private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
580 List<ByteBuffer> columnList = new ArrayList<ByteBuffer>();
581 if (includeA) columnList.add(columnAname);
582 if (includeB) columnList.add(columnBname);
583 return columnList;
584 }
585
586
587
588
589
590
591 private static List<Mutation> getMutations() {
592 List<Mutation> mutations = new ArrayList<Mutation>();
593 mutations.add(new Mutation(false, columnAname, valueAname, true));
594 mutations.add(new Mutation(false, columnBname, valueBname, true));
595 return mutations;
596 }
597
598
599
600
601
602
603
604
605
606 private static List<BatchMutation> getBatchMutations() {
607 List<BatchMutation> batchMutations = new ArrayList<BatchMutation>();
608
609
610 List<Mutation> rowAmutations = new ArrayList<Mutation>();
611 rowAmutations.add(new Mutation(true, columnAname, null, true));
612 batchMutations.add(new BatchMutation(rowAname, rowAmutations));
613
614 rowAmutations = new ArrayList<Mutation>();
615 rowAmutations.add(new Mutation(false, columnBname, valueCname, true));
616 batchMutations.add(new BatchMutation(rowAname, rowAmutations));
617
618
619 List<Mutation> rowBmutations = new ArrayList<Mutation>();
620 rowBmutations.add(new Mutation(false, columnAname, valueCname, true));
621 rowBmutations.add(new Mutation(false, columnBname, valueDname, true));
622 batchMutations.add(new BatchMutation(rowBname, rowBmutations));
623
624 return batchMutations;
625 }
626
627
628
629
630
631
632
633
634
635 private void closeScanner(
636 int scannerId, ThriftServerRunner.HBaseHandler handler) throws Exception {
637 handler.scannerGet(scannerId);
638 handler.scannerClose(scannerId);
639 }
640
641 @org.junit.Rule
642 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
643 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
644 }
645