1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.avro;
20
21 import java.io.IOException;
22 import java.nio.ByteBuffer;
23 import java.util.Collection;
24 import java.util.HashMap;
25 import java.util.Iterator;
26 import java.util.List;
27
28 import org.apache.avro.Schema;
29 import org.apache.avro.generic.GenericArray;
30 import org.apache.avro.generic.GenericData;
31 import org.apache.avro.ipc.HttpServer;
32 import org.apache.avro.specific.SpecificResponder;
33 import org.apache.avro.util.Utf8;
34 import org.apache.commons.logging.Log;
35 import org.apache.commons.logging.LogFactory;
36
37 import org.apache.hadoop.conf.Configuration;
38 import org.apache.hadoop.hbase.HBaseConfiguration;
39 import org.apache.hadoop.hbase.HColumnDescriptor;
40 import org.apache.hadoop.hbase.HTableDescriptor;
41 import org.apache.hadoop.hbase.MasterNotRunningException;
42 import org.apache.hadoop.hbase.TableExistsException;
43 import org.apache.hadoop.hbase.client.Delete;
44 import org.apache.hadoop.hbase.client.Get;
45 import org.apache.hadoop.hbase.client.HBaseAdmin;
46 import org.apache.hadoop.hbase.client.HTable;
47 import org.apache.hadoop.hbase.client.HTableInterface;
48 import org.apache.hadoop.hbase.client.HTablePool;
49 import org.apache.hadoop.hbase.client.Put;
50 import org.apache.hadoop.hbase.client.Result;
51 import org.apache.hadoop.hbase.client.ResultScanner;
52 import org.apache.hadoop.hbase.client.Scan;
53 import org.apache.hadoop.hbase.util.Bytes;
54
55 import org.apache.hadoop.hbase.avro.generated.AClusterStatus;
56 import org.apache.hadoop.hbase.avro.generated.AColumnValue;
57 import org.apache.hadoop.hbase.avro.generated.ACompressionAlgorithm;
58 import org.apache.hadoop.hbase.avro.generated.ADelete;
59 import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
60 import org.apache.hadoop.hbase.avro.generated.AGet;
61 import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
62 import org.apache.hadoop.hbase.avro.generated.AIOError;
63 import org.apache.hadoop.hbase.avro.generated.AMasterNotRunning;
64 import org.apache.hadoop.hbase.avro.generated.APut;
65 import org.apache.hadoop.hbase.avro.generated.AResult;
66 import org.apache.hadoop.hbase.avro.generated.AScan;
67 import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
68 import org.apache.hadoop.hbase.avro.generated.ATableExists;
69 import org.apache.hadoop.hbase.avro.generated.HBase;
70
71
72
73
74 public class AvroServer {
75
76
77
78
79
80 public static class HBaseImpl implements HBase {
81
82
83
84 protected Configuration conf = null;
85 protected HBaseAdmin admin = null;
86 protected HTablePool htablePool = null;
87 protected final Log LOG = LogFactory.getLog(this.getClass().getName());
88
89
90 protected int nextScannerId = 0;
91 protected HashMap<Integer, ResultScanner> scannerMap = null;
92
93
94
95
96
97
98
99
100
101
102
103
104 protected synchronized int addScanner(ResultScanner scanner) {
105 int id = nextScannerId++;
106 scannerMap.put(id, scanner);
107 return id;
108 }
109
110
111
112
113
114
115
116 protected synchronized ResultScanner getScanner(int id) {
117 return scannerMap.get(id);
118 }
119
120
121
122
123
124
125
126
127 protected synchronized ResultScanner removeScanner(int id) {
128 return scannerMap.remove(id);
129 }
130
131
132
133
134
135
136
137
138
139
140
141 HBaseImpl() throws MasterNotRunningException {
142 conf = HBaseConfiguration.create();
143 admin = new HBaseAdmin(conf);
144 htablePool = new HTablePool(conf, 10);
145 scannerMap = new HashMap<Integer, ResultScanner>();
146 }
147
148
149
150
151
152
153
154
155
156
157
158 public Utf8 getHBaseVersion() throws AIOError {
159 try {
160 return new Utf8(admin.getClusterStatus().getHBaseVersion());
161 } catch (IOException e) {
162 AIOError ioe = new AIOError();
163 ioe.message = new Utf8(e.getMessage());
164 throw ioe;
165 }
166 }
167
168 public AClusterStatus getClusterStatus() throws AIOError {
169 try {
170 return AvroUtil.csToACS(admin.getClusterStatus());
171 } catch (IOException e) {
172 AIOError ioe = new AIOError();
173 ioe.message = new Utf8(e.getMessage());
174 throw ioe;
175 }
176 }
177
178 public GenericArray<ATableDescriptor> listTables() throws AIOError {
179 try {
180 HTableDescriptor[] tables = admin.listTables();
181 Schema atdSchema = Schema.createArray(ATableDescriptor.SCHEMA$);
182 GenericData.Array<ATableDescriptor> result = null;
183 result = new GenericData.Array<ATableDescriptor>(tables.length, atdSchema);
184 for (HTableDescriptor table : tables) {
185 result.add(AvroUtil.htdToATD(table));
186 }
187 return result;
188 } catch (IOException e) {
189 AIOError ioe = new AIOError();
190 ioe.message = new Utf8(e.getMessage());
191 throw ioe;
192 }
193 }
194
195
196
197
198
199
200 public ATableDescriptor describeTable(ByteBuffer table) throws AIOError {
201 try {
202 return AvroUtil.htdToATD(admin.getTableDescriptor(Bytes.toBytes(table)));
203 } catch (IOException e) {
204 AIOError ioe = new AIOError();
205 ioe.message = new Utf8(e.getMessage());
206 throw ioe;
207 }
208 }
209
210 public boolean isTableEnabled(ByteBuffer table) throws AIOError {
211 try {
212 return admin.isTableEnabled(Bytes.toBytes(table));
213 } catch (IOException e) {
214 AIOError ioe = new AIOError();
215 ioe.message = new Utf8(e.getMessage());
216 throw ioe;
217 }
218 }
219
220 public boolean tableExists(ByteBuffer table) throws AIOError {
221 try {
222 return admin.tableExists(Bytes.toBytes(table));
223 } catch (IOException e) {
224 AIOError ioe = new AIOError();
225 ioe.message = new Utf8(e.getMessage());
226 throw ioe;
227 }
228 }
229
230
231
232
233
234
235 public AFamilyDescriptor describeFamily(ByteBuffer table, ByteBuffer family) throws AIOError {
236 try {
237 HTableDescriptor htd = admin.getTableDescriptor(Bytes.toBytes(table));
238 return AvroUtil.hcdToAFD(htd.getFamily(Bytes.toBytes(family)));
239 } catch (IOException e) {
240 AIOError ioe = new AIOError();
241 ioe.message = new Utf8(e.getMessage());
242 throw ioe;
243 }
244 }
245
246
247
248
249
250 public Void createTable(ATableDescriptor table) throws AIOError,
251 AIllegalArgument,
252 ATableExists,
253 AMasterNotRunning {
254 try {
255 admin.createTable(AvroUtil.atdToHTD(table));
256 return null;
257 } catch (IllegalArgumentException e) {
258 AIllegalArgument iae = new AIllegalArgument();
259 iae.message = new Utf8(e.getMessage());
260 throw iae;
261 } catch (TableExistsException e) {
262 ATableExists tee = new ATableExists();
263 tee.message = new Utf8(e.getMessage());
264 throw tee;
265 } catch (MasterNotRunningException e) {
266 AMasterNotRunning mnre = new AMasterNotRunning();
267 mnre.message = new Utf8(e.getMessage());
268 throw mnre;
269 } catch (IOException e) {
270 AIOError ioe = new AIOError();
271 ioe.message = new Utf8(e.getMessage());
272 throw ioe;
273 }
274 }
275
276
277
278 public Void deleteTable(ByteBuffer table) throws AIOError {
279 try {
280 admin.deleteTable(Bytes.toBytes(table));
281 return null;
282 } catch (IOException e) {
283 AIOError ioe = new AIOError();
284 ioe.message = new Utf8(e.getMessage());
285 throw ioe;
286 }
287 }
288
289
290 public Void modifyTable(ByteBuffer tableName, ATableDescriptor tableDescriptor) throws AIOError {
291 try {
292 admin.modifyTable(Bytes.toBytes(tableName),
293 AvroUtil.atdToHTD(tableDescriptor));
294 return null;
295 } catch (IOException e) {
296 AIOError ioe = new AIOError();
297 ioe.message = new Utf8(e.getMessage());
298 throw ioe;
299 }
300 }
301
302 public Void enableTable(ByteBuffer table) throws AIOError {
303 try {
304 admin.enableTable(Bytes.toBytes(table));
305 return null;
306 } catch (IOException e) {
307 AIOError ioe = new AIOError();
308 ioe.message = new Utf8(e.getMessage());
309 throw ioe;
310 }
311 }
312
313 public Void disableTable(ByteBuffer table) throws AIOError {
314 try {
315 admin.disableTable(Bytes.toBytes(table));
316 return null;
317 } catch (IOException e) {
318 AIOError ioe = new AIOError();
319 ioe.message = new Utf8(e.getMessage());
320 throw ioe;
321 }
322 }
323
324
325 public Void flush(ByteBuffer table) throws AIOError {
326 try {
327 admin.flush(Bytes.toBytes(table));
328 return null;
329 } catch (IOException e) {
330 AIOError ioe = new AIOError();
331 ioe.message = new Utf8(e.getMessage());
332 throw ioe;
333 }
334 }
335
336
337 public Void split(ByteBuffer table) throws AIOError {
338 try {
339 admin.split(Bytes.toBytes(table));
340 return null;
341 } catch (IOException e) {
342 AIOError ioe = new AIOError();
343 ioe.message = new Utf8(e.getMessage());
344 throw ioe;
345 }
346 }
347
348
349
350
351
352 public Void addFamily(ByteBuffer table, AFamilyDescriptor family) throws AIOError {
353 try {
354 admin.addColumn(Bytes.toBytes(table),
355 AvroUtil.afdToHCD(family));
356 return null;
357 } catch (IOException e) {
358 AIOError ioe = new AIOError();
359 ioe.message = new Utf8(e.getMessage());
360 throw ioe;
361 }
362 }
363
364
365 public Void deleteFamily(ByteBuffer table, ByteBuffer family) throws AIOError {
366 try {
367 admin.deleteColumn(Bytes.toBytes(table), Bytes.toBytes(family));
368 return null;
369 } catch (IOException e) {
370 AIOError ioe = new AIOError();
371 ioe.message = new Utf8(e.getMessage());
372 throw ioe;
373 }
374 }
375
376
377 public Void modifyFamily(ByteBuffer table, ByteBuffer familyName, AFamilyDescriptor familyDescriptor) throws AIOError {
378 try {
379 admin.modifyColumn(Bytes.toBytes(table), Bytes.toBytes(familyName),
380 AvroUtil.afdToHCD(familyDescriptor));
381 return null;
382 } catch (IOException e) {
383 AIOError ioe = new AIOError();
384 ioe.message = new Utf8(e.getMessage());
385 throw ioe;
386 }
387 }
388
389
390
391
392
393
394
395
396
397
398 public AResult get(ByteBuffer table, AGet aget) throws AIOError {
399 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
400 try {
401 return AvroUtil.resultToAResult(htable.get(AvroUtil.agetToGet(aget)));
402 } catch (IOException e) {
403 AIOError ioe = new AIOError();
404 ioe.message = new Utf8(e.getMessage());
405 throw ioe;
406 } finally {
407 htablePool.putTable(htable);
408 }
409 }
410
411 public boolean exists(ByteBuffer table, AGet aget) throws AIOError {
412 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
413 try {
414 return htable.exists(AvroUtil.agetToGet(aget));
415 } catch (IOException e) {
416 AIOError ioe = new AIOError();
417 ioe.message = new Utf8(e.getMessage());
418 throw ioe;
419 } finally {
420 htablePool.putTable(htable);
421 }
422 }
423
424 public Void put(ByteBuffer table, APut aput) throws AIOError {
425 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
426 try {
427 htable.put(AvroUtil.aputToPut(aput));
428 return null;
429 } catch (IOException e) {
430 AIOError ioe = new AIOError();
431 ioe.message = new Utf8(e.getMessage());
432 throw ioe;
433 } finally {
434 htablePool.putTable(htable);
435 }
436 }
437
438 public Void delete(ByteBuffer table, ADelete adelete) throws AIOError {
439 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
440 try {
441 htable.delete(AvroUtil.adeleteToDelete(adelete));
442 return null;
443 } catch (IOException e) {
444 AIOError ioe = new AIOError();
445 ioe.message = new Utf8(e.getMessage());
446 throw ioe;
447 } finally {
448 htablePool.putTable(htable);
449 }
450 }
451
452 public long incrementColumnValue(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, long amount, boolean writeToWAL) throws AIOError {
453 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
454 try {
455 return htable.incrementColumnValue(Bytes.toBytes(row), Bytes.toBytes(family), Bytes.toBytes(qualifier), amount, writeToWAL);
456 } catch (IOException e) {
457 AIOError ioe = new AIOError();
458 ioe.message = new Utf8(e.getMessage());
459 throw ioe;
460 } finally {
461 htablePool.putTable(htable);
462 }
463 }
464
465
466
467
468
469 public int scannerOpen(ByteBuffer table, AScan ascan) throws AIOError {
470 HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
471 try {
472 Scan scan = AvroUtil.ascanToScan(ascan);
473 return addScanner(htable.getScanner(scan));
474 } catch (IOException e) {
475 AIOError ioe = new AIOError();
476 ioe.message = new Utf8(e.getMessage());
477 throw ioe;
478 } finally {
479 htablePool.putTable(htable);
480 }
481 }
482
483 public Void scannerClose(int scannerId) throws AIOError, AIllegalArgument {
484 try {
485 ResultScanner scanner = getScanner(scannerId);
486 if (scanner == null) {
487 AIllegalArgument aie = new AIllegalArgument();
488 aie.message = new Utf8("scanner ID is invalid: " + scannerId);
489 throw aie;
490 }
491 scanner.close();
492 removeScanner(scannerId);
493 return null;
494 } catch (IOException e) {
495 AIOError ioe = new AIOError();
496 ioe.message = new Utf8(e.getMessage());
497 throw ioe;
498 }
499 }
500
501 public GenericArray<AResult> scannerGetRows(int scannerId, int numberOfRows) throws AIOError, AIllegalArgument {
502 try {
503 ResultScanner scanner = getScanner(scannerId);
504 if (scanner == null) {
505 AIllegalArgument aie = new AIllegalArgument();
506 aie.message = new Utf8("scanner ID is invalid: " + scannerId);
507 throw aie;
508 }
509 Result[] results = null;
510 return AvroUtil.resultsToAResults(scanner.next(numberOfRows));
511 } catch (IOException e) {
512 AIOError ioe = new AIOError();
513 ioe.message = new Utf8(e.getMessage());
514 throw ioe;
515 }
516 }
517 }
518
519
520
521
522
523 private static void printUsageAndExit() {
524 printUsageAndExit(null);
525 }
526
527 private static void printUsageAndExit(final String message) {
528 if (message != null) {
529 System.err.println(message);
530 }
531 System.out.println("Usage: java org.apache.hadoop.hbase.avro.AvroServer " +
532 "--help | [--port=PORT] start");
533 System.out.println("Arguments:");
534 System.out.println(" start Start Avro server");
535 System.out.println(" stop Stop Avro server");
536 System.out.println("Options:");
537 System.out.println(" port Port to listen on. Default: 9090");
538 System.out.println(" help Print this message and exit");
539 System.exit(0);
540 }
541
542
543 protected static void doMain(final String[] args) throws Exception {
544 if (args.length < 1) {
545 printUsageAndExit();
546 }
547 int port = 9090;
548 final String portArgKey = "--port=";
549 for (String cmd: args) {
550 if (cmd.startsWith(portArgKey)) {
551 port = Integer.parseInt(cmd.substring(portArgKey.length()));
552 continue;
553 } else if (cmd.equals("--help") || cmd.equals("-h")) {
554 printUsageAndExit();
555 } else if (cmd.equals("start")) {
556 continue;
557 } else if (cmd.equals("stop")) {
558 printUsageAndExit("To shutdown the Avro server run " +
559 "bin/hbase-daemon.sh stop avro or send a kill signal to " +
560 "the Avro server pid");
561 }
562
563
564 printUsageAndExit();
565 }
566 Log LOG = LogFactory.getLog("AvroServer");
567 LOG.info("starting HBase Avro server on port " + Integer.toString(port));
568 SpecificResponder r = new SpecificResponder(HBase.class, new HBaseImpl());
569 HttpServer server = new HttpServer(r, 9090);
570 Thread.sleep(1000000);
571 }
572
573
574
575
576
577 public static void main(String[] args) throws Exception {
578 doMain(args);
579 }
580 }