View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.avro;
20  
21  import java.io.IOException;
22  import java.nio.ByteBuffer;
23  import java.util.Collection;
24  import java.util.HashMap;
25  import java.util.Iterator;
26  import java.util.List;
27  
28  import org.apache.avro.Schema;
29  import org.apache.avro.generic.GenericArray;
30  import org.apache.avro.generic.GenericData;
31  import org.apache.avro.ipc.HttpServer;
32  import org.apache.avro.specific.SpecificResponder;
33  import org.apache.avro.util.Utf8;
34  import org.apache.commons.logging.Log;
35  import org.apache.commons.logging.LogFactory;
36  
37  import org.apache.hadoop.conf.Configuration;
38  import org.apache.hadoop.hbase.HBaseConfiguration;
39  import org.apache.hadoop.hbase.HColumnDescriptor;
40  import org.apache.hadoop.hbase.HTableDescriptor;
41  import org.apache.hadoop.hbase.MasterNotRunningException;
42  import org.apache.hadoop.hbase.TableExistsException;
43  import org.apache.hadoop.hbase.client.Delete;
44  import org.apache.hadoop.hbase.client.Get;
45  import org.apache.hadoop.hbase.client.HBaseAdmin;
46  import org.apache.hadoop.hbase.client.HTable;
47  import org.apache.hadoop.hbase.client.HTableInterface;
48  import org.apache.hadoop.hbase.client.HTablePool;
49  import org.apache.hadoop.hbase.client.Put;
50  import org.apache.hadoop.hbase.client.Result;
51  import org.apache.hadoop.hbase.client.ResultScanner;
52  import org.apache.hadoop.hbase.client.Scan;
53  import org.apache.hadoop.hbase.util.Bytes;
54  
55  import org.apache.hadoop.hbase.avro.generated.AClusterStatus;
56  import org.apache.hadoop.hbase.avro.generated.AColumnValue;
57  import org.apache.hadoop.hbase.avro.generated.ACompressionAlgorithm;
58  import org.apache.hadoop.hbase.avro.generated.ADelete;
59  import org.apache.hadoop.hbase.avro.generated.AFamilyDescriptor;
60  import org.apache.hadoop.hbase.avro.generated.AGet;
61  import org.apache.hadoop.hbase.avro.generated.AIllegalArgument;
62  import org.apache.hadoop.hbase.avro.generated.AIOError;
63  import org.apache.hadoop.hbase.avro.generated.AMasterNotRunning;
64  import org.apache.hadoop.hbase.avro.generated.APut;
65  import org.apache.hadoop.hbase.avro.generated.AResult;
66  import org.apache.hadoop.hbase.avro.generated.AScan;
67  import org.apache.hadoop.hbase.avro.generated.ATableDescriptor;
68  import org.apache.hadoop.hbase.avro.generated.ATableExists;
69  import org.apache.hadoop.hbase.avro.generated.HBase;
70  
71  /**
72   * Start an Avro server
73   */
74  public class AvroServer {
75  
76    /**
77     * The HBaseImpl is a glue object that connects Avro RPC calls to the
78     * HBase client API primarily defined in the HBaseAdmin and HTable objects.
79     */
80    public static class HBaseImpl implements HBase {
81      //
82      // PROPERTIES
83      //
84      protected Configuration conf = null;
85      protected HBaseAdmin admin = null;
86      protected HTablePool htablePool = null;
87      protected final Log LOG = LogFactory.getLog(this.getClass().getName());
88  
89      // nextScannerId and scannerMap are used to manage scanner state
90      protected int nextScannerId = 0;
91      protected HashMap<Integer, ResultScanner> scannerMap = null;
92  
93      //
94      // UTILITY METHODS
95      //
96  
97      /**
98       * Assigns a unique ID to the scanner and adds the mapping to an internal
99       * hash-map.
100      *
101      * @param scanner
102      * @return integer scanner id
103      */
104     protected synchronized int addScanner(ResultScanner scanner) {
105       int id = nextScannerId++;
106       scannerMap.put(id, scanner);
107       return id;
108     }
109 
110     /**
111      * Returns the scanner associated with the specified ID.
112      *
113      * @param id
114      * @return a Scanner, or null if ID was invalid.
115      */
116     protected synchronized ResultScanner getScanner(int id) {
117       return scannerMap.get(id);
118     }
119 
120     /**
121      * Removes the scanner associated with the specified ID from the internal
122      * id->scanner hash-map.
123      *
124      * @param id
125      * @return a Scanner, or null if ID was invalid.
126      */
127     protected synchronized ResultScanner removeScanner(int id) {
128       return scannerMap.remove(id);
129     }
130 
131     //
132     // CTOR METHODS
133     //
134 
135     // TODO(hammer): figure out appropriate setting of maxSize for htablePool
136     /**
137      * Constructs an HBaseImpl object.
138      * 
139      * @throws MasterNotRunningException
140      */
141     HBaseImpl() throws MasterNotRunningException {
142       conf = HBaseConfiguration.create();
143       admin = new HBaseAdmin(conf);
144       htablePool = new HTablePool(conf, 10);
145       scannerMap = new HashMap<Integer, ResultScanner>();
146     }
147 
148     //
149     // SERVICE METHODS
150     //
151 
152     // TODO(hammer): Investigate use of the Command design pattern
153 
154     //
155     // Cluster metadata
156     //
157 
158     public Utf8 getHBaseVersion() throws AIOError {
159       try {
160 	return new Utf8(admin.getClusterStatus().getHBaseVersion());
161       } catch (IOException e) {
162 	AIOError ioe = new AIOError();
163 	ioe.message = new Utf8(e.getMessage());
164         throw ioe;
165       }
166     }
167 
168     public AClusterStatus getClusterStatus() throws AIOError {
169       try {
170 	return AvroUtil.csToACS(admin.getClusterStatus());
171       } catch (IOException e) {
172 	AIOError ioe = new AIOError();
173 	ioe.message = new Utf8(e.getMessage());
174         throw ioe;
175       }
176     }
177 
178     public GenericArray<ATableDescriptor> listTables() throws AIOError {
179       try {
180         HTableDescriptor[] tables = admin.listTables();
181 	Schema atdSchema = Schema.createArray(ATableDescriptor.SCHEMA$);
182         GenericData.Array<ATableDescriptor> result = null;
183 	result = new GenericData.Array<ATableDescriptor>(tables.length, atdSchema);
184         for (HTableDescriptor table : tables) {
185 	  result.add(AvroUtil.htdToATD(table));
186 	}
187         return result;
188       } catch (IOException e) {
189 	AIOError ioe = new AIOError();
190 	ioe.message = new Utf8(e.getMessage());
191         throw ioe;
192       }
193     }
194 
195     //
196     // Table metadata
197     //
198 
199     // TODO(hammer): Handle the case where the table does not exist explicitly?
200     public ATableDescriptor describeTable(ByteBuffer table) throws AIOError {
201       try {
202 	return AvroUtil.htdToATD(admin.getTableDescriptor(Bytes.toBytes(table)));
203       } catch (IOException e) {
204         AIOError ioe = new AIOError();
205         ioe.message = new Utf8(e.getMessage());
206         throw ioe;
207       }
208     }
209 
210     public boolean isTableEnabled(ByteBuffer table) throws AIOError {
211       try {
212 	return admin.isTableEnabled(Bytes.toBytes(table));
213       } catch (IOException e) {
214 	AIOError ioe = new AIOError();
215 	ioe.message = new Utf8(e.getMessage());
216         throw ioe;
217       }
218     }
219 
220     public boolean tableExists(ByteBuffer table) throws AIOError {
221       try {
222 	return admin.tableExists(Bytes.toBytes(table));
223       } catch (IOException e) {
224 	AIOError ioe = new AIOError();
225 	ioe.message = new Utf8(e.getMessage());
226         throw ioe;
227       }
228     }
229 
230     //
231     // Family metadata
232     //
233 
234     // TODO(hammer): Handle the case where the family does not exist explicitly?
235     public AFamilyDescriptor describeFamily(ByteBuffer table, ByteBuffer family) throws AIOError {
236       try {
237 	HTableDescriptor htd = admin.getTableDescriptor(Bytes.toBytes(table));
238 	return AvroUtil.hcdToAFD(htd.getFamily(Bytes.toBytes(family)));
239       } catch (IOException e) {
240         AIOError ioe = new AIOError();
241         ioe.message = new Utf8(e.getMessage());
242         throw ioe;
243       }
244     }
245 
246     //
247     // Table admin
248     //
249 
250     public Void createTable(ATableDescriptor table) throws AIOError, 
251                                                            AIllegalArgument,
252                                                            ATableExists,
253                                                            AMasterNotRunning {
254       try {
255         admin.createTable(AvroUtil.atdToHTD(table));
256 	return null;
257       } catch (IllegalArgumentException e) {
258 	AIllegalArgument iae = new AIllegalArgument();
259 	iae.message = new Utf8(e.getMessage());
260         throw iae;
261       } catch (TableExistsException e) {
262 	ATableExists tee = new ATableExists();
263 	tee.message = new Utf8(e.getMessage());
264         throw tee;
265       } catch (MasterNotRunningException e) {
266 	AMasterNotRunning mnre = new AMasterNotRunning();
267 	mnre.message = new Utf8(e.getMessage());
268         throw mnre;
269       } catch (IOException e) {
270 	AIOError ioe = new AIOError();
271 	ioe.message = new Utf8(e.getMessage());
272         throw ioe;
273       }
274     }
275 
276     // Note that disable, flush and major compaction of .META. needed in client
277     // TODO(hammer): more selective cache dirtying than flush?
278     public Void deleteTable(ByteBuffer table) throws AIOError {
279       try {
280 	admin.deleteTable(Bytes.toBytes(table));
281 	return null;
282       } catch (IOException e) {
283 	AIOError ioe = new AIOError();
284 	ioe.message = new Utf8(e.getMessage());
285         throw ioe;
286       }
287     }
288 
289     // NB: Asynchronous operation
290     public Void modifyTable(ByteBuffer tableName, ATableDescriptor tableDescriptor) throws AIOError {
291       try {
292 	admin.modifyTable(Bytes.toBytes(tableName),
293                           AvroUtil.atdToHTD(tableDescriptor));
294 	return null;
295       } catch (IOException e) {
296 	AIOError ioe = new AIOError();
297 	ioe.message = new Utf8(e.getMessage());
298         throw ioe;
299       }
300     }
301 
302     public Void enableTable(ByteBuffer table) throws AIOError {
303       try {
304 	admin.enableTable(Bytes.toBytes(table));
305 	return null;
306       } catch (IOException e) {
307 	AIOError ioe = new AIOError();
308 	ioe.message = new Utf8(e.getMessage());
309         throw ioe;
310       }
311     }
312     
313     public Void disableTable(ByteBuffer table) throws AIOError {
314       try {
315 	admin.disableTable(Bytes.toBytes(table));
316 	return null;
317       } catch (IOException e) {
318 	AIOError ioe = new AIOError();
319 	ioe.message = new Utf8(e.getMessage());
320         throw ioe;
321       }
322     }
323     
324     // NB: Asynchronous operation
325     public Void flush(ByteBuffer table) throws AIOError {
326       try {
327 	admin.flush(Bytes.toBytes(table));
328 	return null;
329       } catch (IOException e) {
330 	AIOError ioe = new AIOError();
331 	ioe.message = new Utf8(e.getMessage());
332         throw ioe;
333       }
334     }
335 
336     // NB: Asynchronous operation
337     public Void split(ByteBuffer table) throws AIOError {
338       try {
339 	admin.split(Bytes.toBytes(table));
340 	return null;
341       } catch (IOException e) {
342 	AIOError ioe = new AIOError();
343 	ioe.message = new Utf8(e.getMessage());
344         throw ioe;
345       }
346     }
347 
348     //
349     // Family admin
350     //
351 
352     public Void addFamily(ByteBuffer table, AFamilyDescriptor family) throws AIOError {
353       try {
354 	admin.addColumn(Bytes.toBytes(table), 
355                         AvroUtil.afdToHCD(family));
356 	return null;
357       } catch (IOException e) {
358 	AIOError ioe = new AIOError();
359 	ioe.message = new Utf8(e.getMessage());
360         throw ioe;
361       }
362     }
363 
364     // NB: Asynchronous operation
365     public Void deleteFamily(ByteBuffer table, ByteBuffer family) throws AIOError {
366       try {
367 	admin.deleteColumn(Bytes.toBytes(table), Bytes.toBytes(family));
368 	return null;
369       } catch (IOException e) {
370 	AIOError ioe = new AIOError();
371 	ioe.message = new Utf8(e.getMessage());
372         throw ioe;
373       }
374     }
375 
376     // NB: Asynchronous operation
377     public Void modifyFamily(ByteBuffer table, ByteBuffer familyName, AFamilyDescriptor familyDescriptor) throws AIOError {
378       try {
379 	admin.modifyColumn(Bytes.toBytes(table), Bytes.toBytes(familyName),
380                            AvroUtil.afdToHCD(familyDescriptor));
381 	return null;
382       } catch (IOException e) {
383 	AIOError ioe = new AIOError();
384 	ioe.message = new Utf8(e.getMessage());
385         throw ioe;
386       }
387     }
388 
389     //
390     // Single-row DML
391     //
392 
393     // TODO(hammer): Java with statement for htablepool concision?
394     // TODO(hammer): Can Get have timestamp and timerange simultaneously?
395     // TODO(hammer): Do I need to catch the RuntimeException of getTable?
396     // TODO(hammer): Handle gets with no results
397     // TODO(hammer): Uses exists(Get) to ensure columns exist
398     public AResult get(ByteBuffer table, AGet aget) throws AIOError {
399       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
400       try {
401         return AvroUtil.resultToAResult(htable.get(AvroUtil.agetToGet(aget)));
402       } catch (IOException e) {
403     	AIOError ioe = new AIOError();
404 	ioe.message = new Utf8(e.getMessage());
405         throw ioe;
406       } finally {
407         htablePool.putTable(htable);
408       }
409     }
410 
411     public boolean exists(ByteBuffer table, AGet aget) throws AIOError {
412       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
413       try {
414         return htable.exists(AvroUtil.agetToGet(aget));
415       } catch (IOException e) {
416     	AIOError ioe = new AIOError();
417 	ioe.message = new Utf8(e.getMessage());
418         throw ioe;
419       } finally {
420         htablePool.putTable(htable);
421       }
422     }
423 
424     public Void put(ByteBuffer table, APut aput) throws AIOError {
425       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
426       try {
427 	htable.put(AvroUtil.aputToPut(aput));
428         return null;
429       } catch (IOException e) {
430         AIOError ioe = new AIOError();
431         ioe.message = new Utf8(e.getMessage());
432         throw ioe;
433       } finally {
434         htablePool.putTable(htable);
435       }
436     }
437 
438     public Void delete(ByteBuffer table, ADelete adelete) throws AIOError {
439       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
440       try {
441         htable.delete(AvroUtil.adeleteToDelete(adelete));
442         return null;
443       } catch (IOException e) {
444     	AIOError ioe = new AIOError();
445 	ioe.message = new Utf8(e.getMessage());
446         throw ioe;
447       } finally {
448         htablePool.putTable(htable);
449       }
450     }
451 
452     public long incrementColumnValue(ByteBuffer table, ByteBuffer row, ByteBuffer family, ByteBuffer qualifier, long amount, boolean writeToWAL) throws AIOError {
453       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
454       try {
455 	return htable.incrementColumnValue(Bytes.toBytes(row), Bytes.toBytes(family), Bytes.toBytes(qualifier), amount, writeToWAL);
456       } catch (IOException e) {
457         AIOError ioe = new AIOError();
458         ioe.message = new Utf8(e.getMessage());
459         throw ioe;
460       } finally {
461         htablePool.putTable(htable);
462       }
463     }
464 
465     //
466     // Multi-row DML
467     //
468 
469     public int scannerOpen(ByteBuffer table, AScan ascan) throws AIOError {
470       HTableInterface htable = htablePool.getTable(Bytes.toBytes(table));
471       try {
472         Scan scan = AvroUtil.ascanToScan(ascan);
473         return addScanner(htable.getScanner(scan));
474       } catch (IOException e) {
475     	AIOError ioe = new AIOError();
476 	ioe.message = new Utf8(e.getMessage());
477         throw ioe;
478       } finally {
479         htablePool.putTable(htable);
480       }
481     }
482 
483     public Void scannerClose(int scannerId) throws AIOError, AIllegalArgument {
484       try {
485         ResultScanner scanner = getScanner(scannerId);
486         if (scanner == null) {
487       	  AIllegalArgument aie = new AIllegalArgument();
488 	  aie.message = new Utf8("scanner ID is invalid: " + scannerId);
489           throw aie;
490         }
491         scanner.close();
492         removeScanner(scannerId);
493         return null;
494       } catch (IOException e) {
495     	AIOError ioe = new AIOError();
496 	ioe.message = new Utf8(e.getMessage());
497         throw ioe;
498       }
499     }
500 
501     public GenericArray<AResult> scannerGetRows(int scannerId, int numberOfRows) throws AIOError, AIllegalArgument {
502       try {
503         ResultScanner scanner = getScanner(scannerId);
504         if (scanner == null) {
505       	  AIllegalArgument aie = new AIllegalArgument();
506 	  aie.message = new Utf8("scanner ID is invalid: " + scannerId);
507           throw aie;
508         }
509         Result[] results = null;
510         return AvroUtil.resultsToAResults(scanner.next(numberOfRows));
511       } catch (IOException e) {
512     	AIOError ioe = new AIOError();
513 	ioe.message = new Utf8(e.getMessage());
514         throw ioe;
515       }
516     }
517   }
518 
519   //
520   // MAIN PROGRAM
521   //
522 
523   private static void printUsageAndExit() {
524     printUsageAndExit(null);
525   }
526   
527   private static void printUsageAndExit(final String message) {
528     if (message != null) {
529       System.err.println(message);
530     }
531     System.out.println("Usage: java org.apache.hadoop.hbase.avro.AvroServer " +
532       "--help | [--port=PORT] start");
533     System.out.println("Arguments:");
534     System.out.println(" start Start Avro server");
535     System.out.println(" stop  Stop Avro server");
536     System.out.println("Options:");
537     System.out.println(" port  Port to listen on. Default: 9090");
538     System.out.println(" help  Print this message and exit");
539     System.exit(0);
540   }
541 
542   // TODO(hammer): Figure out a better way to keep the server alive!
543   protected static void doMain(final String[] args) throws Exception {
544     if (args.length < 1) {
545       printUsageAndExit();
546     }
547     int port = 9090;
548     final String portArgKey = "--port=";
549     for (String cmd: args) {
550       if (cmd.startsWith(portArgKey)) {
551         port = Integer.parseInt(cmd.substring(portArgKey.length()));
552         continue;
553       } else if (cmd.equals("--help") || cmd.equals("-h")) {
554         printUsageAndExit();
555       } else if (cmd.equals("start")) {
556         continue;
557       } else if (cmd.equals("stop")) {
558         printUsageAndExit("To shutdown the Avro server run " +
559           "bin/hbase-daemon.sh stop avro or send a kill signal to " +
560           "the Avro server pid");
561       }
562       
563       // Print out usage if we get to here.
564       printUsageAndExit();
565     }
566     Log LOG = LogFactory.getLog("AvroServer");
567     LOG.info("starting HBase Avro server on port " + Integer.toString(port));
568     SpecificResponder r = new SpecificResponder(HBase.class, new HBaseImpl());
569     HttpServer server = new HttpServer(r, 9090);
570     Thread.sleep(1000000);
571   }
572 
573   // TODO(hammer): Look at Cassandra's daemonization and integration with JSVC
574   // TODO(hammer): Don't eat it after a single exception
575   // TODO(hammer): Figure out why we do doMain()
576   // TODO(hammer): Figure out if we want String[] or String [] syntax
577   public static void main(String[] args) throws Exception {
578     doMain(args);
579   }
580 }