View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.net.InetSocketAddress;
24  import java.net.UnknownHostException;
25  import java.security.PrivilegedAction;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.concurrent.ExecutorService;
30  import java.util.concurrent.LinkedBlockingQueue;
31  import java.util.concurrent.ThreadPoolExecutor;
32  import java.util.concurrent.TimeUnit;
33  
34  import javax.security.auth.callback.Callback;
35  import javax.security.auth.callback.UnsupportedCallbackException;
36  import javax.security.sasl.AuthorizeCallback;
37  import javax.security.sasl.Sasl;
38  import javax.security.sasl.SaslServer;
39  
40  import org.apache.commons.cli.CommandLine;
41  import org.apache.commons.cli.CommandLineParser;
42  import org.apache.commons.cli.HelpFormatter;
43  import org.apache.commons.cli.Option;
44  import org.apache.commons.cli.OptionGroup;
45  import org.apache.commons.cli.Options;
46  import org.apache.commons.cli.ParseException;
47  import org.apache.commons.cli.PosixParser;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.hbase.classification.InterfaceAudience;
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.HBaseConfiguration;
53  import org.apache.hadoop.hbase.filter.ParseFilter;
54  import org.apache.hadoop.hbase.security.SaslUtil;
55  import org.apache.hadoop.hbase.security.SecurityUtil;
56  import org.apache.hadoop.hbase.security.UserProvider;
57  import org.apache.hadoop.hbase.thrift.CallQueue;
58  import org.apache.hadoop.hbase.thrift.CallQueue.Call;
59  import org.apache.hadoop.hbase.thrift.ThriftMetrics;
60  import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
61  import org.apache.hadoop.hbase.util.DNS;
62  import org.apache.hadoop.hbase.util.InfoServer;
63  import org.apache.hadoop.hbase.util.JvmPauseMonitor;
64  import org.apache.hadoop.hbase.util.Strings;
65  import org.apache.hadoop.security.UserGroupInformation;
66  import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
67  import org.apache.hadoop.util.GenericOptionsParser;
68  import org.apache.thrift.TException;
69  import org.apache.thrift.TProcessor;
70  import org.apache.thrift.protocol.TBinaryProtocol;
71  import org.apache.thrift.protocol.TCompactProtocol;
72  import org.apache.thrift.protocol.TProtocol;
73  import org.apache.thrift.protocol.TProtocolFactory;
74  import org.apache.thrift.server.THsHaServer;
75  import org.apache.thrift.server.TNonblockingServer;
76  import org.apache.thrift.server.TServer;
77  import org.apache.thrift.server.TThreadPoolServer;
78  import org.apache.thrift.transport.TFramedTransport;
79  import org.apache.thrift.transport.TNonblockingServerSocket;
80  import org.apache.thrift.transport.TNonblockingServerTransport;
81  import org.apache.thrift.transport.TSaslServerTransport;
82  import org.apache.thrift.transport.TServerSocket;
83  import org.apache.thrift.transport.TServerTransport;
84  import org.apache.thrift.transport.TTransportException;
85  import org.apache.thrift.transport.TTransportFactory;
86  
87  import com.google.common.util.concurrent.ThreadFactoryBuilder;
88  
89  /**
90   * ThriftServer - this class starts up a Thrift server which implements the HBase API specified in the
91   * HbaseClient.thrift IDL file.
92   */
93  @InterfaceAudience.Private
94  @SuppressWarnings({ "rawtypes", "unchecked" })
95  public class ThriftServer {
96    private static final Log log = LogFactory.getLog(ThriftServer.class);
97  
98    /**
99     * Thrift quality of protection configuration key. Valid values can be:
100    * privacy: authentication, integrity and confidentiality checking
101    * integrity: authentication and integrity checking
102    * authentication: authentication only
103    *
104    * This is used to authenticate the callers and support impersonation.
105    * The thrift server and the HBase cluster must run in secure mode.
106    */
107   static final String THRIFT_QOP_KEY = "hbase.thrift.security.qop";
108 
109   public static final int DEFAULT_LISTEN_PORT = 9090;
110 
111   private static final String READ_TIMEOUT_OPTION = "readTimeout";
112 
113   static final String BACKLOG_CONF_KEY = "hbase.regionserver.thrift.backlog";
114 
115   /**
116    * Amount of time in milliseconds before a server thread will timeout
117    * waiting for client to send data on a connected socket. Currently,
118    * applies only to TBoundedThreadPoolServer
119    */
120   public static final String THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY =
121     "hbase.thrift.server.socket.read.timeout";
122   public static final int THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT = 60000;
123 
124   public ThriftServer() {
125   }
126 
127   private static void printUsage() {
128     HelpFormatter formatter = new HelpFormatter();
129     formatter.printHelp("Thrift", null, getOptions(),
130         "To start the Thrift server run 'bin/hbase-daemon.sh start thrift2'\n" +
131             "To shutdown the thrift server run 'bin/hbase-daemon.sh stop thrift2' or" +
132             " send a kill signal to the thrift server pid",
133         true);
134   }
135 
136   private static Options getOptions() {
137     Options options = new Options();
138     options.addOption("b", "bind", true,
139         "Address to bind the Thrift server to. [default: 0.0.0.0]");
140     options.addOption("p", "port", true, "Port to bind to [default: " + DEFAULT_LISTEN_PORT + "]");
141     options.addOption("f", "framed", false, "Use framed transport");
142     options.addOption("c", "compact", false, "Use the compact protocol");
143     options.addOption("w", "workers", true, "How many worker threads to use.");
144     options.addOption("q", "callQueueSize", true,
145       "Max size of request queue (unbounded by default)");
146     options.addOption("h", "help", false, "Print help information");
147     options.addOption(null, "infoport", true, "Port for web UI");
148     options.addOption("t", READ_TIMEOUT_OPTION, true,
149       "Amount of time in milliseconds before a server thread will timeout " +
150       "waiting for client to send data on a connected socket. Currently, " +
151       "only applies to TBoundedThreadPoolServer");
152     OptionGroup servers = new OptionGroup();
153     servers.addOption(
154         new Option("nonblocking", false, "Use the TNonblockingServer. This implies the framed transport."));
155     servers.addOption(new Option("hsha", false, "Use the THsHaServer. This implies the framed transport."));
156     servers.addOption(new Option("threadpool", false, "Use the TThreadPoolServer. This is the default."));
157     options.addOptionGroup(servers);
158     return options;
159   }
160 
161   private static CommandLine parseArguments(Configuration conf, Options options, String[] args)
162       throws ParseException, IOException {
163     GenericOptionsParser genParser = new GenericOptionsParser(conf, args);
164     String[] remainingArgs = genParser.getRemainingArgs();
165     CommandLineParser parser = new PosixParser();
166     return parser.parse(options, remainingArgs);
167   }
168 
169   private static TProtocolFactory getTProtocolFactory(boolean isCompact) {
170     if (isCompact) {
171       log.debug("Using compact protocol");
172       return new TCompactProtocol.Factory();
173     } else {
174       log.debug("Using binary protocol");
175       return new TBinaryProtocol.Factory();
176     }
177   }
178 
179   private static TTransportFactory getTTransportFactory(
180       SaslUtil.QualityOfProtection qop, String name, String host,
181       boolean framed, int frameSize) {
182     if (framed) {
183       if (qop != null) {
184         throw new RuntimeException("Thrift server authentication"
185           + " doesn't work with framed transport yet");
186       }
187       log.debug("Using framed transport");
188       return new TFramedTransport.Factory(frameSize);
189     } else if (qop == null) {
190       return new TTransportFactory();
191     } else {
192       Map<String, String> saslProperties = new HashMap<String, String>();
193       saslProperties.put(Sasl.QOP, qop.getSaslQop());
194       TSaslServerTransport.Factory saslFactory = new TSaslServerTransport.Factory();
195       saslFactory.addServerDefinition("GSSAPI", name, host, saslProperties,
196         new SaslGssCallbackHandler() {
197           @Override
198           public void handle(Callback[] callbacks)
199               throws UnsupportedCallbackException {
200             AuthorizeCallback ac = null;
201             for (Callback callback : callbacks) {
202               if (callback instanceof AuthorizeCallback) {
203                 ac = (AuthorizeCallback) callback;
204               } else {
205                 throw new UnsupportedCallbackException(callback,
206                     "Unrecognized SASL GSSAPI Callback");
207               }
208             }
209             if (ac != null) {
210               String authid = ac.getAuthenticationID();
211               String authzid = ac.getAuthorizationID();
212               if (!authid.equals(authzid)) {
213                 ac.setAuthorized(false);
214               } else {
215                 ac.setAuthorized(true);
216                 String userName = SecurityUtil.getUserFromPrincipal(authzid);
217                 log.info("Effective user: " + userName);
218                 ac.setAuthorizedID(userName);
219               }
220             }
221           }
222         });
223       return saslFactory;
224     }
225   }
226 
227   /*
228    * If bindValue is null, we don't bind.
229    */
230   private static InetSocketAddress bindToPort(String bindValue, int listenPort)
231       throws UnknownHostException {
232     try {
233       if (bindValue == null) {
234         return new InetSocketAddress(listenPort);
235       } else {
236         return new InetSocketAddress(InetAddress.getByName(bindValue), listenPort);
237       }
238     } catch (UnknownHostException e) {
239       throw new RuntimeException("Could not bind to provided ip address", e);
240     }
241   }
242 
243   private static TServer getTNonBlockingServer(TProtocolFactory protocolFactory, TProcessor processor,
244       TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException {
245     TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
246     log.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString());
247     TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
248     serverArgs.processor(processor);
249     serverArgs.transportFactory(transportFactory);
250     serverArgs.protocolFactory(protocolFactory);
251     return new TNonblockingServer(serverArgs);
252   }
253 
254   private static TServer getTHsHaServer(TProtocolFactory protocolFactory,
255       TProcessor processor, TTransportFactory transportFactory,
256       int workerThreads, int maxCallQueueSize,
257       InetSocketAddress inetSocketAddress, ThriftMetrics metrics)
258       throws TTransportException {
259     TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
260     log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
261     THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
262     if (workerThreads > 0) {
263       serverArgs.workerThreads(workerThreads);
264     }
265     ExecutorService executorService = createExecutor(
266         workerThreads, maxCallQueueSize, metrics);
267     serverArgs.executorService(executorService);
268     serverArgs.processor(processor);
269     serverArgs.transportFactory(transportFactory);
270     serverArgs.protocolFactory(protocolFactory);
271     return new THsHaServer(serverArgs);
272   }
273 
274   private static ExecutorService createExecutor(
275       int workerThreads, int maxCallQueueSize, ThriftMetrics metrics) {
276     CallQueue callQueue;
277     if (maxCallQueueSize > 0) {
278       callQueue = new CallQueue(new LinkedBlockingQueue<Call>(maxCallQueueSize), metrics);
279     } else {
280       callQueue = new CallQueue(new LinkedBlockingQueue<Call>(), metrics);
281     }
282 
283     ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
284     tfb.setDaemon(true);
285     tfb.setNameFormat("thrift2-worker-%d");
286     ThreadPoolExecutor pool = new ThreadPoolExecutor(workerThreads, workerThreads,
287             Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build());
288     pool.prestartAllCoreThreads();
289     return pool;
290   }
291 
292   private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory,
293                                               TProcessor processor,
294                                               TTransportFactory transportFactory,
295                                               int workerThreads,
296                                               InetSocketAddress inetSocketAddress,
297                                               int clientTimeout)
298       throws TTransportException {
299     TServerTransport serverTransport = new TServerSocket(inetSocketAddress, clientTimeout);
300     log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
301     TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport);
302     serverArgs.processor(processor);
303     serverArgs.transportFactory(transportFactory);
304     serverArgs.protocolFactory(protocolFactory);
305     if (workerThreads > 0) {
306       serverArgs.maxWorkerThreads(workerThreads);
307     }
308     return new TThreadPoolServer(serverArgs);
309   }
310 
311   /**
312    * Adds the option to pre-load filters at startup.
313    *
314    * @param conf  The current configuration instance.
315    */
316   protected static void registerFilters(Configuration conf) {
317     String[] filters = conf.getStrings("hbase.thrift.filters");
318     if(filters != null) {
319       for(String filterClass: filters) {
320         String[] filterPart = filterClass.split(":");
321         if(filterPart.length != 2) {
322           log.warn("Invalid filter specification " + filterClass + " - skipping");
323         } else {
324           ParseFilter.registerFilter(filterPart[0], filterPart[1]);
325         }
326       }
327     }
328   }
329 
330   /**
331    * Start up the Thrift2 server.
332    *
333    * @param args
334    */
335   public static void main(String[] args) throws Exception {
336     TServer server = null;
337     Options options = getOptions();
338     Configuration conf = HBaseConfiguration.create();
339     CommandLine cmd = parseArguments(conf, options, args);
340     int workerThreads = 0;
341     int maxCallQueueSize = -1; // use unbounded queue by default
342 
343     /**
344      * This is to please both bin/hbase and bin/hbase-daemon. hbase-daemon provides "start" and "stop" arguments hbase
345      * should print the help if no argument is provided
346      */
347     List<?> argList = cmd.getArgList();
348     if (cmd.hasOption("help") || !argList.contains("start") || argList.contains("stop")) {
349       printUsage();
350       System.exit(1);
351     }
352 
353     // Get address to bind
354     String bindAddress;
355     if (cmd.hasOption("bind")) {
356       bindAddress = cmd.getOptionValue("bind");
357       conf.set("hbase.thrift.info.bindAddress", bindAddress);
358     } else {
359       bindAddress = conf.get("hbase.thrift.info.bindAddress");
360     }
361 
362     // Get read timeout
363     int readTimeout = THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT;
364     if (cmd.hasOption(READ_TIMEOUT_OPTION)) {
365       try {
366         readTimeout = Integer.parseInt(cmd.getOptionValue(READ_TIMEOUT_OPTION));
367       } catch (NumberFormatException e) {
368         throw new RuntimeException("Could not parse the value provided for the timeout option", e);
369       }
370     } else {
371       readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
372         THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
373     }
374 
375     // Get port to bind to
376     int listenPort = 0;
377     try {
378       if (cmd.hasOption("port")) {
379         listenPort = Integer.parseInt(cmd.getOptionValue("port"));
380       } else {
381         listenPort = conf.getInt("hbase.regionserver.thrift.port", DEFAULT_LISTEN_PORT);
382       }
383     } catch (NumberFormatException e) {
384       throw new RuntimeException("Could not parse the value provided for the port option", e);
385     }
386 
387     // Thrift's implementation uses '0' as a placeholder for 'use the default.'
388     int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
389 
390     // Local hostname and user name,
391     // used only if QOP is configured.
392     String host = null;
393     String name = null;
394 
395     UserProvider userProvider = UserProvider.instantiate(conf);
396     // login the server principal (if using secure Hadoop)
397     boolean securityEnabled = userProvider.isHadoopSecurityEnabled()
398       && userProvider.isHBaseSecurityEnabled();
399     if (securityEnabled) {
400       host = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
401         conf.get("hbase.thrift.dns.interface", "default"),
402         conf.get("hbase.thrift.dns.nameserver", "default")));
403       userProvider.login("hbase.thrift.keytab.file",
404         "hbase.thrift.kerberos.principal", host);
405     }
406 
407     UserGroupInformation realUser = userProvider.getCurrent().getUGI();
408     String stringQop = conf.get(THRIFT_QOP_KEY);
409     SaslUtil.QualityOfProtection qop = null;
410     if (stringQop != null) {
411       qop = SaslUtil.getQop(stringQop);
412       if (!securityEnabled) {
413         throw new IOException("Thrift server must"
414           + " run in secure mode to support authentication");
415       }
416       // Extract the name from the principal
417       name = SecurityUtil.getUserFromPrincipal(
418         conf.get("hbase.thrift.kerberos.principal"));
419     }
420 
421     boolean nonblocking = cmd.hasOption("nonblocking");
422     boolean hsha = cmd.hasOption("hsha");
423 
424     ThriftMetrics metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO);
425     final JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(conf, metrics.getSource());
426 
427     String implType = "threadpool";
428     if (nonblocking) {
429       implType = "nonblocking";
430     } else if (hsha) {
431       implType = "hsha";
432     }
433 
434     conf.set("hbase.regionserver.thrift.server.type", implType);
435     conf.setInt("hbase.regionserver.thrift.port", listenPort);
436     registerFilters(conf);
437 
438     // Construct correct ProtocolFactory
439     boolean compact = cmd.hasOption("compact") ||
440         conf.getBoolean("hbase.regionserver.thrift.compact", false);
441     TProtocolFactory protocolFactory = getTProtocolFactory(compact);
442     final ThriftHBaseServiceHandler hbaseHandler =
443       new ThriftHBaseServiceHandler(conf, userProvider);
444     THBaseService.Iface handler =
445       ThriftHBaseServiceHandler.newInstance(hbaseHandler, metrics);
446     final THBaseService.Processor p = new THBaseService.Processor(handler);
447     conf.setBoolean("hbase.regionserver.thrift.compact", compact);
448     TProcessor processor = p;
449 
450     boolean framed = cmd.hasOption("framed") ||
451         conf.getBoolean("hbase.regionserver.thrift.framed", false) || nonblocking || hsha;
452     TTransportFactory transportFactory = getTTransportFactory(qop, name, host, framed,
453         conf.getInt("hbase.regionserver.thrift.framed.max_frame_size_in_mb", 2) * 1024 * 1024);
454     InetSocketAddress inetSocketAddress = bindToPort(bindAddress, listenPort);
455     conf.setBoolean("hbase.regionserver.thrift.framed", framed);
456     if (qop != null) {
457       // Create a processor wrapper, to get the caller
458       processor = new TProcessor() {
459         @Override
460         public boolean process(TProtocol inProt,
461             TProtocol outProt) throws TException {
462           TSaslServerTransport saslServerTransport =
463             (TSaslServerTransport)inProt.getTransport();
464           SaslServer saslServer = saslServerTransport.getSaslServer();
465           String principal = saslServer.getAuthorizationID();
466           hbaseHandler.setEffectiveUser(principal);
467           return p.process(inProt, outProt);
468         }
469       };
470     }
471 
472     if (cmd.hasOption("w")) {
473       workerThreads = Integer.parseInt(cmd.getOptionValue("w"));
474     }
475 
476     if (cmd.hasOption("q")) {
477       maxCallQueueSize = Integer.parseInt(cmd.getOptionValue("q"));
478     }
479 
480     // check for user-defined info server port setting, if so override the conf
481     try {
482       if (cmd.hasOption("infoport")) {
483         String val = cmd.getOptionValue("infoport");
484         conf.setInt("hbase.thrift.info.port", Integer.valueOf(val));
485         log.debug("Web UI port set to " + val);
486       }
487     } catch (NumberFormatException e) {
488       log.error("Could not parse the value provided for the infoport option", e);
489       printUsage();
490       System.exit(1);
491     }
492 
493     // Put up info server.
494     int port = conf.getInt("hbase.thrift.info.port", 9095);
495     if (port >= 0) {
496       conf.setLong("startcode", System.currentTimeMillis());
497       String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0");
498       InfoServer infoServer = new InfoServer("thrift", a, port, false, conf);
499       infoServer.setAttribute("hbase.conf", conf);
500       infoServer.start();
501     }
502 
503     if (nonblocking) {
504       server = getTNonBlockingServer(protocolFactory,
505           processor,
506           transportFactory,
507           inetSocketAddress);
508     } else if (hsha) {
509       server = getTHsHaServer(protocolFactory,
510           processor,
511           transportFactory,
512           workerThreads,
513           maxCallQueueSize,
514           inetSocketAddress,
515           metrics);
516     } else {
517       server = getTThreadPoolServer(protocolFactory,
518           processor,
519           transportFactory,
520           workerThreads,
521           inetSocketAddress,
522           readTimeout);
523     }
524 
525     final TServer tserver = server;
526     realUser.doAs(
527       new PrivilegedAction<Object>() {
528         @Override
529         public Object run() {
530           pauseMonitor.start();
531           try {
532             tserver.serve();
533             return null;
534           } finally {
535             pauseMonitor.stop();
536           }
537         }
538       });
539   }
540 }