View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase.thrift2;
20  
21  import java.io.IOException;
22  import java.net.InetAddress;
23  import java.net.InetSocketAddress;
24  import java.net.UnknownHostException;
25  import java.security.PrivilegedAction;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.concurrent.ExecutorService;
30  import java.util.concurrent.LinkedBlockingQueue;
31  import java.util.concurrent.ThreadPoolExecutor;
32  import java.util.concurrent.TimeUnit;
33  
34  import javax.security.auth.callback.Callback;
35  import javax.security.auth.callback.UnsupportedCallbackException;
36  import javax.security.sasl.AuthorizeCallback;
37  import javax.security.sasl.Sasl;
38  import javax.security.sasl.SaslServer;
39  
40  import org.apache.commons.cli.CommandLine;
41  import org.apache.commons.cli.CommandLineParser;
42  import org.apache.commons.cli.HelpFormatter;
43  import org.apache.commons.cli.Option;
44  import org.apache.commons.cli.OptionGroup;
45  import org.apache.commons.cli.Options;
46  import org.apache.commons.cli.ParseException;
47  import org.apache.commons.cli.PosixParser;
48  import org.apache.commons.logging.Log;
49  import org.apache.commons.logging.LogFactory;
50  import org.apache.hadoop.hbase.classification.InterfaceAudience;
51  import org.apache.hadoop.conf.Configuration;
52  import org.apache.hadoop.hbase.HBaseConfiguration;
53  import org.apache.hadoop.hbase.filter.ParseFilter;
54  import org.apache.hadoop.hbase.security.SaslUtil;
55  import org.apache.hadoop.hbase.security.SecurityUtil;
56  import org.apache.hadoop.hbase.security.UserProvider;
57  import org.apache.hadoop.hbase.thrift.CallQueue;
58  import org.apache.hadoop.hbase.thrift.CallQueue.Call;
59  import org.apache.hadoop.hbase.thrift.ThriftMetrics;
60  import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
61  import org.apache.hadoop.hbase.util.DNS;
62  import org.apache.hadoop.hbase.util.InfoServer;
63  import org.apache.hadoop.hbase.util.Strings;
64  import org.apache.hadoop.security.UserGroupInformation;
65  import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
66  import org.apache.hadoop.util.GenericOptionsParser;
67  import org.apache.thrift.TException;
68  import org.apache.thrift.TProcessor;
69  import org.apache.thrift.protocol.TBinaryProtocol;
70  import org.apache.thrift.protocol.TCompactProtocol;
71  import org.apache.thrift.protocol.TProtocol;
72  import org.apache.thrift.protocol.TProtocolFactory;
73  import org.apache.thrift.server.THsHaServer;
74  import org.apache.thrift.server.TNonblockingServer;
75  import org.apache.thrift.server.TServer;
76  import org.apache.thrift.server.TThreadPoolServer;
77  import org.apache.thrift.transport.TFramedTransport;
78  import org.apache.thrift.transport.TNonblockingServerSocket;
79  import org.apache.thrift.transport.TNonblockingServerTransport;
80  import org.apache.thrift.transport.TSaslServerTransport;
81  import org.apache.thrift.transport.TServerSocket;
82  import org.apache.thrift.transport.TServerTransport;
83  import org.apache.thrift.transport.TTransportException;
84  import org.apache.thrift.transport.TTransportFactory;
85  
86  import com.google.common.util.concurrent.ThreadFactoryBuilder;
87  
88  /**
89   * ThriftServer - this class starts up a Thrift server which implements the HBase API specified in the
90   * HbaseClient.thrift IDL file.
91   */
92  @InterfaceAudience.Private
93  @SuppressWarnings({ "rawtypes", "unchecked" })
94  public class ThriftServer {
95    private static final Log log = LogFactory.getLog(ThriftServer.class);
96  
97    /**
98     * Thrift quality of protection configuration key. Valid values can be:
99     * privacy: authentication, integrity and confidentiality checking
100    * integrity: authentication and integrity checking
101    * authentication: authentication only
102    *
103    * This is used to authenticate the callers and support impersonation.
104    * The thrift server and the HBase cluster must run in secure mode.
105    */
106   static final String THRIFT_QOP_KEY = "hbase.thrift.security.qop";
107 
108   public static final int DEFAULT_LISTEN_PORT = 9090;
109 
110   private static final String READ_TIMEOUT_OPTION = "readTimeout";
111 
112   static final String BACKLOG_CONF_KEY = "hbase.regionserver.thrift.backlog";
113 
114   /**
115    * Amount of time in milliseconds before a server thread will timeout
116    * waiting for client to send data on a connected socket. Currently,
117    * applies only to TBoundedThreadPoolServer
118    */
119   public static final String THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY =
120     "hbase.thrift.server.socket.read.timeout";
121   public static final int THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT = 60000;
122 
123   public ThriftServer() {
124   }
125 
126   private static void printUsage() {
127     HelpFormatter formatter = new HelpFormatter();
128     formatter.printHelp("Thrift", null, getOptions(),
129         "To start the Thrift server run 'bin/hbase-daemon.sh start thrift2'\n" +
130             "To shutdown the thrift server run 'bin/hbase-daemon.sh stop thrift2' or" +
131             " send a kill signal to the thrift server pid",
132         true);
133   }
134 
135   private static Options getOptions() {
136     Options options = new Options();
137     options.addOption("b", "bind", true,
138         "Address to bind the Thrift server to. [default: 0.0.0.0]");
139     options.addOption("p", "port", true, "Port to bind to [default: " + DEFAULT_LISTEN_PORT + "]");
140     options.addOption("f", "framed", false, "Use framed transport");
141     options.addOption("c", "compact", false, "Use the compact protocol");
142     options.addOption("h", "help", false, "Print help information");
143     options.addOption(null, "infoport", true, "Port for web UI");
144     options.addOption("t", READ_TIMEOUT_OPTION, true,
145       "Amount of time in milliseconds before a server thread will timeout " +
146       "waiting for client to send data on a connected socket. Currently, " +
147       "only applies to TBoundedThreadPoolServer");
148     OptionGroup servers = new OptionGroup();
149     servers.addOption(
150         new Option("nonblocking", false, "Use the TNonblockingServer. This implies the framed transport."));
151     servers.addOption(new Option("hsha", false, "Use the THsHaServer. This implies the framed transport."));
152     servers.addOption(new Option("threadpool", false, "Use the TThreadPoolServer. This is the default."));
153     options.addOptionGroup(servers);
154     return options;
155   }
156 
157   private static CommandLine parseArguments(Configuration conf, Options options, String[] args)
158       throws ParseException, IOException {
159     GenericOptionsParser genParser = new GenericOptionsParser(conf, args);
160     String[] remainingArgs = genParser.getRemainingArgs();
161     CommandLineParser parser = new PosixParser();
162     return parser.parse(options, remainingArgs);
163   }
164 
165   private static TProtocolFactory getTProtocolFactory(boolean isCompact) {
166     if (isCompact) {
167       log.debug("Using compact protocol");
168       return new TCompactProtocol.Factory();
169     } else {
170       log.debug("Using binary protocol");
171       return new TBinaryProtocol.Factory();
172     }
173   }
174 
175   private static TTransportFactory getTTransportFactory(
176       SaslUtil.QualityOfProtection qop, String name, String host,
177       boolean framed, int frameSize) {
178     if (framed) {
179       if (qop != null) {
180         throw new RuntimeException("Thrift server authentication"
181           + " doesn't work with framed transport yet");
182       }
183       log.debug("Using framed transport");
184       return new TFramedTransport.Factory(frameSize);
185     } else if (qop == null) {
186       return new TTransportFactory();
187     } else {
188       Map<String, String> saslProperties = new HashMap<String, String>();
189       saslProperties.put(Sasl.QOP, qop.getSaslQop());
190       TSaslServerTransport.Factory saslFactory = new TSaslServerTransport.Factory();
191       saslFactory.addServerDefinition("GSSAPI", name, host, saslProperties,
192         new SaslGssCallbackHandler() {
193           @Override
194           public void handle(Callback[] callbacks)
195               throws UnsupportedCallbackException {
196             AuthorizeCallback ac = null;
197             for (Callback callback : callbacks) {
198               if (callback instanceof AuthorizeCallback) {
199                 ac = (AuthorizeCallback) callback;
200               } else {
201                 throw new UnsupportedCallbackException(callback,
202                     "Unrecognized SASL GSSAPI Callback");
203               }
204             }
205             if (ac != null) {
206               String authid = ac.getAuthenticationID();
207               String authzid = ac.getAuthorizationID();
208               if (!authid.equals(authzid)) {
209                 ac.setAuthorized(false);
210               } else {
211                 ac.setAuthorized(true);
212                 String userName = SecurityUtil.getUserFromPrincipal(authzid);
213                 log.info("Effective user: " + userName);
214                 ac.setAuthorizedID(userName);
215               }
216             }
217           }
218         });
219       return saslFactory;
220     }
221   }
222 
223   /*
224    * If bindValue is null, we don't bind.
225    */
226   private static InetSocketAddress bindToPort(String bindValue, int listenPort)
227       throws UnknownHostException {
228     try {
229       if (bindValue == null) {
230         return new InetSocketAddress(listenPort);
231       } else {
232         return new InetSocketAddress(InetAddress.getByName(bindValue), listenPort);
233       }
234     } catch (UnknownHostException e) {
235       throw new RuntimeException("Could not bind to provided ip address", e);
236     }
237   }
238 
239   private static TServer getTNonBlockingServer(TProtocolFactory protocolFactory, TProcessor processor,
240       TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException {
241     TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
242     log.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString());
243     TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
244     serverArgs.processor(processor);
245     serverArgs.transportFactory(transportFactory);
246     serverArgs.protocolFactory(protocolFactory);
247     return new TNonblockingServer(serverArgs);
248   }
249 
250   private static TServer getTHsHaServer(TProtocolFactory protocolFactory,
251       TProcessor processor, TTransportFactory transportFactory,
252       InetSocketAddress inetSocketAddress, ThriftMetrics metrics)
253       throws TTransportException {
254     TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
255     log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
256     THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
257     ExecutorService executorService = createExecutor(
258         serverArgs.getWorkerThreads(), metrics);
259     serverArgs.executorService(executorService);
260     serverArgs.processor(processor);
261     serverArgs.transportFactory(transportFactory);
262     serverArgs.protocolFactory(protocolFactory);
263     return new THsHaServer(serverArgs);
264   }
265 
266   private static ExecutorService createExecutor(
267       int workerThreads, ThriftMetrics metrics) {
268     CallQueue callQueue = new CallQueue(
269         new LinkedBlockingQueue<Call>(), metrics);
270     ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
271     tfb.setDaemon(true);
272     tfb.setNameFormat("thrift2-worker-%d");
273     return new ThreadPoolExecutor(workerThreads, workerThreads,
274             Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build());
275   }
276 
277   private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory,
278                                               TProcessor processor,
279                                               TTransportFactory transportFactory,
280                                               InetSocketAddress inetSocketAddress,
281                                               int clientTimeout)
282       throws TTransportException {
283     TServerTransport serverTransport = new TServerSocket(inetSocketAddress, clientTimeout);
284     log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
285     TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport);
286     serverArgs.processor(processor);
287     serverArgs.transportFactory(transportFactory);
288     serverArgs.protocolFactory(protocolFactory);
289     return new TThreadPoolServer(serverArgs);
290   }
291 
292   /**
293    * Adds the option to pre-load filters at startup.
294    *
295    * @param conf  The current configuration instance.
296    */
297   protected static void registerFilters(Configuration conf) {
298     String[] filters = conf.getStrings("hbase.thrift.filters");
299     if(filters != null) {
300       for(String filterClass: filters) {
301         String[] filterPart = filterClass.split(":");
302         if(filterPart.length != 2) {
303           log.warn("Invalid filter specification " + filterClass + " - skipping");
304         } else {
305           ParseFilter.registerFilter(filterPart[0], filterPart[1]);
306         }
307       }
308     }
309   }
310 
311   /**
312    * Start up the Thrift2 server.
313    *
314    * @param args
315    */
316   public static void main(String[] args) throws Exception {
317     TServer server = null;
318     Options options = getOptions();
319     Configuration conf = HBaseConfiguration.create();
320     CommandLine cmd = parseArguments(conf, options, args);
321 
322     /**
323      * This is to please both bin/hbase and bin/hbase-daemon. hbase-daemon provides "start" and "stop" arguments hbase
324      * should print the help if no argument is provided
325      */
326     List<?> argList = cmd.getArgList();
327     if (cmd.hasOption("help") || !argList.contains("start") || argList.contains("stop")) {
328       printUsage();
329       System.exit(1);
330     }
331 
332     // Get address to bind
333     String bindAddress;
334     if (cmd.hasOption("bind")) {
335       bindAddress = cmd.getOptionValue("bind");
336       conf.set("hbase.thrift.info.bindAddress", bindAddress);
337     } else {
338       bindAddress = conf.get("hbase.thrift.info.bindAddress");
339     }
340 
341     // Get read timeout
342     int readTimeout = THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT;
343     if (cmd.hasOption(READ_TIMEOUT_OPTION)) {
344       try {
345         readTimeout = Integer.parseInt(cmd.getOptionValue(READ_TIMEOUT_OPTION));
346       } catch (NumberFormatException e) {
347         throw new RuntimeException("Could not parse the value provided for the timeout option", e);
348       }
349     } else {
350       readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
351         THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
352     }
353 
354     // Get port to bind to
355     int listenPort = 0;
356     try {
357       if (cmd.hasOption("port")) {
358         listenPort = Integer.parseInt(cmd.getOptionValue("port"));
359       } else {
360         listenPort = conf.getInt("hbase.regionserver.thrift.port", DEFAULT_LISTEN_PORT);
361       }
362     } catch (NumberFormatException e) {
363       throw new RuntimeException("Could not parse the value provided for the port option", e);
364     }
365 
366     // Thrift's implementation uses '0' as a placeholder for 'use the default.'
367     int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
368 
369     // Local hostname and user name,
370     // used only if QOP is configured.
371     String host = null;
372     String name = null;
373 
374     UserProvider userProvider = UserProvider.instantiate(conf);
375     // login the server principal (if using secure Hadoop)
376     boolean securityEnabled = userProvider.isHadoopSecurityEnabled()
377       && userProvider.isHBaseSecurityEnabled();
378     if (securityEnabled) {
379       host = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
380         conf.get("hbase.thrift.dns.interface", "default"),
381         conf.get("hbase.thrift.dns.nameserver", "default")));
382       userProvider.login("hbase.thrift.keytab.file",
383         "hbase.thrift.kerberos.principal", host);
384     }
385 
386     UserGroupInformation realUser = userProvider.getCurrent().getUGI();
387     String stringQop = conf.get(THRIFT_QOP_KEY);
388     SaslUtil.QualityOfProtection qop = null;
389     if (stringQop != null) {
390       qop = SaslUtil.getQop(stringQop);
391       if (!securityEnabled) {
392         throw new IOException("Thrift server must"
393           + " run in secure mode to support authentication");
394       }
395       // Extract the name from the principal
396       name = SecurityUtil.getUserFromPrincipal(
397         conf.get("hbase.thrift.kerberos.principal"));
398     }
399 
400     boolean nonblocking = cmd.hasOption("nonblocking");
401     boolean hsha = cmd.hasOption("hsha");
402 
403     ThriftMetrics metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO);
404 
405     String implType = "threadpool";
406     if (nonblocking) {
407       implType = "nonblocking";
408     } else if (hsha) {
409       implType = "hsha";
410     }
411 
412     conf.set("hbase.regionserver.thrift.server.type", implType);
413     conf.setInt("hbase.regionserver.thrift.port", listenPort);
414     registerFilters(conf);
415 
416     // Construct correct ProtocolFactory
417     boolean compact = cmd.hasOption("compact") ||
418         conf.getBoolean("hbase.regionserver.thrift.compact", false);
419     TProtocolFactory protocolFactory = getTProtocolFactory(compact);
420     final ThriftHBaseServiceHandler hbaseHandler =
421       new ThriftHBaseServiceHandler(conf, userProvider);
422     THBaseService.Iface handler =
423       ThriftHBaseServiceHandler.newInstance(hbaseHandler, metrics);
424     final THBaseService.Processor p = new THBaseService.Processor(handler);
425     conf.setBoolean("hbase.regionserver.thrift.compact", compact);
426     TProcessor processor = p;
427 
428     boolean framed = cmd.hasOption("framed") ||
429         conf.getBoolean("hbase.regionserver.thrift.framed", false) || nonblocking || hsha;
430     TTransportFactory transportFactory = getTTransportFactory(qop, name, host, framed,
431         conf.getInt("hbase.regionserver.thrift.framed.max_frame_size_in_mb", 2) * 1024 * 1024);
432     InetSocketAddress inetSocketAddress = bindToPort(bindAddress, listenPort);
433     conf.setBoolean("hbase.regionserver.thrift.framed", framed);
434     if (qop != null) {
435       // Create a processor wrapper, to get the caller
436       processor = new TProcessor() {
437         @Override
438         public boolean process(TProtocol inProt,
439             TProtocol outProt) throws TException {
440           TSaslServerTransport saslServerTransport =
441             (TSaslServerTransport)inProt.getTransport();
442           SaslServer saslServer = saslServerTransport.getSaslServer();
443           String principal = saslServer.getAuthorizationID();
444           hbaseHandler.setEffectiveUser(principal);
445           return p.process(inProt, outProt);
446         }
447       };
448     }
449 
450     // check for user-defined info server port setting, if so override the conf
451     try {
452       if (cmd.hasOption("infoport")) {
453         String val = cmd.getOptionValue("infoport");
454         conf.setInt("hbase.thrift.info.port", Integer.valueOf(val));
455         log.debug("Web UI port set to " + val);
456       }
457     } catch (NumberFormatException e) {
458       log.error("Could not parse the value provided for the infoport option", e);
459       printUsage();
460       System.exit(1);
461     }
462 
463     // Put up info server.
464     int port = conf.getInt("hbase.thrift.info.port", 9095);
465     if (port >= 0) {
466       conf.setLong("startcode", System.currentTimeMillis());
467       String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0");
468       InfoServer infoServer = new InfoServer("thrift", a, port, false, conf);
469       infoServer.setAttribute("hbase.conf", conf);
470       infoServer.start();
471     }
472 
473     if (nonblocking) {
474       server = getTNonBlockingServer(protocolFactory, processor, transportFactory, inetSocketAddress);
475     } else if (hsha) {
476       server = getTHsHaServer(protocolFactory, processor, transportFactory, inetSocketAddress, metrics);
477     } else {
478       server = getTThreadPoolServer(protocolFactory,
479           processor,
480           transportFactory,
481           inetSocketAddress,
482           readTimeout);
483     }
484 
485     final TServer tserver = server;
486     realUser.doAs(
487       new PrivilegedAction<Object>() {
488         @Override
489         public Object run() {
490           tserver.serve();
491           return null;
492         }
493       });
494   }
495 }