1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.thrift2;
20
21 import java.io.IOException;
22 import java.net.InetAddress;
23 import java.net.InetSocketAddress;
24 import java.net.UnknownHostException;
25 import java.security.PrivilegedAction;
26 import java.util.HashMap;
27 import java.util.List;
28 import java.util.Map;
29 import java.util.concurrent.ExecutorService;
30 import java.util.concurrent.LinkedBlockingQueue;
31 import java.util.concurrent.ThreadPoolExecutor;
32 import java.util.concurrent.TimeUnit;
33
34 import javax.security.auth.callback.Callback;
35 import javax.security.auth.callback.UnsupportedCallbackException;
36 import javax.security.sasl.AuthorizeCallback;
37 import javax.security.sasl.Sasl;
38 import javax.security.sasl.SaslServer;
39
40 import org.apache.commons.cli.CommandLine;
41 import org.apache.commons.cli.CommandLineParser;
42 import org.apache.commons.cli.HelpFormatter;
43 import org.apache.commons.cli.Option;
44 import org.apache.commons.cli.OptionGroup;
45 import org.apache.commons.cli.Options;
46 import org.apache.commons.cli.ParseException;
47 import org.apache.commons.cli.PosixParser;
48 import org.apache.commons.logging.Log;
49 import org.apache.commons.logging.LogFactory;
50 import org.apache.hadoop.hbase.classification.InterfaceAudience;
51 import org.apache.hadoop.conf.Configuration;
52 import org.apache.hadoop.hbase.HBaseConfiguration;
53 import org.apache.hadoop.hbase.filter.ParseFilter;
54 import org.apache.hadoop.hbase.security.SaslUtil;
55 import org.apache.hadoop.hbase.security.SecurityUtil;
56 import org.apache.hadoop.hbase.security.UserProvider;
57 import org.apache.hadoop.hbase.thrift.CallQueue;
58 import org.apache.hadoop.hbase.thrift.CallQueue.Call;
59 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
60 import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
61 import org.apache.hadoop.hbase.util.DNS;
62 import org.apache.hadoop.hbase.util.InfoServer;
63 import org.apache.hadoop.hbase.util.JvmPauseMonitor;
64 import org.apache.hadoop.hbase.util.Strings;
65 import org.apache.hadoop.security.UserGroupInformation;
66 import org.apache.hadoop.security.SaslRpcServer.SaslGssCallbackHandler;
67 import org.apache.hadoop.util.GenericOptionsParser;
68 import org.apache.thrift.TException;
69 import org.apache.thrift.TProcessor;
70 import org.apache.thrift.protocol.TBinaryProtocol;
71 import org.apache.thrift.protocol.TCompactProtocol;
72 import org.apache.thrift.protocol.TProtocol;
73 import org.apache.thrift.protocol.TProtocolFactory;
74 import org.apache.thrift.server.THsHaServer;
75 import org.apache.thrift.server.TNonblockingServer;
76 import org.apache.thrift.server.TServer;
77 import org.apache.thrift.server.TThreadPoolServer;
78 import org.apache.thrift.transport.TFramedTransport;
79 import org.apache.thrift.transport.TNonblockingServerSocket;
80 import org.apache.thrift.transport.TNonblockingServerTransport;
81 import org.apache.thrift.transport.TSaslServerTransport;
82 import org.apache.thrift.transport.TServerSocket;
83 import org.apache.thrift.transport.TServerTransport;
84 import org.apache.thrift.transport.TTransportException;
85 import org.apache.thrift.transport.TTransportFactory;
86
87 import com.google.common.util.concurrent.ThreadFactoryBuilder;
88
89
90
91
92
93 @InterfaceAudience.Private
94 @SuppressWarnings({ "rawtypes", "unchecked" })
95 public class ThriftServer {
96 private static final Log log = LogFactory.getLog(ThriftServer.class);
97
98
99
100
101
102
103
104
105
106
107 static final String THRIFT_QOP_KEY = "hbase.thrift.security.qop";
108
109 public static final int DEFAULT_LISTEN_PORT = 9090;
110
111 private static final String READ_TIMEOUT_OPTION = "readTimeout";
112
113 static final String BACKLOG_CONF_KEY = "hbase.regionserver.thrift.backlog";
114
115
116
117
118
119
120 public static final String THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY =
121 "hbase.thrift.server.socket.read.timeout";
122 public static final int THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT = 60000;
123
124 public ThriftServer() {
125 }
126
127 private static void printUsage() {
128 HelpFormatter formatter = new HelpFormatter();
129 formatter.printHelp("Thrift", null, getOptions(),
130 "To start the Thrift server run 'bin/hbase-daemon.sh start thrift2'\n" +
131 "To shutdown the thrift server run 'bin/hbase-daemon.sh stop thrift2' or" +
132 " send a kill signal to the thrift server pid",
133 true);
134 }
135
136 private static Options getOptions() {
137 Options options = new Options();
138 options.addOption("b", "bind", true,
139 "Address to bind the Thrift server to. [default: 0.0.0.0]");
140 options.addOption("p", "port", true, "Port to bind to [default: " + DEFAULT_LISTEN_PORT + "]");
141 options.addOption("f", "framed", false, "Use framed transport");
142 options.addOption("c", "compact", false, "Use the compact protocol");
143 options.addOption("w", "workers", true, "How many worker threads to use.");
144 options.addOption("q", "callQueueSize", true,
145 "Max size of request queue (unbounded by default)");
146 options.addOption("h", "help", false, "Print help information");
147 options.addOption(null, "infoport", true, "Port for web UI");
148 options.addOption("t", READ_TIMEOUT_OPTION, true,
149 "Amount of time in milliseconds before a server thread will timeout " +
150 "waiting for client to send data on a connected socket. Currently, " +
151 "only applies to TBoundedThreadPoolServer");
152 OptionGroup servers = new OptionGroup();
153 servers.addOption(
154 new Option("nonblocking", false, "Use the TNonblockingServer. This implies the framed transport."));
155 servers.addOption(new Option("hsha", false, "Use the THsHaServer. This implies the framed transport."));
156 servers.addOption(new Option("threadpool", false, "Use the TThreadPoolServer. This is the default."));
157 options.addOptionGroup(servers);
158 return options;
159 }
160
161 private static CommandLine parseArguments(Configuration conf, Options options, String[] args)
162 throws ParseException, IOException {
163 GenericOptionsParser genParser = new GenericOptionsParser(conf, args);
164 String[] remainingArgs = genParser.getRemainingArgs();
165 CommandLineParser parser = new PosixParser();
166 return parser.parse(options, remainingArgs);
167 }
168
169 private static TProtocolFactory getTProtocolFactory(boolean isCompact) {
170 if (isCompact) {
171 log.debug("Using compact protocol");
172 return new TCompactProtocol.Factory();
173 } else {
174 log.debug("Using binary protocol");
175 return new TBinaryProtocol.Factory();
176 }
177 }
178
179 private static TTransportFactory getTTransportFactory(
180 SaslUtil.QualityOfProtection qop, String name, String host,
181 boolean framed, int frameSize) {
182 if (framed) {
183 if (qop != null) {
184 throw new RuntimeException("Thrift server authentication"
185 + " doesn't work with framed transport yet");
186 }
187 log.debug("Using framed transport");
188 return new TFramedTransport.Factory(frameSize);
189 } else if (qop == null) {
190 return new TTransportFactory();
191 } else {
192 Map<String, String> saslProperties = new HashMap<String, String>();
193 saslProperties.put(Sasl.QOP, qop.getSaslQop());
194 TSaslServerTransport.Factory saslFactory = new TSaslServerTransport.Factory();
195 saslFactory.addServerDefinition("GSSAPI", name, host, saslProperties,
196 new SaslGssCallbackHandler() {
197 @Override
198 public void handle(Callback[] callbacks)
199 throws UnsupportedCallbackException {
200 AuthorizeCallback ac = null;
201 for (Callback callback : callbacks) {
202 if (callback instanceof AuthorizeCallback) {
203 ac = (AuthorizeCallback) callback;
204 } else {
205 throw new UnsupportedCallbackException(callback,
206 "Unrecognized SASL GSSAPI Callback");
207 }
208 }
209 if (ac != null) {
210 String authid = ac.getAuthenticationID();
211 String authzid = ac.getAuthorizationID();
212 if (!authid.equals(authzid)) {
213 ac.setAuthorized(false);
214 } else {
215 ac.setAuthorized(true);
216 String userName = SecurityUtil.getUserFromPrincipal(authzid);
217 log.info("Effective user: " + userName);
218 ac.setAuthorizedID(userName);
219 }
220 }
221 }
222 });
223 return saslFactory;
224 }
225 }
226
227
228
229
230 private static InetSocketAddress bindToPort(String bindValue, int listenPort)
231 throws UnknownHostException {
232 try {
233 if (bindValue == null) {
234 return new InetSocketAddress(listenPort);
235 } else {
236 return new InetSocketAddress(InetAddress.getByName(bindValue), listenPort);
237 }
238 } catch (UnknownHostException e) {
239 throw new RuntimeException("Could not bind to provided ip address", e);
240 }
241 }
242
243 private static TServer getTNonBlockingServer(TProtocolFactory protocolFactory, TProcessor processor,
244 TTransportFactory transportFactory, InetSocketAddress inetSocketAddress) throws TTransportException {
245 TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
246 log.info("starting HBase Nonblocking Thrift server on " + inetSocketAddress.toString());
247 TNonblockingServer.Args serverArgs = new TNonblockingServer.Args(serverTransport);
248 serverArgs.processor(processor);
249 serverArgs.transportFactory(transportFactory);
250 serverArgs.protocolFactory(protocolFactory);
251 return new TNonblockingServer(serverArgs);
252 }
253
254 private static TServer getTHsHaServer(TProtocolFactory protocolFactory,
255 TProcessor processor, TTransportFactory transportFactory,
256 int workerThreads, int maxCallQueueSize,
257 InetSocketAddress inetSocketAddress, ThriftMetrics metrics)
258 throws TTransportException {
259 TNonblockingServerTransport serverTransport = new TNonblockingServerSocket(inetSocketAddress);
260 log.info("starting HBase HsHA Thrift server on " + inetSocketAddress.toString());
261 THsHaServer.Args serverArgs = new THsHaServer.Args(serverTransport);
262 if (workerThreads > 0) {
263 serverArgs.workerThreads(workerThreads);
264 }
265 ExecutorService executorService = createExecutor(
266 workerThreads, maxCallQueueSize, metrics);
267 serverArgs.executorService(executorService);
268 serverArgs.processor(processor);
269 serverArgs.transportFactory(transportFactory);
270 serverArgs.protocolFactory(protocolFactory);
271 return new THsHaServer(serverArgs);
272 }
273
274 private static ExecutorService createExecutor(
275 int workerThreads, int maxCallQueueSize, ThriftMetrics metrics) {
276 CallQueue callQueue;
277 if (maxCallQueueSize > 0) {
278 callQueue = new CallQueue(new LinkedBlockingQueue<Call>(maxCallQueueSize), metrics);
279 } else {
280 callQueue = new CallQueue(new LinkedBlockingQueue<Call>(), metrics);
281 }
282
283 ThreadFactoryBuilder tfb = new ThreadFactoryBuilder();
284 tfb.setDaemon(true);
285 tfb.setNameFormat("thrift2-worker-%d");
286 ThreadPoolExecutor pool = new ThreadPoolExecutor(workerThreads, workerThreads,
287 Long.MAX_VALUE, TimeUnit.SECONDS, callQueue, tfb.build());
288 pool.prestartAllCoreThreads();
289 return pool;
290 }
291
292 private static TServer getTThreadPoolServer(TProtocolFactory protocolFactory,
293 TProcessor processor,
294 TTransportFactory transportFactory,
295 int workerThreads,
296 InetSocketAddress inetSocketAddress,
297 int clientTimeout)
298 throws TTransportException {
299 TServerTransport serverTransport = new TServerSocket(inetSocketAddress, clientTimeout);
300 log.info("starting HBase ThreadPool Thrift server on " + inetSocketAddress.toString());
301 TThreadPoolServer.Args serverArgs = new TThreadPoolServer.Args(serverTransport);
302 serverArgs.processor(processor);
303 serverArgs.transportFactory(transportFactory);
304 serverArgs.protocolFactory(protocolFactory);
305 if (workerThreads > 0) {
306 serverArgs.maxWorkerThreads(workerThreads);
307 }
308 return new TThreadPoolServer(serverArgs);
309 }
310
311
312
313
314
315
316 protected static void registerFilters(Configuration conf) {
317 String[] filters = conf.getStrings("hbase.thrift.filters");
318 if(filters != null) {
319 for(String filterClass: filters) {
320 String[] filterPart = filterClass.split(":");
321 if(filterPart.length != 2) {
322 log.warn("Invalid filter specification " + filterClass + " - skipping");
323 } else {
324 ParseFilter.registerFilter(filterPart[0], filterPart[1]);
325 }
326 }
327 }
328 }
329
330
331
332
333
334
335 public static void main(String[] args) throws Exception {
336 TServer server = null;
337 Options options = getOptions();
338 Configuration conf = HBaseConfiguration.create();
339 CommandLine cmd = parseArguments(conf, options, args);
340 int workerThreads = 0;
341 int maxCallQueueSize = -1;
342
343
344
345
346
347 List<?> argList = cmd.getArgList();
348 if (cmd.hasOption("help") || !argList.contains("start") || argList.contains("stop")) {
349 printUsage();
350 System.exit(1);
351 }
352
353
354 String bindAddress;
355 if (cmd.hasOption("bind")) {
356 bindAddress = cmd.getOptionValue("bind");
357 conf.set("hbase.thrift.info.bindAddress", bindAddress);
358 } else {
359 bindAddress = conf.get("hbase.thrift.info.bindAddress");
360 }
361
362
363 int readTimeout = THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT;
364 if (cmd.hasOption(READ_TIMEOUT_OPTION)) {
365 try {
366 readTimeout = Integer.parseInt(cmd.getOptionValue(READ_TIMEOUT_OPTION));
367 } catch (NumberFormatException e) {
368 throw new RuntimeException("Could not parse the value provided for the timeout option", e);
369 }
370 } else {
371 readTimeout = conf.getInt(THRIFT_SERVER_SOCKET_READ_TIMEOUT_KEY,
372 THRIFT_SERVER_SOCKET_READ_TIMEOUT_DEFAULT);
373 }
374
375
376 int listenPort = 0;
377 try {
378 if (cmd.hasOption("port")) {
379 listenPort = Integer.parseInt(cmd.getOptionValue("port"));
380 } else {
381 listenPort = conf.getInt("hbase.regionserver.thrift.port", DEFAULT_LISTEN_PORT);
382 }
383 } catch (NumberFormatException e) {
384 throw new RuntimeException("Could not parse the value provided for the port option", e);
385 }
386
387
388 int backlog = conf.getInt(BACKLOG_CONF_KEY, 0);
389
390
391
392 String host = null;
393 String name = null;
394
395 UserProvider userProvider = UserProvider.instantiate(conf);
396
397 boolean securityEnabled = userProvider.isHadoopSecurityEnabled()
398 && userProvider.isHBaseSecurityEnabled();
399 if (securityEnabled) {
400 host = Strings.domainNamePointerToHostName(DNS.getDefaultHost(
401 conf.get("hbase.thrift.dns.interface", "default"),
402 conf.get("hbase.thrift.dns.nameserver", "default")));
403 userProvider.login("hbase.thrift.keytab.file",
404 "hbase.thrift.kerberos.principal", host);
405 }
406
407 UserGroupInformation realUser = userProvider.getCurrent().getUGI();
408 String stringQop = conf.get(THRIFT_QOP_KEY);
409 SaslUtil.QualityOfProtection qop = null;
410 if (stringQop != null) {
411 qop = SaslUtil.getQop(stringQop);
412 if (!securityEnabled) {
413 throw new IOException("Thrift server must"
414 + " run in secure mode to support authentication");
415 }
416
417 name = SecurityUtil.getUserFromPrincipal(
418 conf.get("hbase.thrift.kerberos.principal"));
419 }
420
421 boolean nonblocking = cmd.hasOption("nonblocking");
422 boolean hsha = cmd.hasOption("hsha");
423
424 ThriftMetrics metrics = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO);
425 final JvmPauseMonitor pauseMonitor = new JvmPauseMonitor(conf, metrics.getSource());
426
427 String implType = "threadpool";
428 if (nonblocking) {
429 implType = "nonblocking";
430 } else if (hsha) {
431 implType = "hsha";
432 }
433
434 conf.set("hbase.regionserver.thrift.server.type", implType);
435 conf.setInt("hbase.regionserver.thrift.port", listenPort);
436 registerFilters(conf);
437
438
439 boolean compact = cmd.hasOption("compact") ||
440 conf.getBoolean("hbase.regionserver.thrift.compact", false);
441 TProtocolFactory protocolFactory = getTProtocolFactory(compact);
442 final ThriftHBaseServiceHandler hbaseHandler =
443 new ThriftHBaseServiceHandler(conf, userProvider);
444 THBaseService.Iface handler =
445 ThriftHBaseServiceHandler.newInstance(hbaseHandler, metrics);
446 final THBaseService.Processor p = new THBaseService.Processor(handler);
447 conf.setBoolean("hbase.regionserver.thrift.compact", compact);
448 TProcessor processor = p;
449
450 boolean framed = cmd.hasOption("framed") ||
451 conf.getBoolean("hbase.regionserver.thrift.framed", false) || nonblocking || hsha;
452 TTransportFactory transportFactory = getTTransportFactory(qop, name, host, framed,
453 conf.getInt("hbase.regionserver.thrift.framed.max_frame_size_in_mb", 2) * 1024 * 1024);
454 InetSocketAddress inetSocketAddress = bindToPort(bindAddress, listenPort);
455 conf.setBoolean("hbase.regionserver.thrift.framed", framed);
456 if (qop != null) {
457
458 processor = new TProcessor() {
459 @Override
460 public boolean process(TProtocol inProt,
461 TProtocol outProt) throws TException {
462 TSaslServerTransport saslServerTransport =
463 (TSaslServerTransport)inProt.getTransport();
464 SaslServer saslServer = saslServerTransport.getSaslServer();
465 String principal = saslServer.getAuthorizationID();
466 hbaseHandler.setEffectiveUser(principal);
467 return p.process(inProt, outProt);
468 }
469 };
470 }
471
472 if (cmd.hasOption("w")) {
473 workerThreads = Integer.parseInt(cmd.getOptionValue("w"));
474 }
475
476 if (cmd.hasOption("q")) {
477 maxCallQueueSize = Integer.parseInt(cmd.getOptionValue("q"));
478 }
479
480
481 try {
482 if (cmd.hasOption("infoport")) {
483 String val = cmd.getOptionValue("infoport");
484 conf.setInt("hbase.thrift.info.port", Integer.valueOf(val));
485 log.debug("Web UI port set to " + val);
486 }
487 } catch (NumberFormatException e) {
488 log.error("Could not parse the value provided for the infoport option", e);
489 printUsage();
490 System.exit(1);
491 }
492
493
494 int port = conf.getInt("hbase.thrift.info.port", 9095);
495 if (port >= 0) {
496 conf.setLong("startcode", System.currentTimeMillis());
497 String a = conf.get("hbase.thrift.info.bindAddress", "0.0.0.0");
498 InfoServer infoServer = new InfoServer("thrift", a, port, false, conf);
499 infoServer.setAttribute("hbase.conf", conf);
500 infoServer.start();
501 }
502
503 if (nonblocking) {
504 server = getTNonBlockingServer(protocolFactory,
505 processor,
506 transportFactory,
507 inetSocketAddress);
508 } else if (hsha) {
509 server = getTHsHaServer(protocolFactory,
510 processor,
511 transportFactory,
512 workerThreads,
513 maxCallQueueSize,
514 inetSocketAddress,
515 metrics);
516 } else {
517 server = getTThreadPoolServer(protocolFactory,
518 processor,
519 transportFactory,
520 workerThreads,
521 inetSocketAddress,
522 readTimeout);
523 }
524
525 final TServer tserver = server;
526 realUser.doAs(
527 new PrivilegedAction<Object>() {
528 @Override
529 public Object run() {
530 pauseMonitor.start();
531 try {
532 tserver.serve();
533 return null;
534 } finally {
535 pauseMonitor.stop();
536 }
537 }
538 });
539 }
540 }