View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.ipc;
20  
21  import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION;
22  
23  import java.io.ByteArrayInputStream;
24  import java.io.ByteArrayOutputStream;
25  import java.io.DataOutputStream;
26  import java.io.IOException;
27  import java.net.BindException;
28  import java.net.InetAddress;
29  import java.net.InetSocketAddress;
30  import java.net.ServerSocket;
31  import java.net.Socket;
32  import java.net.SocketException;
33  import java.net.UnknownHostException;
34  import java.nio.ByteBuffer;
35  import java.nio.channels.CancelledKeyException;
36  import java.nio.channels.Channels;
37  import java.nio.channels.ClosedChannelException;
38  import java.nio.channels.GatheringByteChannel;
39  import java.nio.channels.ReadableByteChannel;
40  import java.nio.channels.SelectionKey;
41  import java.nio.channels.Selector;
42  import java.nio.channels.ServerSocketChannel;
43  import java.nio.channels.SocketChannel;
44  import java.nio.channels.WritableByteChannel;
45  import java.security.PrivilegedExceptionAction;
46  import java.util.ArrayList;
47  import java.util.Collections;
48  import java.util.HashMap;
49  import java.util.Iterator;
50  import java.util.LinkedList;
51  import java.util.List;
52  import java.util.Map;
53  import java.util.Random;
54  import java.util.concurrent.ExecutorService;
55  import java.util.concurrent.Executors;
56  import java.util.concurrent.atomic.AtomicInteger;
57  
58  import javax.security.sasl.Sasl;
59  import javax.security.sasl.SaslException;
60  import javax.security.sasl.SaslServer;
61  
62  import org.apache.commons.logging.Log;
63  import org.apache.commons.logging.LogFactory;
64  import org.apache.hadoop.hbase.classification.InterfaceAudience;
65  import org.apache.hadoop.hbase.classification.InterfaceStability;
66  import org.apache.hadoop.conf.Configuration;
67  import org.apache.hadoop.hbase.CellScanner;
68  import org.apache.hadoop.hbase.DoNotRetryIOException;
69  import org.apache.hadoop.hbase.HBaseInterfaceAudience;
70  import org.apache.hadoop.hbase.HConstants;
71  import org.apache.hadoop.hbase.HRegionInfo;
72  import org.apache.hadoop.hbase.Server;
73  import org.apache.hadoop.hbase.TableName;
74  import org.apache.hadoop.hbase.client.Operation;
75  import org.apache.hadoop.hbase.codec.Codec;
76  import org.apache.hadoop.hbase.exceptions.RegionMovedException;
77  import org.apache.hadoop.hbase.io.ByteBufferOutputStream;
78  import org.apache.hadoop.hbase.io.BoundedByteBufferPool;
79  import org.apache.hadoop.hbase.monitoring.MonitoredRPCHandler;
80  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
81  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.CellBlockMeta;
82  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ConnectionHeader;
83  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ExceptionResponse;
84  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.RequestHeader;
85  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.ResponseHeader;
86  import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;
87  import org.apache.hadoop.hbase.regionserver.HRegionServer;
88  import org.apache.hadoop.hbase.security.AccessDeniedException;
89  import org.apache.hadoop.hbase.security.AuthMethod;
90  import org.apache.hadoop.hbase.security.HBasePolicyProvider;
91  import org.apache.hadoop.hbase.security.HBaseSaslRpcServer;
92  import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslDigestCallbackHandler;
93  import org.apache.hadoop.hbase.security.HBaseSaslRpcServer.SaslGssCallbackHandler;
94  import org.apache.hadoop.hbase.security.SaslStatus;
95  import org.apache.hadoop.hbase.security.SaslUtil;
96  import org.apache.hadoop.hbase.security.User;
97  import org.apache.hadoop.hbase.security.UserProvider;
98  import org.apache.hadoop.hbase.security.token.AuthenticationTokenSecretManager;
99  import org.apache.hadoop.hbase.util.Bytes;
100 import org.apache.hadoop.hbase.util.Pair;
101 import org.apache.hadoop.io.BytesWritable;
102 import org.apache.hadoop.io.IntWritable;
103 import org.apache.hadoop.io.Writable;
104 import org.apache.hadoop.io.WritableUtils;
105 import org.apache.hadoop.io.compress.CompressionCodec;
106 import org.apache.hadoop.security.UserGroupInformation;
107 import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
108 import org.apache.hadoop.security.authorize.AuthorizationException;
109 import org.apache.hadoop.security.authorize.PolicyProvider;
110 import org.apache.hadoop.security.authorize.ProxyUsers;
111 import org.apache.hadoop.security.authorize.ServiceAuthorizationManager;
112 import org.apache.hadoop.security.token.SecretManager;
113 import org.apache.hadoop.security.token.SecretManager.InvalidToken;
114 import org.apache.hadoop.security.token.TokenIdentifier;
115 import org.apache.hadoop.util.StringUtils;
116 import org.cliffc.high_scale_lib.Counter;
117 import org.cloudera.htrace.TraceInfo;
118 import org.codehaus.jackson.map.ObjectMapper;
119 
120 import com.google.common.util.concurrent.ThreadFactoryBuilder;
121 import com.google.protobuf.BlockingService;
122 import com.google.protobuf.CodedInputStream;
123 import com.google.protobuf.Descriptors.MethodDescriptor;
124 import com.google.protobuf.Message;
125 import com.google.protobuf.ServiceException;
126 import com.google.protobuf.TextFormat;
127 // Uses Writables doing sasl
128 
129 /**
130  * An RPC server that hosts protobuf described Services.
131  *
132  * An RpcServer instance has a Listener that hosts the socket.  Listener has fixed number
133  * of Readers in an ExecutorPool, 10 by default.  The Listener does an accept and then
134  * round robin a Reader is chosen to do the read.  The reader is registered on Selector.  Read does
135  * total read off the channel and the parse from which it makes a Call.  The call is wrapped in a
136  * CallRunner and passed to the scheduler to be run.  Reader goes back to see if more to be done
137  * and loops till done.
138  *
139  * <p>Scheduler can be variously implemented but default simple scheduler has handlers to which it
140  * has given the queues into which calls (i.e. CallRunner instances) are inserted.  Handlers run
141  * taking from the queue.  They run the CallRunner#run method on each item gotten from queue
142  * and keep taking while the server is up.
143  *
144  * CallRunner#run executes the call.  When done, asks the included Call to put itself on new
145  * queue for {@link Responder} to pull from and return result to client.
146  *
147  * @see RpcClient
148  */
149 @InterfaceAudience.LimitedPrivate({HBaseInterfaceAudience.COPROC, HBaseInterfaceAudience.PHOENIX})
150 @InterfaceStability.Evolving
151 public class RpcServer implements RpcServerInterface {
152   // The logging package is deliberately outside of standard o.a.h.h package so it is not on
153   // by default.
154   public static final Log LOG = LogFactory.getLog("org.apache.hadoop.ipc.RpcServer");
155   private static final CallQueueTooBigException CALL_QUEUE_TOO_BIG_EXCEPTION
156       = new CallQueueTooBigException();
157 
158   private final boolean authorize;
159   private boolean isSecurityEnabled;
160 
161   public static final byte CURRENT_VERSION = 0;
162 
163   /**
164    * How many calls/handler are allowed in the queue.
165    */
166   static final int DEFAULT_MAX_CALLQUEUE_LENGTH_PER_HANDLER = 10;
167 
168   /**
169    * The maximum size that we can hold in the RPC queue
170    */
171   private static final int DEFAULT_MAX_CALLQUEUE_SIZE = 1024 * 1024 * 1024;
172 
173   static final int BUFFER_INITIAL_SIZE = 1024;
174 
175   private static final String WARN_DELAYED_CALLS = "hbase.ipc.warn.delayedrpc.number";
176 
177   private static final int DEFAULT_WARN_DELAYED_CALLS = 1000;
178 
179   private final int warnDelayedCalls;
180 
181   private AtomicInteger delayedCalls;
182   private final IPCUtil ipcUtil;
183 
184   private static final String AUTH_FAILED_FOR = "Auth failed for ";
185   private static final String AUTH_SUCCESSFUL_FOR = "Auth successful for ";
186   private static final Log AUDITLOG = LogFactory.getLog("SecurityLogger." +
187     Server.class.getName());
188   protected SecretManager<TokenIdentifier> secretManager;
189   protected ServiceAuthorizationManager authManager;
190 
191   /** This is set to Call object before Handler invokes an RPC and ybdie
192    * after the call returns.
193    */
194   protected static final ThreadLocal<Call> CurCall = new ThreadLocal<Call>();
195 
196   /** Keeps MonitoredRPCHandler per handler thread. */
197   static final ThreadLocal<MonitoredRPCHandler> MONITORED_RPC
198       = new ThreadLocal<MonitoredRPCHandler>();
199 
200   protected final InetSocketAddress isa;
201   protected int port;                             // port we listen on
202   private int readThreads;                        // number of read threads
203   protected int maxIdleTime;                      // the maximum idle time after
204                                                   // which a client may be
205                                                   // disconnected
206   protected int thresholdIdleConnections;         // the number of idle
207                                                   // connections after which we
208                                                   // will start cleaning up idle
209                                                   // connections
210   int maxConnectionsToNuke;                       // the max number of
211                                                   // connections to nuke
212                                                   // during a cleanup
213 
214   protected MetricsHBaseServer metrics;
215 
216   protected final Configuration conf;
217 
218   private int maxQueueSize;
219   protected int socketSendBufferSize;
220   protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
221   protected final boolean tcpKeepAlive; // if T then use keepalives
222   protected final long purgeTimeout;    // in milliseconds
223 
224   /**
225    * This flag is used to indicate to sub threads when they should go down.  When we call
226    * {@link #startThreads()}, all threads started will consult this flag on whether they should
227    * keep going.  It is set to false when {@link #stop()} is called.
228    */
229   volatile boolean running = true;
230 
231   /**
232    * This flag is set to true after all threads are up and 'running' and the server is then opened
233    * for business by the calle to {@link #openServer()}.
234    */
235   volatile boolean started = false;
236 
237   /**
238    * This is a running count of the size of all outstanding calls by size.
239    */
240   protected final Counter callQueueSize = new Counter();
241 
242   protected final List<Connection> connectionList =
243     Collections.synchronizedList(new LinkedList<Connection>());
244   //maintain a list
245   //of client connections
246   private Listener listener = null;
247   protected Responder responder = null;
248   protected AuthenticationTokenSecretManager authTokenSecretMgr = null;
249   protected int numConnections = 0;
250 
251   protected HBaseRPCErrorHandler errorHandler = null;
252 
253   private static final String WARN_RESPONSE_TIME = "hbase.ipc.warn.response.time";
254   private static final String WARN_RESPONSE_SIZE = "hbase.ipc.warn.response.size";
255 
256   /** Default value for above params */
257   private static final int DEFAULT_WARN_RESPONSE_TIME = 10000; // milliseconds
258   private static final int DEFAULT_WARN_RESPONSE_SIZE = 100 * 1024 * 1024;
259 
260   private static final ObjectMapper MAPPER = new ObjectMapper();
261 
262   private final int warnResponseTime;
263   private final int warnResponseSize;
264   private final Object serverInstance;
265   private final List<BlockingServiceAndInterface> services;
266 
267   private final RpcScheduler scheduler;
268 
269   private UserProvider userProvider;
270 
271   private final BoundedByteBufferPool reservoir;
272 
273 
274   /**
275    * Datastructure that holds all necessary to a method invocation and then afterward, carries
276    * the result.
277    */
278   class Call implements RpcCallContext {
279     protected int id;                             // the client's call id
280     protected BlockingService service;
281     protected MethodDescriptor md;
282     protected RequestHeader header;
283     protected Message param;                      // the parameter passed
284     // Optional cell data passed outside of protobufs.
285     protected CellScanner cellScanner;
286     protected Connection connection;              // connection to client
287     protected long timestamp;      // the time received when response is null
288                                    // the time served when response is not null
289     /**
290      * Chain of buffers to send as response.
291      */
292     protected BufferChain response;
293     protected boolean delayResponse;
294     protected Responder responder;
295     protected boolean delayReturnValue;           // if the return value should be
296                                                   // set at call completion
297     protected long size;                          // size of current call
298     protected boolean isError;
299     protected TraceInfo tinfo;
300     private ByteBuffer cellBlock = null;
301 
302     private User user;
303     private InetAddress remoteAddress;
304 
305     /**
306      * Deprecated, do not use
307      */
308     @Deprecated
309     Call(int id, final BlockingService service, final MethodDescriptor md, RequestHeader header,
310       Message param, CellScanner cellScanner, Connection connection, Responder responder,
311       long size, TraceInfo tinfo) {
312       this(id, service, md, header, param, cellScanner, connection, responder, size, tinfo, null);
313     }
314 
315     Call(int id, final BlockingService service, final MethodDescriptor md, RequestHeader header,
316          Message param, CellScanner cellScanner, Connection connection, Responder responder,
317          long size, TraceInfo tinfo, InetAddress remoteAddress) {
318       this.id = id;
319       this.service = service;
320       this.md = md;
321       this.header = header;
322       this.param = param;
323       this.cellScanner = cellScanner;
324       this.connection = connection;
325       this.timestamp = System.currentTimeMillis();
326       this.response = null;
327       this.delayResponse = false;
328       this.responder = responder;
329       this.isError = false;
330       this.size = size;
331       this.tinfo = tinfo;
332       if (connection != null && connection.user != null) {
333         this.user = userProvider.create(connection.user);
334       } else {
335         this.user = null;
336       }
337       this.remoteAddress = remoteAddress;
338     }
339 
340     /**
341      * Call is done. Execution happened and we returned results to client. It is now safe to
342      * cleanup.
343      */
344     void done() {
345       if (this.cellBlock != null) {
346         // Return buffer to reservoir now we are done with it.
347         reservoir.putBuffer(this.cellBlock);
348         this.cellBlock = null;
349       }
350     }
351 
352     @Override
353     public String toString() {
354       return toShortString() + " param: " +
355         (this.param != null? ProtobufUtil.getShortTextFormat(this.param): "") +
356         " connection: " + connection.toString();
357     }
358 
359     protected RequestHeader getHeader() {
360       return this.header;
361     }
362 
363     @Override
364     public User getRequestUser() {
365       return user;
366     }
367 
368     @Override
369     public String getRequestUserName() {
370       User user = getRequestUser();
371       return user == null? null: user.getShortName();
372     }
373 
374     @Override
375     public InetAddress getRemoteAddress() {
376       return remoteAddress;
377     }
378 
379     /*
380      * Short string representation without param info because param itself could be huge depends on
381      * the payload of a command
382      */
383     String toShortString() {
384       String serviceName = this.connection.service != null?
385         this.connection.service.getDescriptorForType().getName() : "null";
386       StringBuilder sb = new StringBuilder();
387       sb.append("callId: ");
388       sb.append(this.id);
389       sb.append(" service: ");
390       sb.append(serviceName);
391       sb.append(" methodName: ");
392       sb.append((this.md != null) ? this.md.getName() : "");
393       sb.append(" size: ");
394       sb.append(StringUtils.humanReadableInt(this.size));
395       sb.append(" connection: ");
396       sb.append(connection.toString());
397       return sb.toString();
398     }
399 
400     String toTraceString() {
401       String serviceName = this.connection.service != null ?
402                            this.connection.service.getDescriptorForType().getName() : "";
403       String methodName = (this.md != null) ? this.md.getName() : "";
404       String result = serviceName + "." + methodName;
405       return result;
406     }
407 
408     protected synchronized void setSaslTokenResponse(ByteBuffer response) {
409       this.response = new BufferChain(response);
410     }
411 
412     protected synchronized void setResponse(Object m, final CellScanner cells,
413         Throwable t, String errorMsg) {
414       if (this.isError) return;
415       if (t != null) this.isError = true;
416       BufferChain bc = null;
417       try {
418         ResponseHeader.Builder headerBuilder = ResponseHeader.newBuilder();
419         // Presume it a pb Message.  Could be null.
420         Message result = (Message)m;
421         // Call id.
422         headerBuilder.setCallId(this.id);
423         if (t != null) {
424           ExceptionResponse.Builder exceptionBuilder = ExceptionResponse.newBuilder();
425           exceptionBuilder.setExceptionClassName(t.getClass().getName());
426           exceptionBuilder.setStackTrace(errorMsg);
427           exceptionBuilder.setDoNotRetry(t instanceof DoNotRetryIOException);
428           if (t instanceof RegionMovedException) {
429             // Special casing for this exception.  This is only one carrying a payload.
430             // Do this instead of build a generic system for allowing exceptions carry
431             // any kind of payload.
432             RegionMovedException rme = (RegionMovedException)t;
433             exceptionBuilder.setHostname(rme.getHostname());
434             exceptionBuilder.setPort(rme.getPort());
435           }
436           // Set the exception as the result of the method invocation.
437           headerBuilder.setException(exceptionBuilder.build());
438         }
439         // Pass reservoir to buildCellBlock. Keep reference to returne so can add it back to the
440         // reservoir when finished. This is hacky and the hack is not contained but benefits are
441         // high when we can avoid a big buffer allocation on each rpc.
442         this.cellBlock = ipcUtil.buildCellBlock(this.connection.codec,
443           this.connection.compressionCodec, cells, reservoir);
444         if (this.cellBlock != null) {
445           CellBlockMeta.Builder cellBlockBuilder = CellBlockMeta.newBuilder();
446           // Presumes the cellBlock bytebuffer has been flipped so limit has total size in it.
447           cellBlockBuilder.setLength(this.cellBlock.limit());
448           headerBuilder.setCellBlockMeta(cellBlockBuilder.build());
449         }
450         Message header = headerBuilder.build();
451 
452         // Organize the response as a set of bytebuffers rather than collect it all together inside
453         // one big byte array; save on allocations.
454         ByteBuffer bbHeader = IPCUtil.getDelimitedMessageAsByteBuffer(header);
455         ByteBuffer bbResult = IPCUtil.getDelimitedMessageAsByteBuffer(result);
456         int totalSize = bbHeader.capacity() + (bbResult == null? 0: bbResult.limit()) +
457           (this.cellBlock == null? 0: this.cellBlock.limit());
458         ByteBuffer bbTotalSize = ByteBuffer.wrap(Bytes.toBytes(totalSize));
459         bc = new BufferChain(bbTotalSize, bbHeader, bbResult, this.cellBlock);
460         if (connection.useWrap) {
461           bc = wrapWithSasl(bc);
462         }
463       } catch (IOException e) {
464         LOG.warn("Exception while creating response " + e);
465       }
466       this.response = bc;
467     }
468 
469     private BufferChain wrapWithSasl(BufferChain bc)
470         throws IOException {
471       if (bc == null) return bc;
472       if (!this.connection.useSasl) return bc;
473       // Looks like no way around this; saslserver wants a byte array.  I have to make it one.
474       // THIS IS A BIG UGLY COPY.
475       byte [] responseBytes = bc.getBytes();
476       byte [] token;
477       // synchronization may be needed since there can be multiple Handler
478       // threads using saslServer to wrap responses.
479       synchronized (connection.saslServer) {
480         token = connection.saslServer.wrap(responseBytes, 0, responseBytes.length);
481       }
482       if (LOG.isDebugEnabled())
483         LOG.debug("Adding saslServer wrapped token of size " + token.length
484             + " as call response.");
485 
486       ByteBuffer bbTokenLength = ByteBuffer.wrap(Bytes.toBytes(token.length));
487       ByteBuffer bbTokenBytes = ByteBuffer.wrap(token);
488       return new BufferChain(bbTokenLength, bbTokenBytes);
489     }
490 
491     @Override
492     public synchronized void endDelay(Object result) throws IOException {
493       assert this.delayResponse;
494       assert this.delayReturnValue || result == null;
495       this.delayResponse = false;
496       delayedCalls.decrementAndGet();
497       if (this.delayReturnValue) {
498         this.setResponse(result, null, null, null);
499       }
500       this.responder.doRespond(this);
501     }
502 
503     @Override
504     public synchronized void endDelay() throws IOException {
505       this.endDelay(null);
506     }
507 
508     @Override
509     public synchronized void startDelay(boolean delayReturnValue) {
510       assert !this.delayResponse;
511       this.delayResponse = true;
512       this.delayReturnValue = delayReturnValue;
513       int numDelayed = delayedCalls.incrementAndGet();
514       if (numDelayed > warnDelayedCalls) {
515         LOG.warn("Too many delayed calls: limit " + warnDelayedCalls + " current " + numDelayed);
516       }
517     }
518 
519     @Override
520     public synchronized void endDelayThrowing(Throwable t) throws IOException {
521       this.setResponse(null, null, t, StringUtils.stringifyException(t));
522       this.delayResponse = false;
523       this.sendResponseIfReady();
524     }
525 
526     @Override
527     public synchronized boolean isDelayed() {
528       return this.delayResponse;
529     }
530 
531     @Override
532     public synchronized boolean isReturnValueDelayed() {
533       return this.delayReturnValue;
534     }
535 
536     @Override
537     public boolean isClientCellBlockSupport() {
538       return this.connection != null && this.connection.codec != null;
539     }
540 
541     @Override
542     public long disconnectSince() {
543       if (!connection.channel.isOpen()) {
544         return System.currentTimeMillis() - timestamp;
545       } else {
546         return -1L;
547       }
548     }
549 
550     public long getSize() {
551       return this.size;
552     }
553 
554     /**
555      * If we have a response, and delay is not set, then respond
556      * immediately.  Otherwise, do not respond to client.  This is
557      * called by the RPC code in the context of the Handler thread.
558      */
559     public synchronized void sendResponseIfReady() throws IOException {
560       if (!this.delayResponse) {
561         this.responder.doRespond(this);
562       }
563     }
564   }
565 
566   /** Listens on the socket. Creates jobs for the handler threads*/
567   private class Listener extends Thread {
568 
569     private ServerSocketChannel acceptChannel = null; //the accept channel
570     private Selector selector = null; //the selector that we use for the server
571     private Reader[] readers = null;
572     private int currentReader = 0;
573     private Random rand = new Random();
574     private long lastCleanupRunTime = 0; //the last time when a cleanup connec-
575                                          //-tion (for idle connections) ran
576     private long cleanupInterval = 10000; //the minimum interval between
577                                           //two cleanup runs
578     private int backlogLength = conf.getInt("hbase.ipc.server.listen.queue.size",
579       conf.getInt("ipc.server.listen.queue.size", 128));
580 
581     private ExecutorService readPool;
582 
583     public Listener(final String name) throws IOException {
584       super(name);
585       // Create a new server socket and set to non blocking mode
586       acceptChannel = ServerSocketChannel.open();
587       acceptChannel.configureBlocking(false);
588 
589       // Bind the server socket to the local host and port
590       bind(acceptChannel.socket(), isa, backlogLength);
591       port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
592       // create a selector;
593       selector= Selector.open();
594 
595       readers = new Reader[readThreads];
596       readPool = Executors.newFixedThreadPool(readThreads,
597         new ThreadFactoryBuilder().setNameFormat(
598           "RpcServer.reader=%d,port=" + port).setDaemon(true).build());
599       for (int i = 0; i < readThreads; ++i) {
600         Reader reader = new Reader();
601         readers[i] = reader;
602         readPool.execute(reader);
603       }
604       LOG.info(getName() + ": started " + readThreads + " reader(s).");
605 
606       // Register accepts on the server socket with the selector.
607       acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
608       this.setName("RpcServer.listener,port=" + port);
609       this.setDaemon(true);
610     }
611 
612 
613     private class Reader implements Runnable {
614       private volatile boolean adding = false;
615       private final Selector readSelector;
616 
617       Reader() throws IOException {
618         this.readSelector = Selector.open();
619       }
620       public void run() {
621         try {
622           doRunLoop();
623         } finally {
624           try {
625             readSelector.close();
626           } catch (IOException ioe) {
627             LOG.error(getName() + ": error closing read selector in " + getName(), ioe);
628           }
629         }
630       }
631 
632       private synchronized void doRunLoop() {
633         while (running) {
634           SelectionKey key = null;
635           try {
636             readSelector.select();
637             while (adding) {
638               this.wait(1000);
639             }
640 
641             Iterator<SelectionKey> iter = readSelector.selectedKeys().iterator();
642             while (iter.hasNext()) {
643               key = iter.next();
644               iter.remove();
645               if (key.isValid()) {
646                 if (key.isReadable()) {
647                   doRead(key);
648                 }
649               }
650               key = null;
651             }
652           } catch (InterruptedException e) {
653             if (running) {                      // unexpected -- log it
654               LOG.info(getName() + ": unexpectedly interrupted: " +
655                 StringUtils.stringifyException(e));
656             }
657           } catch (IOException ex) {
658             LOG.error(getName() + ": error in Reader", ex);
659           }
660         }
661       }
662 
663       /**
664        * This gets reader into the state that waits for the new channel
665        * to be registered with readSelector. If it was waiting in select()
666        * the thread will be woken up, otherwise whenever select() is called
667        * it will return even if there is nothing to read and wait
668        * in while(adding) for finishAdd call
669        */
670       public void startAdd() {
671         adding = true;
672         readSelector.wakeup();
673       }
674 
675       public synchronized SelectionKey registerChannel(SocketChannel channel)
676         throws IOException {
677         return channel.register(readSelector, SelectionKey.OP_READ);
678       }
679 
680       public synchronized void finishAdd() {
681         adding = false;
682         this.notify();
683       }
684     }
685 
686     /** cleanup connections from connectionList. Choose a random range
687      * to scan and also have a limit on the number of the connections
688      * that will be cleanedup per run. The criteria for cleanup is the time
689      * for which the connection was idle. If 'force' is true then all
690      * connections will be looked at for the cleanup.
691      * @param force all connections will be looked at for cleanup
692      */
693     private void cleanupConnections(boolean force) {
694       if (force || numConnections > thresholdIdleConnections) {
695         long currentTime = System.currentTimeMillis();
696         if (!force && (currentTime - lastCleanupRunTime) < cleanupInterval) {
697           return;
698         }
699         int start = 0;
700         int end = numConnections - 1;
701         if (!force) {
702           start = rand.nextInt() % numConnections;
703           end = rand.nextInt() % numConnections;
704           int temp;
705           if (end < start) {
706             temp = start;
707             start = end;
708             end = temp;
709           }
710         }
711         int i = start;
712         int numNuked = 0;
713         while (i <= end) {
714           Connection c;
715           synchronized (connectionList) {
716             try {
717               c = connectionList.get(i);
718             } catch (Exception e) {return;}
719           }
720           if (c.timedOut(currentTime)) {
721             if (LOG.isDebugEnabled())
722               LOG.debug(getName() + ": disconnecting client " + c.getHostAddress());
723             closeConnection(c);
724             numNuked++;
725             end--;
726             //noinspection UnusedAssignment
727             c = null;
728             if (!force && numNuked == maxConnectionsToNuke) break;
729           }
730           else i++;
731         }
732         lastCleanupRunTime = System.currentTimeMillis();
733       }
734     }
735 
736     @Override
737     public void run() {
738       LOG.info(getName() + ": starting");
739       while (running) {
740         SelectionKey key = null;
741         try {
742           selector.select(); // FindBugs IS2_INCONSISTENT_SYNC
743           Iterator<SelectionKey> iter = selector.selectedKeys().iterator();
744           while (iter.hasNext()) {
745             key = iter.next();
746             iter.remove();
747             try {
748               if (key.isValid()) {
749                 if (key.isAcceptable())
750                   doAccept(key);
751               }
752             } catch (IOException ignored) {
753             }
754             key = null;
755           }
756         } catch (OutOfMemoryError e) {
757           if (errorHandler != null) {
758             if (errorHandler.checkOOME(e)) {
759               LOG.info(getName() + ": exiting on OutOfMemoryError");
760               closeCurrentConnection(key, e);
761               cleanupConnections(true);
762               return;
763             }
764           } else {
765             // we can run out of memory if we have too many threads
766             // log the event and sleep for a minute and give
767             // some thread(s) a chance to finish
768             LOG.warn(getName() + ": OutOfMemoryError in server select", e);
769             closeCurrentConnection(key, e);
770             cleanupConnections(true);
771             try { Thread.sleep(60000); } catch (Exception ignored) {}
772           }
773         } catch (Exception e) {
774           closeCurrentConnection(key, e);
775         }
776         cleanupConnections(false);
777       }
778       LOG.info(getName() + ": stopping");
779 
780       synchronized (this) {
781         try {
782           acceptChannel.close();
783           selector.close();
784         } catch (IOException ignored) { }
785 
786         selector= null;
787         acceptChannel= null;
788 
789         // clean up all connections
790         while (!connectionList.isEmpty()) {
791           closeConnection(connectionList.remove(0));
792         }
793       }
794     }
795 
796     private void closeCurrentConnection(SelectionKey key, Throwable e) {
797       if (key != null) {
798         Connection c = (Connection)key.attachment();
799         if (c != null) {
800           if (LOG.isDebugEnabled()) {
801             LOG.debug(getName() + ": disconnecting client " + c.getHostAddress() +
802                 (e != null ? " on error " + e.getMessage() : ""));
803           }
804           closeConnection(c);
805           key.attach(null);
806         }
807       }
808     }
809 
810     InetSocketAddress getAddress() {
811       return (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();
812     }
813 
814     void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {
815       Connection c;
816       ServerSocketChannel server = (ServerSocketChannel) key.channel();
817 
818       SocketChannel channel;
819       while ((channel = server.accept()) != null) {
820         try {
821           channel.configureBlocking(false);
822           channel.socket().setTcpNoDelay(tcpNoDelay);
823           channel.socket().setKeepAlive(tcpKeepAlive);
824         } catch (IOException ioe) {
825           channel.close();
826           throw ioe;
827         }
828 
829         Reader reader = getReader();
830         try {
831           reader.startAdd();
832           SelectionKey readKey = reader.registerChannel(channel);
833           c = getConnection(channel, System.currentTimeMillis());
834           readKey.attach(c);
835           synchronized (connectionList) {
836             connectionList.add(numConnections, c);
837             numConnections++;
838           }
839           if (LOG.isDebugEnabled())
840             LOG.debug(getName() + ": connection from " + c.toString() +
841                 "; # active connections: " + numConnections);
842         } finally {
843           reader.finishAdd();
844         }
845       }
846     }
847 
848     void doRead(SelectionKey key) throws InterruptedException {
849       int count = 0;
850       Connection c = (Connection)key.attachment();
851       if (c == null) {
852         return;
853       }
854       c.setLastContact(System.currentTimeMillis());
855       try {
856         count = c.readAndProcess();
857       } catch (InterruptedException ieo) {
858         throw ieo;
859       } catch (Exception e) {
860         LOG.warn(getName() + ": count of bytes read: " + count, e);
861         count = -1; //so that the (count < 0) block is executed
862       }
863       if (count < 0) {
864         if (LOG.isDebugEnabled()) {
865           LOG.debug(getName() + ": DISCONNECTING client " + c.toString() +
866             " because read count=" + count +
867             ". Number of active connections: " + numConnections);
868         }
869         closeConnection(c);
870         // c = null;
871       } else {
872         c.setLastContact(System.currentTimeMillis());
873       }
874     }
875 
876     synchronized void doStop() {
877       if (selector != null) {
878         selector.wakeup();
879         Thread.yield();
880       }
881       if (acceptChannel != null) {
882         try {
883           acceptChannel.socket().close();
884         } catch (IOException e) {
885           LOG.info(getName() + ": exception in closing listener socket. " + e);
886         }
887       }
888       readPool.shutdownNow();
889     }
890 
891     // The method that will return the next reader to work with
892     // Simplistic implementation of round robin for now
893     Reader getReader() {
894       currentReader = (currentReader + 1) % readers.length;
895       return readers[currentReader];
896     }
897   }
898 
899   // Sends responses of RPC back to clients.
900   protected class Responder extends Thread {
901     private final Selector writeSelector;
902     private int pending;         // connections waiting to register
903 
904     Responder() throws IOException {
905       this.setName("RpcServer.responder");
906       this.setDaemon(true);
907       writeSelector = Selector.open(); // create a selector
908       pending = 0;
909     }
910 
911     @Override
912     public void run() {
913       LOG.info(getName() + ": starting");
914       try {
915         doRunLoop();
916       } finally {
917         LOG.info(getName() + ": stopping");
918         try {
919           writeSelector.close();
920         } catch (IOException ioe) {
921           LOG.error(getName() + ": couldn't close write selector", ioe);
922         }
923       }
924     }
925 
926     private void doRunLoop() {
927       long lastPurgeTime = 0;   // last check for old calls.
928 
929       while (running) {
930         try {
931           waitPending();     // If a channel is being registered, wait.
932           writeSelector.select(purgeTimeout);
933           Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();
934           while (iter.hasNext()) {
935             SelectionKey key = iter.next();
936             iter.remove();
937             try {
938               if (key.isValid() && key.isWritable()) {
939                   doAsyncWrite(key);
940               }
941             } catch (IOException e) {
942               LOG.info(getName() + ": asyncWrite", e);
943             }
944           }
945           long now = System.currentTimeMillis();
946           if (now < lastPurgeTime + purgeTimeout) {
947             continue;
948           }
949           lastPurgeTime = now;
950           //
951           // If there were some calls that have not been sent out for a
952           // long time, discard them.
953           //
954           if (LOG.isDebugEnabled()) LOG.debug(getName() + ": checking for old call responses.");
955           ArrayList<Call> calls;
956 
957           // get the list of channels from list of keys.
958           synchronized (writeSelector.keys()) {
959             calls = new ArrayList<Call>(writeSelector.keys().size());
960             iter = writeSelector.keys().iterator();
961             while (iter.hasNext()) {
962               SelectionKey key = iter.next();
963               Call call = (Call)key.attachment();
964               if (call != null && key.channel() == call.connection.channel) {
965                 calls.add(call);
966               }
967             }
968           }
969 
970           for(Call call : calls) {
971             try {
972               doPurge(call, now);
973             } catch (IOException e) {
974               LOG.warn(getName() + ": error in purging old calls " + e);
975             }
976           }
977         } catch (OutOfMemoryError e) {
978           if (errorHandler != null) {
979             if (errorHandler.checkOOME(e)) {
980               LOG.info(getName() + ": exiting on OutOfMemoryError");
981               return;
982             }
983           } else {
984             //
985             // we can run out of memory if we have too many threads
986             // log the event and sleep for a minute and give
987             // some thread(s) a chance to finish
988             //
989             LOG.warn(getName() + ": OutOfMemoryError in server select", e);
990             try { Thread.sleep(60000); } catch (Exception ignored) {}
991           }
992         } catch (Exception e) {
993           LOG.warn(getName() + ": exception in Responder " +
994                    StringUtils.stringifyException(e));
995         }
996       }
997       LOG.info(getName() + ": stopped");
998     }
999 
1000     private void doAsyncWrite(SelectionKey key) throws IOException {
1001       Call call = (Call)key.attachment();
1002       if (call == null) {
1003         return;
1004       }
1005       if (key.channel() != call.connection.channel) {
1006         throw new IOException("doAsyncWrite: bad channel");
1007       }
1008 
1009       synchronized(call.connection.responseQueue) {
1010         if (processResponse(call.connection.responseQueue, false)) {
1011           try {
1012             key.interestOps(0);
1013           } catch (CancelledKeyException e) {
1014             /* The Listener/reader might have closed the socket.
1015              * We don't explicitly cancel the key, so not sure if this will
1016              * ever fire.
1017              * This warning could be removed.
1018              */
1019             LOG.warn("Exception while changing ops : " + e);
1020           }
1021         }
1022       }
1023     }
1024 
1025     //
1026     // Remove calls that have been pending in the responseQueue
1027     // for a long time.
1028     //
1029     private void doPurge(Call call, long now) throws IOException {
1030       synchronized (call.connection.responseQueue) {
1031         Iterator<Call> iter = call.connection.responseQueue.listIterator(0);
1032         while (iter.hasNext()) {
1033           Call nextCall = iter.next();
1034           if (now > nextCall.timestamp + purgeTimeout) {
1035             closeConnection(nextCall.connection);
1036             break;
1037           }
1038         }
1039       }
1040     }
1041 
1042     // Processes one response. Returns true if there are no more pending
1043     // data for this channel.
1044     //
1045     private boolean processResponse(final LinkedList<Call> responseQueue, boolean inHandler)
1046     throws IOException {
1047       boolean error = true;
1048       boolean done = false;       // there is more data for this channel.
1049       int numElements;
1050       Call call = null;
1051       try {
1052         //noinspection SynchronizationOnLocalVariableOrMethodParameter
1053         synchronized (responseQueue) {
1054           //
1055           // If there are no items for this channel, then we are done
1056           //
1057           numElements = responseQueue.size();
1058           if (numElements == 0) {
1059             error = false;
1060             return true;              // no more data for this channel.
1061           }
1062           //
1063           // Extract the first call
1064           //
1065           call = responseQueue.removeFirst();
1066           SocketChannel channel = call.connection.channel;
1067           //
1068           // Send as much data as we can in the non-blocking fashion
1069           //
1070           long numBytes = channelWrite(channel, call.response);
1071           if (numBytes < 0) {
1072             return true;
1073           }
1074           if (!call.response.hasRemaining()) {
1075             call.connection.decRpcCount();
1076             //noinspection RedundantIfStatement
1077             if (numElements == 1) {    // last call fully processes.
1078               done = true;             // no more data for this channel.
1079             } else {
1080               done = false;            // more calls pending to be sent.
1081             }
1082             if (LOG.isDebugEnabled()) {
1083               LOG.debug(getName() + ": callId: " + call.id + " wrote " + numBytes + " bytes.");
1084             }
1085           } else {
1086             //
1087             // If we were unable to write the entire response out, then
1088             // insert in Selector queue.
1089             //
1090             call.connection.responseQueue.addFirst(call);
1091 
1092             if (inHandler) {
1093               // set the serve time when the response has to be sent later
1094               call.timestamp = System.currentTimeMillis();
1095               if (enqueueInSelector(call))
1096                 done = true;
1097             }
1098             if (LOG.isDebugEnabled()) {
1099               LOG.debug(getName() + call.toShortString() + " partially sent, wrote " +
1100                 numBytes + " bytes.");
1101             }
1102           }
1103           error = false;              // everything went off well
1104         }
1105       } finally {
1106         if (error && call != null) {
1107           LOG.warn(getName() + call.toShortString() + ": output error");
1108           done = true;               // error. no more data for this channel.
1109           closeConnection(call.connection);
1110         }
1111       }
1112       if (done) call.done();
1113       return done;
1114     }
1115 
1116     //
1117     // Enqueue for background thread to send responses out later.
1118     //
1119     private boolean enqueueInSelector(Call call) throws IOException {
1120       boolean done = false;
1121       incPending();
1122       try {
1123         // Wake up the thread blocked on select, only then can the call
1124         // to channel.register() complete.
1125         SocketChannel channel = call.connection.channel;
1126         writeSelector.wakeup();
1127         channel.register(writeSelector, SelectionKey.OP_WRITE, call);
1128       } catch (ClosedChannelException e) {
1129         //It's OK.  Channel might be closed else where.
1130         done = true;
1131       } finally {
1132         decPending();
1133       }
1134       return done;
1135     }
1136 
1137     //
1138     // Enqueue a response from the application.
1139     //
1140     void doRespond(Call call) throws IOException {
1141       // set the serve time when the response has to be sent later
1142       call.timestamp = System.currentTimeMillis();
1143 
1144       boolean doRegister = false;
1145       synchronized (call.connection.responseQueue) {
1146         call.connection.responseQueue.addLast(call);
1147         if (call.connection.responseQueue.size() == 1) {
1148           doRegister = !processResponse(call.connection.responseQueue, false);
1149         }
1150       }
1151       if (doRegister) {
1152         enqueueInSelector(call);
1153       }
1154     }
1155 
1156     private synchronized void incPending() {   // call waiting to be enqueued.
1157       pending++;
1158     }
1159 
1160     private synchronized void decPending() { // call done enqueueing.
1161       pending--;
1162       notify();
1163     }
1164 
1165     private synchronized void waitPending() throws InterruptedException {
1166       while (pending > 0) {
1167         wait();
1168       }
1169     }
1170   }
1171 
1172   @SuppressWarnings("serial")
1173   public static class CallQueueTooBigException extends IOException {
1174     CallQueueTooBigException() {
1175       super();
1176     }
1177   }
1178 
1179   /** Reads calls from a connection and queues them for handling. */
1180   @edu.umd.cs.findbugs.annotations.SuppressWarnings(
1181       value="VO_VOLATILE_INCREMENT",
1182       justification="False positive according to http://sourceforge.net/p/findbugs/bugs/1032/")
1183   public class Connection {
1184     // If initial preamble with version and magic has been read or not.
1185     private boolean connectionPreambleRead = false;
1186     // If the connection header has been read or not.
1187     private boolean connectionHeaderRead = false;
1188     protected SocketChannel channel;
1189     private ByteBuffer data;
1190     private ByteBuffer dataLengthBuffer;
1191     protected final LinkedList<Call> responseQueue;
1192     private Counter rpcCount = new Counter(); // number of outstanding rpcs
1193     private long lastContact;
1194     private InetAddress addr;
1195     protected Socket socket;
1196     // Cache the remote host & port info so that even if the socket is
1197     // disconnected, we can say where it used to connect to.
1198     protected String hostAddress;
1199     protected int remotePort;
1200     ConnectionHeader connectionHeader;
1201     /**
1202      * Codec the client asked use.
1203      */
1204     private Codec codec;
1205     /**
1206      * Compression codec the client asked us use.
1207      */
1208     private CompressionCodec compressionCodec;
1209     BlockingService service;
1210     protected UserGroupInformation user = null;
1211     private AuthMethod authMethod;
1212     private boolean saslContextEstablished;
1213     private boolean skipInitialSaslHandshake;
1214     private ByteBuffer unwrappedData;
1215     // When is this set?  FindBugs wants to know!  Says NP
1216     private ByteBuffer unwrappedDataLengthBuffer = ByteBuffer.allocate(4);
1217     boolean useSasl;
1218     SaslServer saslServer;
1219     private boolean useWrap = false;
1220     // Fake 'call' for failed authorization response
1221     private static final int AUTHROIZATION_FAILED_CALLID = -1;
1222     private final Call authFailedCall =
1223       new Call(AUTHROIZATION_FAILED_CALLID, this.service, null,
1224         null, null, null, this, null, 0, null, null);
1225     private ByteArrayOutputStream authFailedResponse =
1226         new ByteArrayOutputStream();
1227     // Fake 'call' for SASL context setup
1228     private static final int SASL_CALLID = -33;
1229     private final Call saslCall =
1230       new Call(SASL_CALLID, this.service, null, null, null, null, this, null, 0, null, null);
1231 
1232     public UserGroupInformation attemptingUser = null; // user name before auth
1233 
1234     public Connection(SocketChannel channel, long lastContact) {
1235       this.channel = channel;
1236       this.lastContact = lastContact;
1237       this.data = null;
1238       this.dataLengthBuffer = ByteBuffer.allocate(4);
1239       this.socket = channel.socket();
1240       this.addr = socket.getInetAddress();
1241       if (addr == null) {
1242         this.hostAddress = "*Unknown*";
1243       } else {
1244         this.hostAddress = addr.getHostAddress();
1245       }
1246       this.remotePort = socket.getPort();
1247       this.responseQueue = new LinkedList<Call>();
1248       if (socketSendBufferSize != 0) {
1249         try {
1250           socket.setSendBufferSize(socketSendBufferSize);
1251         } catch (IOException e) {
1252           LOG.warn("Connection: unable to set socket send buffer size to " +
1253                    socketSendBufferSize);
1254         }
1255       }
1256     }
1257 
1258     @Override
1259     public String toString() {
1260       return getHostAddress() + ":" + remotePort;
1261     }
1262 
1263     public String getHostAddress() {
1264       return hostAddress;
1265     }
1266 
1267     public InetAddress getHostInetAddress() {
1268       return addr;
1269     }
1270 
1271     public int getRemotePort() {
1272       return remotePort;
1273     }
1274 
1275     public void setLastContact(long lastContact) {
1276       this.lastContact = lastContact;
1277     }
1278 
1279     public long getLastContact() {
1280       return lastContact;
1281     }
1282 
1283     /* Return true if the connection has no outstanding rpc */
1284     private boolean isIdle() {
1285       return rpcCount.get() == 0;
1286     }
1287 
1288     /* Decrement the outstanding RPC count */
1289     protected void decRpcCount() {
1290       rpcCount.decrement();
1291     }
1292 
1293     /* Increment the outstanding RPC count */
1294     protected void incRpcCount() {
1295       rpcCount.increment();
1296     }
1297 
1298     protected boolean timedOut(long currentTime) {
1299       return isIdle() && currentTime - lastContact > maxIdleTime;
1300     }
1301 
1302     private UserGroupInformation getAuthorizedUgi(String authorizedId)
1303         throws IOException {
1304       if (authMethod == AuthMethod.DIGEST) {
1305         TokenIdentifier tokenId = HBaseSaslRpcServer.getIdentifier(authorizedId,
1306             secretManager);
1307         UserGroupInformation ugi = tokenId.getUser();
1308         if (ugi == null) {
1309           throw new AccessDeniedException(
1310               "Can't retrieve username from tokenIdentifier.");
1311         }
1312         ugi.addTokenIdentifier(tokenId);
1313         return ugi;
1314       } else {
1315         return UserGroupInformation.createRemoteUser(authorizedId);
1316       }
1317     }
1318 
1319     private void saslReadAndProcess(byte[] saslToken) throws IOException,
1320         InterruptedException {
1321       if (saslContextEstablished) {
1322         if (LOG.isDebugEnabled())
1323           LOG.debug("Have read input token of size " + saslToken.length
1324               + " for processing by saslServer.unwrap()");
1325 
1326         if (!useWrap) {
1327           processOneRpc(saslToken);
1328         } else {
1329           byte [] plaintextData = saslServer.unwrap(saslToken, 0, saslToken.length);
1330           processUnwrappedData(plaintextData);
1331         }
1332       } else {
1333         byte[] replyToken = null;
1334         try {
1335           if (saslServer == null) {
1336             switch (authMethod) {
1337             case DIGEST:
1338               if (secretManager == null) {
1339                 throw new AccessDeniedException(
1340                     "Server is not configured to do DIGEST authentication.");
1341               }
1342               saslServer = Sasl.createSaslServer(AuthMethod.DIGEST
1343                   .getMechanismName(), null, SaslUtil.SASL_DEFAULT_REALM,
1344                   SaslUtil.SASL_PROPS, new SaslDigestCallbackHandler(
1345                       secretManager, this));
1346               break;
1347             default:
1348               UserGroupInformation current = UserGroupInformation
1349               .getCurrentUser();
1350               String fullName = current.getUserName();
1351               if (LOG.isDebugEnabled()) {
1352                 LOG.debug("Kerberos principal name is " + fullName);
1353               }
1354               final String names[] = SaslUtil.splitKerberosName(fullName);
1355               if (names.length != 3) {
1356                 throw new AccessDeniedException(
1357                     "Kerberos principal name does NOT have the expected "
1358                         + "hostname part: " + fullName);
1359               }
1360               current.doAs(new PrivilegedExceptionAction<Object>() {
1361                 @Override
1362                 public Object run() throws SaslException {
1363                   saslServer = Sasl.createSaslServer(AuthMethod.KERBEROS
1364                       .getMechanismName(), names[0], names[1],
1365                       SaslUtil.SASL_PROPS, new SaslGssCallbackHandler());
1366                   return null;
1367                 }
1368               });
1369             }
1370             if (saslServer == null)
1371               throw new AccessDeniedException(
1372                   "Unable to find SASL server implementation for "
1373                       + authMethod.getMechanismName());
1374             if (LOG.isDebugEnabled()) {
1375               LOG.debug("Created SASL server with mechanism = " + authMethod.getMechanismName());
1376             }
1377           }
1378           if (LOG.isDebugEnabled()) {
1379             LOG.debug("Have read input token of size " + saslToken.length
1380                 + " for processing by saslServer.evaluateResponse()");
1381           }
1382           replyToken = saslServer.evaluateResponse(saslToken);
1383         } catch (IOException e) {
1384           IOException sendToClient = e;
1385           Throwable cause = e;
1386           while (cause != null) {
1387             if (cause instanceof InvalidToken) {
1388               sendToClient = (InvalidToken) cause;
1389               break;
1390             }
1391             cause = cause.getCause();
1392           }
1393           doRawSaslReply(SaslStatus.ERROR, null, sendToClient.getClass().getName(),
1394             sendToClient.getLocalizedMessage());
1395           metrics.authenticationFailure();
1396           String clientIP = this.toString();
1397           // attempting user could be null
1398           AUDITLOG.warn(AUTH_FAILED_FOR + clientIP + ":" + attemptingUser);
1399           throw e;
1400         }
1401         if (replyToken != null) {
1402           if (LOG.isDebugEnabled()) {
1403             LOG.debug("Will send token of size " + replyToken.length
1404                 + " from saslServer.");
1405           }
1406           doRawSaslReply(SaslStatus.SUCCESS, new BytesWritable(replyToken), null,
1407               null);
1408         }
1409         if (saslServer.isComplete()) {
1410           String qop = (String) saslServer.getNegotiatedProperty(Sasl.QOP);
1411           useWrap = qop != null && !"auth".equalsIgnoreCase(qop);
1412           user = getAuthorizedUgi(saslServer.getAuthorizationID());
1413           if (LOG.isDebugEnabled()) {
1414             LOG.debug("SASL server context established. Authenticated client: "
1415               + user + ". Negotiated QoP is "
1416               + saslServer.getNegotiatedProperty(Sasl.QOP));
1417           }
1418           metrics.authenticationSuccess();
1419           AUDITLOG.info(AUTH_SUCCESSFUL_FOR + user);
1420           saslContextEstablished = true;
1421         }
1422       }
1423     }
1424 
1425     /**
1426      * No protobuf encoding of raw sasl messages
1427      */
1428     private void doRawSaslReply(SaslStatus status, Writable rv,
1429         String errorClass, String error) throws IOException {
1430       ByteBufferOutputStream saslResponse = null;
1431       DataOutputStream out = null;
1432       try {
1433         // In my testing, have noticed that sasl messages are usually
1434         // in the ballpark of 100-200. That's why the initial capacity is 256.
1435         saslResponse = new ByteBufferOutputStream(256);
1436         out = new DataOutputStream(saslResponse);
1437         out.writeInt(status.state); // write status
1438         if (status == SaslStatus.SUCCESS) {
1439           rv.write(out);
1440         } else {
1441           WritableUtils.writeString(out, errorClass);
1442           WritableUtils.writeString(out, error);
1443         }
1444         saslCall.setSaslTokenResponse(saslResponse.getByteBuffer());
1445         saslCall.responder = responder;
1446         saslCall.sendResponseIfReady();
1447       } finally {
1448         if (saslResponse != null) {
1449           saslResponse.close();
1450         }
1451         if (out != null) {
1452           out.close();
1453         }
1454       }
1455     }
1456 
1457     private void disposeSasl() {
1458       if (saslServer != null) {
1459         try {
1460           saslServer.dispose();
1461           saslServer = null;
1462         } catch (SaslException ignored) {
1463           // Ignored. This is being disposed of anyway.
1464         }
1465       }
1466     }
1467 
1468     /**
1469      * Read off the wire.
1470      * @return Returns -1 if failure (and caller will close connection) else return how many
1471      * bytes were read and processed
1472      * @throws IOException
1473      * @throws InterruptedException
1474      */
1475     public int readAndProcess() throws IOException, InterruptedException {
1476       while (true) {
1477         // Try and read in an int.  If new connection, the int will hold the 'HBas' HEADER.  If it
1478         // does, read in the rest of the connection preamble, the version and the auth method.
1479         // Else it will be length of the data to read (or -1 if a ping).  We catch the integer
1480         // length into the 4-byte this.dataLengthBuffer.
1481         int count;
1482         if (this.dataLengthBuffer.remaining() > 0) {
1483           count = channelRead(channel, this.dataLengthBuffer);
1484           if (count < 0 || this.dataLengthBuffer.remaining() > 0) {
1485             return count;
1486           }
1487         }
1488         // If we have not read the connection setup preamble, look to see if that is on the wire.
1489         if (!connectionPreambleRead) {
1490           // Check for 'HBas' magic.
1491           this.dataLengthBuffer.flip();
1492           if (!HConstants.RPC_HEADER.equals(dataLengthBuffer)) {
1493             return doBadPreambleHandling("Expected HEADER=" +
1494               Bytes.toStringBinary(HConstants.RPC_HEADER.array()) +
1495               " but received HEADER=" + Bytes.toStringBinary(dataLengthBuffer.array()) +
1496               " from " + toString());
1497           }
1498           // Now read the next two bytes, the version and the auth to use.
1499           ByteBuffer versionAndAuthBytes = ByteBuffer.allocate(2);
1500           count = channelRead(channel, versionAndAuthBytes);
1501           if (count < 0 || versionAndAuthBytes.remaining() > 0) {
1502             return count;
1503           }
1504           int version = versionAndAuthBytes.get(0);
1505           byte authbyte = versionAndAuthBytes.get(1);
1506           this.authMethod = AuthMethod.valueOf(authbyte);
1507           if (version != CURRENT_VERSION) {
1508             String msg = getFatalConnectionString(version, authbyte);
1509             return doBadPreambleHandling(msg, new WrongVersionException(msg));
1510           }
1511           if (authMethod == null) {
1512             String msg = getFatalConnectionString(version, authbyte);
1513             return doBadPreambleHandling(msg, new BadAuthException(msg));
1514           }
1515           if (isSecurityEnabled && authMethod == AuthMethod.SIMPLE) {
1516             AccessDeniedException ae = new AccessDeniedException("Authentication is required");
1517             setupResponse(authFailedResponse, authFailedCall, ae, ae.getMessage());
1518             responder.doRespond(authFailedCall);
1519             throw ae;
1520           }
1521           if (!isSecurityEnabled && authMethod != AuthMethod.SIMPLE) {
1522             doRawSaslReply(SaslStatus.SUCCESS, new IntWritable(
1523                 SaslUtil.SWITCH_TO_SIMPLE_AUTH), null, null);
1524             authMethod = AuthMethod.SIMPLE;
1525             // client has already sent the initial Sasl message and we
1526             // should ignore it. Both client and server should fall back
1527             // to simple auth from now on.
1528             skipInitialSaslHandshake = true;
1529           }
1530           if (authMethod != AuthMethod.SIMPLE) {
1531             useSasl = true;
1532           }
1533           connectionPreambleRead = true;
1534           // Preamble checks out. Go around again to read actual connection header.
1535           dataLengthBuffer.clear();
1536           continue;
1537         }
1538         // We have read a length and we have read the preamble.  It is either the connection header
1539         // or it is a request.
1540         if (data == null) {
1541           dataLengthBuffer.flip();
1542           int dataLength = dataLengthBuffer.getInt();
1543           if (dataLength == RpcClient.PING_CALL_ID) {
1544             if (!useWrap) { //covers the !useSasl too
1545               dataLengthBuffer.clear();
1546               return 0;  //ping message
1547             }
1548           }
1549           if (dataLength < 0) {
1550             throw new IllegalArgumentException("Unexpected data length "
1551                 + dataLength + "!! from " + getHostAddress());
1552           }
1553           data = ByteBuffer.allocate(dataLength);
1554           incRpcCount();  // Increment the rpc count
1555         }
1556         count = channelRead(channel, data);
1557         if (count < 0) {
1558           return count;
1559         } else if (data.remaining() == 0) {
1560           dataLengthBuffer.clear();
1561           data.flip();
1562           if (skipInitialSaslHandshake) {
1563             data = null;
1564             skipInitialSaslHandshake = false;
1565             continue;
1566           }
1567           boolean headerRead = connectionHeaderRead;
1568           if (useSasl) {
1569             saslReadAndProcess(data.array());
1570           } else {
1571             processOneRpc(data.array());
1572           }
1573           this.data = null;
1574           if (!headerRead) {
1575             continue;
1576           }
1577         } else if (count > 0) {
1578           // We got some data and there is more to read still; go around again.
1579           if (LOG.isTraceEnabled()) LOG.trace("Continue to read rest of data " + data.remaining());
1580           continue;
1581         }
1582         return count;
1583       }
1584     }
1585 
1586     private String getFatalConnectionString(final int version, final byte authByte) {
1587       return "serverVersion=" + CURRENT_VERSION +
1588       ", clientVersion=" + version + ", authMethod=" + authByte +
1589       ", authSupported=" + (authMethod != null) + " from " + toString();
1590     }
1591 
1592     private int doBadPreambleHandling(final String msg) throws IOException {
1593       return doBadPreambleHandling(msg, new FatalConnectionException(msg));
1594     }
1595 
1596     private int doBadPreambleHandling(final String msg, final Exception e) throws IOException {
1597       LOG.warn(msg);
1598       Call fakeCall = new Call(-1, null, null, null, null, null, this, responder, -1, null, null);
1599       setupResponse(null, fakeCall, e, msg);
1600       responder.doRespond(fakeCall);
1601       // Returning -1 closes out the connection.
1602       return -1;
1603     }
1604 
1605     // Reads the connection header following version
1606     private void processConnectionHeader(byte[] buf) throws IOException {
1607       this.connectionHeader = ConnectionHeader.parseFrom(buf);
1608       String serviceName = connectionHeader.getServiceName();
1609       if (serviceName == null) throw new EmptyServiceNameException();
1610       this.service = getService(services, serviceName);
1611       if (this.service == null) throw new UnknownServiceException(serviceName);
1612       setupCellBlockCodecs(this.connectionHeader);
1613       UserGroupInformation protocolUser = createUser(connectionHeader);
1614       if (!useSasl) {
1615         user = protocolUser;
1616         if (user != null) {
1617           user.setAuthenticationMethod(AuthMethod.SIMPLE.authenticationMethod);
1618         }
1619       } else {
1620         // user is authenticated
1621         user.setAuthenticationMethod(authMethod.authenticationMethod);
1622         //Now we check if this is a proxy user case. If the protocol user is
1623         //different from the 'user', it is a proxy user scenario. However,
1624         //this is not allowed if user authenticated with DIGEST.
1625         if ((protocolUser != null)
1626             && (!protocolUser.getUserName().equals(user.getUserName()))) {
1627           if (authMethod == AuthMethod.DIGEST) {
1628             // Not allowed to doAs if token authentication is used
1629             throw new AccessDeniedException("Authenticated user (" + user
1630                 + ") doesn't match what the client claims to be ("
1631                 + protocolUser + ")");
1632           } else {
1633             // Effective user can be different from authenticated user
1634             // for simple auth or kerberos auth
1635             // The user is the real user. Now we create a proxy user
1636             UserGroupInformation realUser = user;
1637             user = UserGroupInformation.createProxyUser(protocolUser
1638                 .getUserName(), realUser);
1639             // Now the user is a proxy user, set Authentication method Proxy.
1640             user.setAuthenticationMethod(AuthenticationMethod.PROXY);
1641           }
1642         }
1643       }
1644       if (connectionHeader.hasVersionInfo()) {
1645         AUDITLOG.info("Connection from " + this.hostAddress + " port: " + this.remotePort
1646             + " with version info: "
1647             + TextFormat.shortDebugString(connectionHeader.getVersionInfo()));
1648       } else {
1649         AUDITLOG.info("Connection from " + this.hostAddress + " port: " + this.remotePort
1650             + " with unknown version info");
1651       }
1652     }
1653 
1654     /**
1655      * Set up cell block codecs
1656      * @param header
1657      * @throws FatalConnectionException
1658      */
1659     private void setupCellBlockCodecs(final ConnectionHeader header)
1660     throws FatalConnectionException {
1661       // TODO: Plug in other supported decoders.
1662       if (!header.hasCellBlockCodecClass()) return;
1663       String className = header.getCellBlockCodecClass();
1664       if (className == null || className.length() == 0) return;
1665       try {
1666         this.codec = (Codec)Class.forName(className).newInstance();
1667       } catch (Exception e) {
1668         throw new UnsupportedCellCodecException(className, e);
1669       }
1670       if (!header.hasCellBlockCompressorClass()) return;
1671       className = header.getCellBlockCompressorClass();
1672       try {
1673         this.compressionCodec = (CompressionCodec)Class.forName(className).newInstance();
1674       } catch (Exception e) {
1675         throw new UnsupportedCompressionCodecException(className, e);
1676       }
1677     }
1678 
1679     private void processUnwrappedData(byte[] inBuf) throws IOException,
1680     InterruptedException {
1681       ReadableByteChannel ch = Channels.newChannel(new ByteArrayInputStream(inBuf));
1682       // Read all RPCs contained in the inBuf, even partial ones
1683       while (true) {
1684         int count = -1;
1685         if (unwrappedDataLengthBuffer.remaining() > 0) {
1686           count = channelRead(ch, unwrappedDataLengthBuffer);
1687           if (count <= 0 || unwrappedDataLengthBuffer.remaining() > 0)
1688             return;
1689         }
1690 
1691         if (unwrappedData == null) {
1692           unwrappedDataLengthBuffer.flip();
1693           int unwrappedDataLength = unwrappedDataLengthBuffer.getInt();
1694 
1695           if (unwrappedDataLength == RpcClient.PING_CALL_ID) {
1696             if (LOG.isDebugEnabled())
1697               LOG.debug("Received ping message");
1698             unwrappedDataLengthBuffer.clear();
1699             continue; // ping message
1700           }
1701           unwrappedData = ByteBuffer.allocate(unwrappedDataLength);
1702         }
1703 
1704         count = channelRead(ch, unwrappedData);
1705         if (count <= 0 || unwrappedData.remaining() > 0)
1706           return;
1707 
1708         if (unwrappedData.remaining() == 0) {
1709           unwrappedDataLengthBuffer.clear();
1710           unwrappedData.flip();
1711           processOneRpc(unwrappedData.array());
1712           unwrappedData = null;
1713         }
1714       }
1715     }
1716 
1717     private void processOneRpc(byte[] buf) throws IOException, InterruptedException {
1718       if (connectionHeaderRead) {
1719         processRequest(buf);
1720       } else {
1721         processConnectionHeader(buf);
1722         this.connectionHeaderRead = true;
1723         if (!authorizeConnection()) {
1724           // Throw FatalConnectionException wrapping ACE so client does right thing and closes
1725           // down the connection instead of trying to read non-existent retun.
1726           throw new AccessDeniedException("Connection from " + this + " for service " +
1727             connectionHeader.getServiceName() + " is unauthorized for user: " + user);
1728         }
1729       }
1730     }
1731 
1732     /**
1733      * @param buf Has the request header and the request param and optionally encoded data buffer
1734      * all in this one array.
1735      * @throws IOException
1736      * @throws InterruptedException
1737      */
1738     protected void processRequest(byte[] buf) throws IOException, InterruptedException {
1739       long totalRequestSize = buf.length;
1740       int offset = 0;
1741       // Here we read in the header.  We avoid having pb
1742       // do its default 4k allocation for CodedInputStream.  We force it to use backing array.
1743       CodedInputStream cis = CodedInputStream.newInstance(buf, offset, buf.length);
1744       int headerSize = cis.readRawVarint32();
1745       offset = cis.getTotalBytesRead();
1746       Message.Builder builder = RequestHeader.newBuilder();
1747       ProtobufUtil.mergeFrom(builder, buf, offset, headerSize);
1748       RequestHeader header = (RequestHeader) builder.build();
1749       offset += headerSize;
1750       int id = header.getCallId();
1751       if (LOG.isTraceEnabled()) {
1752         LOG.trace("RequestHeader " + TextFormat.shortDebugString(header) +
1753           " totalRequestSize: " + totalRequestSize + " bytes");
1754       }
1755       // Enforcing the call queue size, this triggers a retry in the client
1756       // This is a bit late to be doing this check - we have already read in the total request.
1757       if ((totalRequestSize + callQueueSize.get()) > maxQueueSize) {
1758         final Call callTooBig =
1759           new Call(id, this.service, null, null, null, null, this,
1760             responder, totalRequestSize, null, null);
1761         ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
1762         metrics.exception(CALL_QUEUE_TOO_BIG_EXCEPTION);
1763         setupResponse(responseBuffer, callTooBig, CALL_QUEUE_TOO_BIG_EXCEPTION,
1764             "Call queue is full on " + getListenerAddress() +
1765                 ", is hbase.ipc.server.max.callqueue.size too small?");
1766         responder.doRespond(callTooBig);
1767         return;
1768       }
1769       MethodDescriptor md = null;
1770       Message param = null;
1771       CellScanner cellScanner = null;
1772       try {
1773         if (header.hasRequestParam() && header.getRequestParam()) {
1774           md = this.service.getDescriptorForType().findMethodByName(header.getMethodName());
1775           if (md == null) throw new UnsupportedOperationException(header.getMethodName());
1776           builder = this.service.getRequestPrototype(md).newBuilderForType();
1777           // To read the varint, I need an inputstream; might as well be a CIS.
1778           cis = CodedInputStream.newInstance(buf, offset, buf.length);
1779           int paramSize = cis.readRawVarint32();
1780           offset += cis.getTotalBytesRead();
1781           if (builder != null) {
1782             ProtobufUtil.mergeFrom(builder, buf, offset, paramSize);
1783             param = builder.build();
1784           }
1785           offset += paramSize;
1786         }
1787         if (header.hasCellBlockMeta()) {
1788           cellScanner = ipcUtil.createCellScanner(this.codec, this.compressionCodec,
1789             buf, offset, buf.length);
1790         }
1791       } catch (Throwable t) {
1792         String msg = getListenerAddress() + " is unable to read call parameter from client " +
1793             getHostAddress();
1794         LOG.warn(msg, t);
1795 
1796         metrics.exception(t);
1797 
1798         // probably the hbase hadoop version does not match the running hadoop version
1799         if (t instanceof LinkageError) {
1800           t = new DoNotRetryIOException(t);
1801         }
1802         // If the method is not present on the server, do not retry.
1803         if (t instanceof UnsupportedOperationException) {
1804           t = new DoNotRetryIOException(t);
1805         }
1806 
1807         final Call readParamsFailedCall =
1808           new Call(id, this.service, null, null, null, null, this,
1809             responder, totalRequestSize, null, null);
1810         ByteArrayOutputStream responseBuffer = new ByteArrayOutputStream();
1811         setupResponse(responseBuffer, readParamsFailedCall, t,
1812           msg + "; " + t.getMessage());
1813         responder.doRespond(readParamsFailedCall);
1814         return;
1815       }
1816 
1817       TraceInfo traceInfo = header.hasTraceInfo()
1818           ? new TraceInfo(header.getTraceInfo().getTraceId(), header.getTraceInfo().getParentId())
1819           : null;
1820       Call call = new Call(id, this.service, md, header, param, cellScanner, this, responder,
1821               totalRequestSize,
1822               traceInfo, RpcServer.getRemoteIp());
1823       scheduler.dispatch(new CallRunner(RpcServer.this, call));
1824     }
1825 
1826     private boolean authorizeConnection() throws IOException {
1827       try {
1828         // If auth method is DIGEST, the token was obtained by the
1829         // real user for the effective user, therefore not required to
1830         // authorize real user. doAs is allowed only for simple or kerberos
1831         // authentication
1832         if (user != null && user.getRealUser() != null
1833             && (authMethod != AuthMethod.DIGEST)) {
1834           ProxyUsers.authorize(user, this.getHostAddress(), conf);
1835         }
1836         authorize(user, connectionHeader, getHostInetAddress());
1837         if (LOG.isDebugEnabled()) {
1838           LOG.debug("Authorized " + TextFormat.shortDebugString(connectionHeader));
1839         }
1840         metrics.authorizationSuccess();
1841       } catch (AuthorizationException ae) {
1842         LOG.debug("Connection authorization failed: " + ae.getMessage(), ae);
1843         metrics.authorizationFailure();
1844         setupResponse(authFailedResponse, authFailedCall,
1845           new AccessDeniedException(ae), ae.getMessage());
1846         responder.doRespond(authFailedCall);
1847         return false;
1848       }
1849       return true;
1850     }
1851 
1852     protected synchronized void close() {
1853       disposeSasl();
1854       data = null;
1855       if (!channel.isOpen())
1856         return;
1857       try {socket.shutdownOutput();} catch(Exception ignored) {} // FindBugs DE_MIGHT_IGNORE
1858       if (channel.isOpen()) {
1859         try {channel.close();} catch(Exception ignored) {}
1860       }
1861       try {socket.close();} catch(Exception ignored) {}
1862     }
1863 
1864     private UserGroupInformation createUser(ConnectionHeader head) {
1865       UserGroupInformation ugi = null;
1866 
1867       if (!head.hasUserInfo()) {
1868         return null;
1869       }
1870       UserInformation userInfoProto = head.getUserInfo();
1871       String effectiveUser = null;
1872       if (userInfoProto.hasEffectiveUser()) {
1873         effectiveUser = userInfoProto.getEffectiveUser();
1874       }
1875       String realUser = null;
1876       if (userInfoProto.hasRealUser()) {
1877         realUser = userInfoProto.getRealUser();
1878       }
1879       if (effectiveUser != null) {
1880         if (realUser != null) {
1881           UserGroupInformation realUserUgi =
1882               UserGroupInformation.createRemoteUser(realUser);
1883           ugi = UserGroupInformation.createProxyUser(effectiveUser, realUserUgi);
1884         } else {
1885           ugi = UserGroupInformation.createRemoteUser(effectiveUser);
1886         }
1887       }
1888       return ugi;
1889     }
1890   }
1891 
1892   /**
1893    * Datastructure for passing a {@link BlockingService} and its associated class of
1894    * protobuf service interface.  For example, a server that fielded what is defined
1895    * in the client protobuf service would pass in an implementation of the client blocking service
1896    * and then its ClientService.BlockingInterface.class.  Used checking connection setup.
1897    */
1898   public static class BlockingServiceAndInterface {
1899     private final BlockingService service;
1900     private final Class<?> serviceInterface;
1901     public BlockingServiceAndInterface(final BlockingService service,
1902         final Class<?> serviceInterface) {
1903       this.service = service;
1904       this.serviceInterface = serviceInterface;
1905     }
1906     public Class<?> getServiceInterface() {
1907       return this.serviceInterface;
1908     }
1909     public BlockingService getBlockingService() {
1910       return this.service;
1911     }
1912   }
1913 
1914   /**
1915    * Constructs a server listening on the named port and address.
1916    * @param serverInstance hosting instance of {@link Server}. We will do authentications if an
1917    * instance else pass null for no authentication check.
1918    * @param name Used keying this rpc servers' metrics and for naming the Listener thread.
1919    * @param services A list of services.
1920    * @param isa Where to listen
1921    * @param conf
1922    * @throws IOException
1923    */
1924   public RpcServer(final Server serverInstance, final String name,
1925       final List<BlockingServiceAndInterface> services,
1926       final InetSocketAddress isa, Configuration conf,
1927       RpcScheduler scheduler)
1928       throws IOException {
1929     this.serverInstance = serverInstance;
1930     this.reservoir = new BoundedByteBufferPool(
1931       conf.getInt("hbase.ipc.server.reservoir.max.buffer.size",  1024 * 1024),
1932       conf.getInt("hbase.ipc.server.reservoir.initial.buffer.size", 16 * 1024),
1933       // Make the max twice the number of handlers to be safe.
1934       conf.getInt("hbase.ipc.server.reservoir.initial.max",
1935         conf.getInt(HConstants.REGION_SERVER_HANDLER_COUNT,
1936           HConstants.DEFAULT_REGION_SERVER_HANDLER_COUNT) * 2));
1937     this.services = services;
1938     this.isa = isa;
1939     this.conf = conf;
1940     this.socketSendBufferSize = 0;
1941     this.maxQueueSize = conf.getInt("hbase.ipc.server.max.callqueue.size",
1942       conf.getInt("ipc.server.max.callqueue.size", DEFAULT_MAX_CALLQUEUE_SIZE));
1943     this.readThreads = conf.getInt("hbase.ipc.server.read.threadpool.size",
1944       conf.getInt("ipc.server.read.threadpool.size", 10));
1945     this.maxIdleTime = 2 * conf.getInt("hbase.ipc.client.connection.maxidletime",
1946       conf.getInt("ipc.client.connection.maxidletime", 1000));
1947     this.maxConnectionsToNuke = conf.getInt("hbase.ipc.client.kill.max",
1948       conf.getInt("ipc.client.kill.max", 10));
1949     this.thresholdIdleConnections = conf.getInt("hbase.ipc.client.idlethreshold",
1950       conf.getInt("ipc.client.idlethreshold", 4000));
1951     this.purgeTimeout = conf.getLong("hbase.ipc.client.call.purge.timeout",
1952       conf.getLong("ipc.client.call.purge.timeout",
1953         2 * HConstants.DEFAULT_HBASE_RPC_TIMEOUT));
1954     this.warnResponseTime = conf.getInt(WARN_RESPONSE_TIME, DEFAULT_WARN_RESPONSE_TIME);
1955     this.warnResponseSize = conf.getInt(WARN_RESPONSE_SIZE, DEFAULT_WARN_RESPONSE_SIZE);
1956 
1957     // Start the listener here and let it bind to the port
1958     listener = new Listener(name);
1959     this.port = listener.getAddress().getPort();
1960 
1961     this.metrics = new MetricsHBaseServer(name, new MetricsHBaseServerWrapperImpl(this));
1962     this.tcpNoDelay = conf.getBoolean("hbase.ipc.server.tcpnodelay", true);
1963     this.tcpKeepAlive = conf.getBoolean("hbase.ipc.server.tcpkeepalive",
1964       conf.getBoolean("ipc.server.tcpkeepalive", true));
1965 
1966     this.warnDelayedCalls = conf.getInt(WARN_DELAYED_CALLS, DEFAULT_WARN_DELAYED_CALLS);
1967     this.delayedCalls = new AtomicInteger(0);
1968     this.ipcUtil = new IPCUtil(conf);
1969 
1970 
1971     // Create the responder here
1972     responder = new Responder();
1973     this.authorize = conf.getBoolean(HADOOP_SECURITY_AUTHORIZATION, false);
1974     this.userProvider = UserProvider.instantiate(conf);
1975     this.isSecurityEnabled = userProvider.isHBaseSecurityEnabled();
1976     if (isSecurityEnabled) {
1977       HBaseSaslRpcServer.init(conf);
1978     }
1979     this.scheduler = scheduler;
1980     this.scheduler.init(new RpcSchedulerContext(this));
1981   }
1982 
1983   /**
1984    * Subclasses of HBaseServer can override this to provide their own
1985    * Connection implementations.
1986    */
1987   protected Connection getConnection(SocketChannel channel, long time) {
1988     return new Connection(channel, time);
1989   }
1990 
1991   /**
1992    * Setup response for the RPC Call.
1993    *
1994    * @param response buffer to serialize the response into
1995    * @param call {@link Call} to which we are setting up the response
1996    * @param error error message, if the call failed
1997    * @param t
1998    * @throws IOException
1999    */
2000   private void setupResponse(ByteArrayOutputStream response, Call call, Throwable t, String error)
2001   throws IOException {
2002     if (response != null) response.reset();
2003     call.setResponse(null, null, t, error);
2004   }
2005 
2006   protected void closeConnection(Connection connection) {
2007     synchronized (connectionList) {
2008       if (connectionList.remove(connection)) {
2009         numConnections--;
2010       }
2011     }
2012     connection.close();
2013   }
2014 
2015   Configuration getConf() {
2016     return conf;
2017   }
2018 
2019   /** Sets the socket buffer size used for responding to RPCs.
2020    * @param size send size
2021    */
2022   @Override
2023   public void setSocketSendBufSize(int size) { this.socketSendBufferSize = size; }
2024 
2025   /** Starts the service.  Must be called before any calls will be handled. */
2026   @Override
2027   public void start() {
2028     startThreads();
2029     openServer();
2030   }
2031 
2032   /**
2033    * Open a previously started server.
2034    */
2035   @Override
2036   public void openServer() {
2037     this.started = true;
2038   }
2039 
2040   @Override
2041   public boolean isStarted() {
2042     return this.started;
2043   }
2044 
2045   /**
2046    * Starts the service threads but does not allow requests to be responded yet.
2047    * Client will get {@link ServerNotRunningYetException} instead.
2048    */
2049   @Override
2050   public synchronized void startThreads() {
2051     authTokenSecretMgr = createSecretManager();
2052     if (authTokenSecretMgr != null) {
2053       setSecretManager(authTokenSecretMgr);
2054       authTokenSecretMgr.start();
2055     }
2056     this.authManager = new ServiceAuthorizationManager();
2057     HBasePolicyProvider.init(conf, authManager);
2058     responder.start();
2059     listener.start();
2060     scheduler.start();
2061   }
2062 
2063   @Override
2064   public void refreshAuthManager(PolicyProvider pp) {
2065     // Ignore warnings that this should be accessed in a static way instead of via an instance;
2066     // it'll break if you go via static route.
2067     this.authManager.refresh(this.conf, pp);
2068   }
2069 
2070   private AuthenticationTokenSecretManager createSecretManager() {
2071     if (!isSecurityEnabled) return null;
2072     if (serverInstance == null) return null;
2073     if (!(serverInstance instanceof org.apache.hadoop.hbase.Server)) return null;
2074     org.apache.hadoop.hbase.Server server = (org.apache.hadoop.hbase.Server)serverInstance;
2075     Configuration conf = server.getConfiguration();
2076     long keyUpdateInterval =
2077         conf.getLong("hbase.auth.key.update.interval", 24*60*60*1000);
2078     long maxAge =
2079         conf.getLong("hbase.auth.token.max.lifetime", 7*24*60*60*1000);
2080     return new AuthenticationTokenSecretManager(conf, server.getZooKeeper(),
2081         server.getServerName().toString(), keyUpdateInterval, maxAge);
2082   }
2083 
2084   public SecretManager<? extends TokenIdentifier> getSecretManager() {
2085     return this.secretManager;
2086   }
2087 
2088   @SuppressWarnings("unchecked")
2089   public void setSecretManager(SecretManager<? extends TokenIdentifier> secretManager) {
2090     this.secretManager = (SecretManager<TokenIdentifier>) secretManager;
2091   }
2092 
2093   /**
2094    * This is a server side method, which is invoked over RPC. On success
2095    * the return response has protobuf response payload. On failure, the
2096    * exception name and the stack trace are returned in the protobuf response.
2097    */
2098   public Pair<Message, CellScanner> call(BlockingService service, MethodDescriptor md,
2099       Message param, CellScanner cellScanner, long receiveTime, MonitoredRPCHandler status)
2100   throws IOException {
2101     try {
2102       status.setRPC(md.getName(), new Object[]{param}, receiveTime);
2103       // TODO: Review after we add in encoded data blocks.
2104       status.setRPCPacket(param);
2105       status.resume("Servicing call");
2106       //get an instance of the method arg type
2107       long startTime = System.currentTimeMillis();
2108       PayloadCarryingRpcController controller = new PayloadCarryingRpcController(cellScanner);
2109       Message result = service.callBlockingMethod(md, controller, param);
2110       long endTime = System.currentTimeMillis();
2111       int processingTime = (int) (endTime - startTime);
2112       int qTime = (int) (startTime - receiveTime);
2113       int totalTime = (int) (endTime - receiveTime);
2114       if (LOG.isTraceEnabled()) {
2115         LOG.trace(CurCall.get().toString() +
2116             ", response " + TextFormat.shortDebugString(result) +
2117             " queueTime: " + qTime +
2118             " processingTime: " + processingTime +
2119             " totalTime: " + totalTime);
2120       }
2121       metrics.dequeuedCall(qTime);
2122       metrics.processedCall(processingTime);
2123       metrics.totalCall(totalTime);
2124       long responseSize = result.getSerializedSize();
2125       // log any RPC responses that are slower than the configured warn
2126       // response time or larger than configured warning size
2127       boolean tooSlow = (processingTime > warnResponseTime && warnResponseTime > -1);
2128       boolean tooLarge = (responseSize > warnResponseSize && warnResponseSize > -1);
2129       if (tooSlow || tooLarge) {
2130         // when tagging, we let TooLarge trump TooSmall to keep output simple
2131         // note that large responses will often also be slow.
2132         StringBuilder buffer = new StringBuilder(256);
2133         buffer.append(md.getName());
2134         buffer.append("(");
2135         buffer.append(param.getClass().getName());
2136         buffer.append(")");
2137         logResponse(new Object[]{param},
2138             md.getName(), buffer.toString(), (tooLarge ? "TooLarge" : "TooSlow"),
2139             status.getClient(), startTime, processingTime, qTime,
2140             responseSize);
2141       }
2142       return new Pair<Message, CellScanner>(result,
2143         controller != null? controller.cellScanner(): null);
2144     } catch (Throwable e) {
2145       // The above callBlockingMethod will always return a SE.  Strip the SE wrapper before
2146       // putting it on the wire.  Its needed to adhere to the pb Service Interface but we don't
2147       // need to pass it over the wire.
2148       if (e instanceof ServiceException) e = e.getCause();
2149 
2150       // increment the number of requests that were exceptions.
2151       metrics.exception(e);
2152 
2153       if (e instanceof LinkageError) throw new DoNotRetryIOException(e);
2154       if (e instanceof IOException) throw (IOException)e;
2155       LOG.error("Unexpected throwable object ", e);
2156       throw new IOException(e.getMessage(), e);
2157     }
2158   }
2159 
2160   /**
2161    * Logs an RPC response to the LOG file, producing valid JSON objects for
2162    * client Operations.
2163    * @param params The parameters received in the call.
2164    * @param methodName The name of the method invoked
2165    * @param call The string representation of the call
2166    * @param tag  The tag that will be used to indicate this event in the log.
2167    * @param clientAddress   The address of the client who made this call.
2168    * @param startTime       The time that the call was initiated, in ms.
2169    * @param processingTime  The duration that the call took to run, in ms.
2170    * @param qTime           The duration that the call spent on the queue
2171    *                        prior to being initiated, in ms.
2172    * @param responseSize    The size in bytes of the response buffer.
2173    */
2174   void logResponse(Object[] params, String methodName, String call, String tag,
2175       String clientAddress, long startTime, int processingTime, int qTime,
2176       long responseSize)
2177           throws IOException {
2178     // base information that is reported regardless of type of call
2179     Map<String, Object> responseInfo = new HashMap<String, Object>();
2180     responseInfo.put("starttimems", startTime);
2181     responseInfo.put("processingtimems", processingTime);
2182     responseInfo.put("queuetimems", qTime);
2183     responseInfo.put("responsesize", responseSize);
2184     responseInfo.put("client", clientAddress);
2185     responseInfo.put("class", serverInstance == null? "": serverInstance.getClass().getSimpleName());
2186     responseInfo.put("method", methodName);
2187     if (params.length == 2 && serverInstance instanceof HRegionServer &&
2188         params[0] instanceof byte[] &&
2189         params[1] instanceof Operation) {
2190       // if the slow process is a query, we want to log its table as well
2191       // as its own fingerprint
2192       TableName tableName = TableName.valueOf(
2193           HRegionInfo.parseRegionName((byte[]) params[0])[0]);
2194       responseInfo.put("table", tableName.getNameAsString());
2195       // annotate the response map with operation details
2196       responseInfo.putAll(((Operation) params[1]).toMap());
2197       // report to the log file
2198       LOG.warn("(operation" + tag + "): " +
2199                MAPPER.writeValueAsString(responseInfo));
2200     } else if (params.length == 1 && serverInstance instanceof HRegionServer &&
2201         params[0] instanceof Operation) {
2202       // annotate the response map with operation details
2203       responseInfo.putAll(((Operation) params[0]).toMap());
2204       // report to the log file
2205       LOG.warn("(operation" + tag + "): " +
2206                MAPPER.writeValueAsString(responseInfo));
2207     } else {
2208       // can't get JSON details, so just report call.toString() along with
2209       // a more generic tag.
2210       responseInfo.put("call", call);
2211       LOG.warn("(response" + tag + "): " + MAPPER.writeValueAsString(responseInfo));
2212     }
2213   }
2214 
2215   /** Stops the service.  No new calls will be handled after this is called. */
2216   @Override
2217   public synchronized void stop() {
2218     LOG.info("Stopping server on " + port);
2219     running = false;
2220     if (authTokenSecretMgr != null) {
2221       authTokenSecretMgr.stop();
2222       authTokenSecretMgr = null;
2223     }
2224     listener.interrupt();
2225     listener.doStop();
2226     responder.interrupt();
2227     scheduler.stop();
2228     notifyAll();
2229   }
2230 
2231   /** Wait for the server to be stopped.
2232    * Does not wait for all subthreads to finish.
2233    *  See {@link #stop()}.
2234    * @throws InterruptedException e
2235    */
2236   @Override
2237   public synchronized void join() throws InterruptedException {
2238     while (running) {
2239       wait();
2240     }
2241   }
2242 
2243   /**
2244    * Return the socket (ip+port) on which the RPC server is listening to.
2245    * @return the socket (ip+port) on which the RPC server is listening to.
2246    */
2247   @Override
2248   public synchronized InetSocketAddress getListenerAddress() {
2249     return listener.getAddress();
2250   }
2251 
2252   /**
2253    * Set the handler for calling out of RPC for error conditions.
2254    * @param handler the handler implementation
2255    */
2256   @Override
2257   public void setErrorHandler(HBaseRPCErrorHandler handler) {
2258     this.errorHandler = handler;
2259   }
2260 
2261   @Override
2262   public HBaseRPCErrorHandler getErrorHandler() {
2263     return this.errorHandler;
2264   }
2265 
2266   /**
2267    * Returns the metrics instance for reporting RPC call statistics
2268    */
2269   public MetricsHBaseServer getMetrics() {
2270     return metrics;
2271   }
2272 
2273   @Override
2274   public void addCallSize(final long diff) {
2275     this.callQueueSize.add(diff);
2276   }
2277 
2278   /**
2279    * Authorize the incoming client connection.
2280    *
2281    * @param user client user
2282    * @param connection incoming connection
2283    * @param addr InetAddress of incoming connection
2284    * @throws org.apache.hadoop.security.authorize.AuthorizationException when the client isn't authorized to talk the protocol
2285    */
2286   @SuppressWarnings("static-access")
2287   public void authorize(UserGroupInformation user, ConnectionHeader connection, InetAddress addr)
2288   throws AuthorizationException {
2289     if (authorize) {
2290       Class<?> c = getServiceInterface(services, connection.getServiceName());
2291       this.authManager.authorize(user != null ? user : null, c, getConf(), addr);
2292     }
2293   }
2294 
2295   /**
2296    * When the read or write buffer size is larger than this limit, i/o will be
2297    * done in chunks of this size. Most RPC requests and responses would be
2298    * be smaller.
2299    */
2300   private static int NIO_BUFFER_LIMIT = 64 * 1024; //should not be more than 64KB.
2301 
2302   /**
2303    * This is a wrapper around {@link java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer)}.
2304    * If the amount of data is large, it writes to channel in smaller chunks.
2305    * This is to avoid jdk from creating many direct buffers as the size of
2306    * buffer increases. This also minimizes extra copies in NIO layer
2307    * as a result of multiple write operations required to write a large
2308    * buffer.
2309    *
2310    * @param channel writable byte channel to write to
2311    * @param bufferChain Chain of buffers to write
2312    * @return number of bytes written
2313    * @throws java.io.IOException e
2314    * @see java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer)
2315    */
2316   protected long channelWrite(GatheringByteChannel channel, BufferChain bufferChain)
2317   throws IOException {
2318     long count =  bufferChain.write(channel, NIO_BUFFER_LIMIT);
2319     if (count > 0) this.metrics.sentBytes(count);
2320     return count;
2321   }
2322 
2323   /**
2324    * This is a wrapper around {@link java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)}.
2325    * If the amount of data is large, it writes to channel in smaller chunks.
2326    * This is to avoid jdk from creating many direct buffers as the size of
2327    * ByteBuffer increases. There should not be any performance degredation.
2328    *
2329    * @param channel writable byte channel to write on
2330    * @param buffer buffer to write
2331    * @return number of bytes written
2332    * @throws java.io.IOException e
2333    * @see java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer)
2334    */
2335   protected int channelRead(ReadableByteChannel channel,
2336                                    ByteBuffer buffer) throws IOException {
2337 
2338     int count = (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
2339            channel.read(buffer) : channelIO(channel, null, buffer);
2340     if (count > 0) {
2341       metrics.receivedBytes(count);
2342     }
2343     return count;
2344   }
2345 
2346   /**
2347    * Helper for {@link #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)}
2348    * and {@link #channelWrite(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer)}. Only
2349    * one of readCh or writeCh should be non-null.
2350    *
2351    * @param readCh read channel
2352    * @param writeCh write channel
2353    * @param buf buffer to read or write into/out of
2354    * @return bytes written
2355    * @throws java.io.IOException e
2356    * @see #channelRead(java.nio.channels.ReadableByteChannel, java.nio.ByteBuffer)
2357    * @see #channelWrite(java.nio.channels.WritableByteChannel, java.nio.ByteBuffer)
2358    */
2359   private static int channelIO(ReadableByteChannel readCh,
2360                                WritableByteChannel writeCh,
2361                                ByteBuffer buf) throws IOException {
2362 
2363     int originalLimit = buf.limit();
2364     int initialRemaining = buf.remaining();
2365     int ret = 0;
2366 
2367     while (buf.remaining() > 0) {
2368       try {
2369         int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
2370         buf.limit(buf.position() + ioSize);
2371 
2372         ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
2373 
2374         if (ret < ioSize) {
2375           break;
2376         }
2377 
2378       } finally {
2379         buf.limit(originalLimit);
2380       }
2381     }
2382 
2383     int nBytes = initialRemaining - buf.remaining();
2384     return (nBytes > 0) ? nBytes : ret;
2385   }
2386 
2387   /**
2388    * Needed for features such as delayed calls.  We need to be able to store the current call
2389    * so that we can complete it later or ask questions of what is supported by the current ongoing
2390    * call.
2391    * @return An RpcCallConext backed by the currently ongoing call (gotten from a thread local)
2392    */
2393   public static RpcCallContext getCurrentCall() {
2394     return CurCall.get();
2395   }
2396 
2397   public static boolean isInRpcCallContext() {
2398     return CurCall.get() != null;
2399   }
2400 
2401   /**
2402    * Returns the user credentials associated with the current RPC request or
2403    * <code>null</code> if no credentials were provided.
2404    * @return A User
2405    */
2406   public static User getRequestUser() {
2407     RpcCallContext ctx = getCurrentCall();
2408     return ctx == null? null: ctx.getRequestUser();
2409   }
2410 
2411   /**
2412    * Returns the username for any user associated with the current RPC
2413    * request or <code>null</code> if no user is set.
2414    */
2415   public static String getRequestUserName() {
2416     User user = getRequestUser();
2417     return user == null? null: user.getShortName();
2418   }
2419 
2420   /**
2421    * @return Address of remote client if a request is ongoing, else null
2422    */
2423   public static InetAddress getRemoteAddress() {
2424     RpcCallContext ctx = getCurrentCall();
2425     return ctx == null? null: ctx.getRemoteAddress();
2426   }
2427 
2428   /**
2429    * @param serviceName Some arbitrary string that represents a 'service'.
2430    * @param services Available service instances
2431    * @return Matching BlockingServiceAndInterface pair
2432    */
2433   static BlockingServiceAndInterface getServiceAndInterface(
2434       final List<BlockingServiceAndInterface> services, final String serviceName) {
2435     for (BlockingServiceAndInterface bs : services) {
2436       if (bs.getBlockingService().getDescriptorForType().getName().equals(serviceName)) {
2437         return bs;
2438       }
2439     }
2440     return null;
2441   }
2442 
2443   /**
2444    * @param serviceName Some arbitrary string that represents a 'service'.
2445    * @param services Available services and their service interfaces.
2446    * @return Service interface class for <code>serviceName</code>
2447    */
2448   static Class<?> getServiceInterface(
2449       final List<BlockingServiceAndInterface> services,
2450       final String serviceName) {
2451     BlockingServiceAndInterface bsasi =
2452         getServiceAndInterface(services, serviceName);
2453     return bsasi == null? null: bsasi.getServiceInterface();
2454   }
2455 
2456   /**
2457    * @param serviceName Some arbitrary string that represents a 'service'.
2458    * @param services Available services and their service interfaces.
2459    * @return BlockingService that goes with the passed <code>serviceName</code>
2460    */
2461   static BlockingService getService(
2462       final List<BlockingServiceAndInterface> services,
2463       final String serviceName) {
2464     BlockingServiceAndInterface bsasi =
2465         getServiceAndInterface(services, serviceName);
2466     return bsasi == null? null: bsasi.getBlockingService();
2467   }
2468 
2469   /** Returns the remote side ip address when invoked inside an RPC
2470    *  Returns null incase of an error.
2471    *  @return InetAddress
2472    */
2473   public static InetAddress getRemoteIp() {
2474     Call call = CurCall.get();
2475     if (call != null && call.connection.socket != null) {
2476       return call.connection.socket.getInetAddress();
2477     }
2478     return null;
2479   }
2480 
2481   /**
2482    * A convenience method to bind to a given address and report
2483    * better exceptions if the address is not a valid host.
2484    * @param socket the socket to bind
2485    * @param address the address to bind to
2486    * @param backlog the number of connections allowed in the queue
2487    * @throws BindException if the address can't be bound
2488    * @throws UnknownHostException if the address isn't a valid host name
2489    * @throws IOException other random errors from bind
2490    */
2491   public static void bind(ServerSocket socket, InetSocketAddress address,
2492                           int backlog) throws IOException {
2493     try {
2494       socket.bind(address, backlog);
2495     } catch (BindException e) {
2496       BindException bindException =
2497         new BindException("Problem binding to " + address + " : " +
2498             e.getMessage());
2499       bindException.initCause(e);
2500       throw bindException;
2501     } catch (SocketException e) {
2502       // If they try to bind to a different host's address, give a better
2503       // error message.
2504       if ("Unresolved address".equals(e.getMessage())) {
2505         throw new UnknownHostException("Invalid hostname for server: " +
2506                                        address.getHostName());
2507       }
2508       throw e;
2509     }
2510   }
2511 
2512   public RpcScheduler getScheduler() {
2513     return scheduler;
2514   }
2515 }