View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.ipc;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.io.ObjectWritable;
27  import org.apache.hadoop.io.Writable;
28  import org.apache.hadoop.io.WritableUtils;
29  import org.apache.hadoop.security.UserGroupInformation;
30  import org.apache.hadoop.util.ReflectionUtils;
31  import org.apache.hadoop.util.StringUtils;
32  
33  import java.io.ByteArrayInputStream;
34  import java.io.ByteArrayOutputStream;
35  import java.io.DataInputStream;
36  import java.io.DataOutputStream;
37  import java.io.IOException;
38  import java.net.BindException;
39  import java.net.InetAddress;
40  import java.net.InetSocketAddress;
41  import java.net.ServerSocket;
42  import java.net.Socket;
43  import java.net.SocketException;
44  import java.net.UnknownHostException;
45  import java.nio.ByteBuffer;
46  import java.nio.channels.CancelledKeyException;
47  import java.nio.channels.ClosedChannelException;
48  import java.nio.channels.ReadableByteChannel;
49  import java.nio.channels.SelectionKey;
50  import java.nio.channels.Selector;
51  import java.nio.channels.ServerSocketChannel;
52  import java.nio.channels.SocketChannel;
53  import java.nio.channels.WritableByteChannel;
54  import java.util.ArrayList;
55  import java.util.Collections;
56  import java.util.Iterator;
57  import java.util.LinkedList;
58  import java.util.List;
59  import java.util.Random;
60  import java.util.concurrent.BlockingQueue;
61  import java.util.concurrent.LinkedBlockingQueue;
62  
63  /** An abstract IPC service.  IPC calls take a single {@link Writable} as a
64   * parameter, and return a {@link Writable} as their value.  A service runs on
65   * a port and is defined by a parameter class and a value class.
66   *
67   *
68   * <p>Copied local so can fix HBASE-900.
69   *
70   * @see HBaseClient
71   */
72  public abstract class HBaseServer {
73  
74    /**
75     * The first four bytes of Hadoop RPC connections
76     */
77    public static final ByteBuffer HEADER = ByteBuffer.wrap("hrpc".getBytes());
78  
79    // 1 : Introduce ping and server does not throw away RPCs
80    // 3 : RPC was refactored in 0.19
81    public static final byte CURRENT_VERSION = 3;
82  
83    /**
84     * How many calls/handler are allowed in the queue.
85     */
86    private static final int MAX_QUEUE_SIZE_PER_HANDLER = 100;
87  
88    public static final Log LOG =
89      LogFactory.getLog("org.apache.hadoop.ipc.HBaseServer");
90  
91    protected static final ThreadLocal<HBaseServer> SERVER =
92      new ThreadLocal<HBaseServer>();
93  
94    /** Returns the server instance called under or null.  May be called under
95     * {@link #call(Writable, long)} implementations, and under {@link Writable}
96     * methods of paramters and return values.  Permits applications to access
97     * the server context.
98     * @return HBaseServer
99     */
100   public static HBaseServer get() {
101     return SERVER.get();
102   }
103 
104   /** This is set to Call object before Handler invokes an RPC and reset
105    * after the call returns.
106    */
107   protected static final ThreadLocal<Call> CurCall = new ThreadLocal<Call>();
108 
109   /** Returns the remote side ip address when invoked inside an RPC
110    *  Returns null incase of an error.
111    *  @return InetAddress
112    */
113   public static InetAddress getRemoteIp() {
114     Call call = CurCall.get();
115     if (call != null) {
116       return call.connection.socket.getInetAddress();
117     }
118     return null;
119   }
120   /** Returns remote address as a string when invoked inside an RPC.
121    *  Returns null in case of an error.
122    *  @return String
123    */
124   public static String getRemoteAddress() {
125     InetAddress addr = getRemoteIp();
126     return (addr == null) ? null : addr.getHostAddress();
127   }
128 
129   protected String bindAddress;
130   protected int port;                             // port we listen on
131   private int handlerCount;                       // number of handler threads
132   protected Class<? extends Writable> paramClass; // class of call parameters
133   protected int maxIdleTime;                      // the maximum idle time after
134                                                   // which a client may be
135                                                   // disconnected
136   protected int thresholdIdleConnections;         // the number of idle
137                                                   // connections after which we
138                                                   // will start cleaning up idle
139                                                   // connections
140   int maxConnectionsToNuke;                       // the max number of
141                                                   // connections to nuke
142                                                   // during a cleanup
143 
144   protected HBaseRpcMetrics  rpcMetrics;
145 
146   protected Configuration conf;
147 
148   @SuppressWarnings({"FieldCanBeLocal"})
149   private int maxQueueSize;
150   protected int socketSendBufferSize;
151   protected final boolean tcpNoDelay;   // if T then disable Nagle's Algorithm
152   protected final boolean tcpKeepAlive; // if T then use keepalives
153 
154   volatile protected boolean running = true;         // true while server runs
155   protected BlockingQueue<Call> callQueue; // queued calls
156 
157   protected final List<Connection> connectionList =
158     Collections.synchronizedList(new LinkedList<Connection>());
159   //maintain a list
160   //of client connections
161   private Listener listener = null;
162   protected Responder responder = null;
163   protected int numConnections = 0;
164   private Handler[] handlers = null;
165   protected HBaseRPCErrorHandler errorHandler = null;
166 
167   /**
168    * A convenience method to bind to a given address and report
169    * better exceptions if the address is not a valid host.
170    * @param socket the socket to bind
171    * @param address the address to bind to
172    * @param backlog the number of connections allowed in the queue
173    * @throws BindException if the address can't be bound
174    * @throws UnknownHostException if the address isn't a valid host name
175    * @throws IOException other random errors from bind
176    */
177   public static void bind(ServerSocket socket, InetSocketAddress address,
178                           int backlog) throws IOException {
179     try {
180       socket.bind(address, backlog);
181     } catch (BindException e) {
182       BindException bindException =
183         new BindException("Problem binding to " + address + " : " +
184             e.getMessage());
185       bindException.initCause(e);
186       throw bindException;
187     } catch (SocketException e) {
188       // If they try to bind to a different host's address, give a better
189       // error message.
190       if ("Unresolved address".equals(e.getMessage())) {
191         throw new UnknownHostException("Invalid hostname for server: " +
192                                        address.getHostName());
193       }
194       throw e;
195     }
196   }
197 
198   /** A call queued for handling. */
199   private static class Call {
200     protected int id;                             // the client's call id
201     protected Writable param;                     // the parameter passed
202     protected Connection connection;              // connection to client
203     protected long timestamp;      // the time received when response is null
204                                    // the time served when response is not null
205     protected ByteBuffer response;                // the response for this call
206 
207     public Call(int id, Writable param, Connection connection) {
208       this.id = id;
209       this.param = param;
210       this.connection = connection;
211       this.timestamp = System.currentTimeMillis();
212       this.response = null;
213     }
214 
215     @Override
216     public String toString() {
217       return param.toString() + " from " + connection.toString();
218     }
219 
220     public void setResponse(ByteBuffer response) {
221       this.response = response;
222     }
223   }
224 
225   /** Listens on the socket. Creates jobs for the handler threads*/
226   private class Listener extends Thread {
227 
228     private ServerSocketChannel acceptChannel = null; //the accept channel
229     private Selector selector = null; //the selector that we use for the server
230     private InetSocketAddress address; //the address we bind at
231     private Random rand = new Random();
232     private long lastCleanupRunTime = 0; //the last time when a cleanup connec-
233                                          //-tion (for idle connections) ran
234     private long cleanupInterval = 10000; //the minimum interval between
235                                           //two cleanup runs
236     private int backlogLength = conf.getInt("ipc.server.listen.queue.size", 128);
237 
238     public Listener() throws IOException {
239       address = new InetSocketAddress(bindAddress, port);
240       // Create a new server socket and set to non blocking mode
241       acceptChannel = ServerSocketChannel.open();
242       acceptChannel.configureBlocking(false);
243 
244       // Bind the server socket to the local host and port
245       bind(acceptChannel.socket(), address, backlogLength);
246       port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
247       // create a selector;
248       selector= Selector.open();
249 
250       // Register accepts on the server socket with the selector.
251       acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
252       this.setName("IPC Server listener on " + port);
253       this.setDaemon(true);
254     }
255     /** cleanup connections from connectionList. Choose a random range
256      * to scan and also have a limit on the number of the connections
257      * that will be cleanedup per run. The criteria for cleanup is the time
258      * for which the connection was idle. If 'force' is true then all
259      * connections will be looked at for the cleanup.
260      * @param force all connections will be looked at for cleanup
261      */
262     private void cleanupConnections(boolean force) {
263       if (force || numConnections > thresholdIdleConnections) {
264         long currentTime = System.currentTimeMillis();
265         if (!force && (currentTime - lastCleanupRunTime) < cleanupInterval) {
266           return;
267         }
268         int start = 0;
269         int end = numConnections - 1;
270         if (!force) {
271           start = rand.nextInt() % numConnections;
272           end = rand.nextInt() % numConnections;
273           int temp;
274           if (end < start) {
275             temp = start;
276             start = end;
277             end = temp;
278           }
279         }
280         int i = start;
281         int numNuked = 0;
282         while (i <= end) {
283           Connection c;
284           synchronized (connectionList) {
285             try {
286               c = connectionList.get(i);
287             } catch (Exception e) {return;}
288           }
289           if (c.timedOut(currentTime)) {
290             if (LOG.isDebugEnabled())
291               LOG.debug(getName() + ": disconnecting client " + c.getHostAddress());
292             closeConnection(c);
293             numNuked++;
294             end--;
295             //noinspection UnusedAssignment
296             c = null;
297             if (!force && numNuked == maxConnectionsToNuke) break;
298           }
299           else i++;
300         }
301         lastCleanupRunTime = System.currentTimeMillis();
302       }
303     }
304 
305     @Override
306     public void run() {
307       LOG.info(getName() + ": starting");
308       SERVER.set(HBaseServer.this);
309 
310       while (running) {
311         SelectionKey key = null;
312         try {
313           selector.select(); // FindBugs IS2_INCONSISTENT_SYNC
314           Iterator<SelectionKey> iter = selector.selectedKeys().iterator();
315           while (iter.hasNext()) {
316             key = iter.next();
317             iter.remove();
318             try {
319               if (key.isValid()) {
320                 if (key.isAcceptable())
321                   doAccept(key);
322                 else if (key.isReadable())
323                   doRead(key);
324               }
325             } catch (IOException ignored) {
326             }
327             key = null;
328           }
329         } catch (OutOfMemoryError e) {
330           if (errorHandler != null) {
331             if (errorHandler.checkOOME(e)) {
332               LOG.info(getName() + ": exiting on OOME");
333               closeCurrentConnection(key);
334               cleanupConnections(true);
335               return;
336             }
337           } else {
338             // we can run out of memory if we have too many threads
339             // log the event and sleep for a minute and give
340             // some thread(s) a chance to finish
341             LOG.warn("Out of Memory in server select", e);
342             closeCurrentConnection(key);
343             cleanupConnections(true);
344             try { Thread.sleep(60000); } catch (Exception ignored) {}
345       }
346         } catch (InterruptedException e) {
347           if (running) {                          // unexpected -- log it
348             LOG.info(getName() + " caught: " +
349                      StringUtils.stringifyException(e));
350           }
351         } catch (Exception e) {
352           closeCurrentConnection(key);
353         }
354         cleanupConnections(false);
355       }
356       LOG.info("Stopping " + this.getName());
357 
358       synchronized (this) {
359         try {
360           acceptChannel.close();
361           selector.close();
362         } catch (IOException ignored) { }
363 
364         selector= null;
365         acceptChannel= null;
366 
367         // clean up all connections
368         while (!connectionList.isEmpty()) {
369           closeConnection(connectionList.remove(0));
370         }
371       }
372     }
373 
374     private void closeCurrentConnection(SelectionKey key) {
375       if (key != null) {
376         Connection c = (Connection)key.attachment();
377         if (c != null) {
378           if (LOG.isDebugEnabled())
379             LOG.debug(getName() + ": disconnecting client " + c.getHostAddress());
380           closeConnection(c);
381         }
382       }
383     }
384 
385     InetSocketAddress getAddress() {
386       return (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();
387     }
388 
389     void doAccept(SelectionKey key) throws IOException, OutOfMemoryError {
390       Connection c;
391       ServerSocketChannel server = (ServerSocketChannel) key.channel();
392       // accept up to 10 connections
393       for (int i=0; i<10; i++) {
394         SocketChannel channel = server.accept();
395         if (channel==null) return;
396 
397         channel.configureBlocking(false);
398         channel.socket().setTcpNoDelay(tcpNoDelay);
399         channel.socket().setKeepAlive(tcpKeepAlive);
400         SelectionKey readKey = channel.register(selector, SelectionKey.OP_READ);
401         c = new Connection(channel, System.currentTimeMillis());
402         readKey.attach(c);
403         synchronized (connectionList) {
404           connectionList.add(numConnections, c);
405           numConnections++;
406         }
407         if (LOG.isDebugEnabled())
408           LOG.debug("Server connection from " + c.toString() +
409               "; # active connections: " + numConnections +
410               "; # queued calls: " + callQueue.size());
411       }
412     }
413 
414     void doRead(SelectionKey key) throws InterruptedException {
415       int count = 0;
416       Connection c = (Connection)key.attachment();
417       if (c == null) {
418         return;
419       }
420       c.setLastContact(System.currentTimeMillis());
421 
422       try {
423         count = c.readAndProcess();
424       } catch (InterruptedException ieo) {
425         throw ieo;
426       } catch (Exception e) {
427         LOG.debug(getName() + ": readAndProcess threw exception " + e + ". Count of bytes read: " + count, e);
428         count = -1; //so that the (count < 0) block is executed
429       }
430       if (count < 0) {
431         if (LOG.isDebugEnabled())
432           LOG.debug(getName() + ": disconnecting client " +
433                     c.getHostAddress() + ". Number of active connections: "+
434                     numConnections);
435         closeConnection(c);
436         // c = null;
437       }
438       else {
439         c.setLastContact(System.currentTimeMillis());
440       }
441     }
442 
443     synchronized void doStop() {
444       if (selector != null) {
445         selector.wakeup();
446         Thread.yield();
447       }
448       if (acceptChannel != null) {
449         try {
450           acceptChannel.socket().close();
451         } catch (IOException e) {
452           LOG.info(getName() + ":Exception in closing listener socket. " + e);
453         }
454       }
455     }
456   }
457 
458   // Sends responses of RPC back to clients.
459   private class Responder extends Thread {
460     private Selector writeSelector;
461     private int pending;         // connections waiting to register
462 
463     final static int PURGE_INTERVAL = 900000; // 15mins
464 
465     Responder() throws IOException {
466       this.setName("IPC Server Responder");
467       this.setDaemon(true);
468       writeSelector = Selector.open(); // create a selector
469       pending = 0;
470     }
471 
472     @Override
473     public void run() {
474       LOG.info(getName() + ": starting");
475       SERVER.set(HBaseServer.this);
476       long lastPurgeTime = 0;   // last check for old calls.
477 
478       while (running) {
479         try {
480           waitPending();     // If a channel is being registered, wait.
481           writeSelector.select(PURGE_INTERVAL);
482           Iterator<SelectionKey> iter = writeSelector.selectedKeys().iterator();
483           while (iter.hasNext()) {
484             SelectionKey key = iter.next();
485             iter.remove();
486             try {
487               if (key.isValid() && key.isWritable()) {
488                   doAsyncWrite(key);
489               }
490             } catch (IOException e) {
491               LOG.info(getName() + ": doAsyncWrite threw exception " + e);
492             }
493           }
494           long now = System.currentTimeMillis();
495           if (now < lastPurgeTime + PURGE_INTERVAL) {
496             continue;
497           }
498           lastPurgeTime = now;
499           //
500           // If there were some calls that have not been sent out for a
501           // long time, discard them.
502           //
503           LOG.debug("Checking for old call responses.");
504           ArrayList<Call> calls;
505 
506           // get the list of channels from list of keys.
507           synchronized (writeSelector.keys()) {
508             calls = new ArrayList<Call>(writeSelector.keys().size());
509             iter = writeSelector.keys().iterator();
510             while (iter.hasNext()) {
511               SelectionKey key = iter.next();
512               Call call = (Call)key.attachment();
513               if (call != null && key.channel() == call.connection.channel) {
514                 calls.add(call);
515               }
516             }
517           }
518 
519           for(Call call : calls) {
520             doPurge(call, now);
521           }
522         } catch (OutOfMemoryError e) {
523           if (errorHandler != null) {
524             if (errorHandler.checkOOME(e)) {
525               LOG.info(getName() + ": exiting on OOME");
526               return;
527             }
528           } else {
529             //
530             // we can run out of memory if we have too many threads
531             // log the event and sleep for a minute and give
532             // some thread(s) a chance to finish
533             //
534             LOG.warn("Out of Memory in server select", e);
535             try { Thread.sleep(60000); } catch (Exception ignored) {}
536       }
537         } catch (Exception e) {
538           LOG.warn("Exception in Responder " +
539                    StringUtils.stringifyException(e));
540         }
541       }
542       LOG.info("Stopping " + this.getName());
543     }
544 
545     private void doAsyncWrite(SelectionKey key) throws IOException {
546       Call call = (Call)key.attachment();
547       if (call == null) {
548         return;
549       }
550       if (key.channel() != call.connection.channel) {
551         throw new IOException("doAsyncWrite: bad channel");
552       }
553 
554       synchronized(call.connection.responseQueue) {
555         if (processResponse(call.connection.responseQueue, false)) {
556           try {
557             key.interestOps(0);
558           } catch (CancelledKeyException e) {
559             /* The Listener/reader might have closed the socket.
560              * We don't explicitly cancel the key, so not sure if this will
561              * ever fire.
562              * This warning could be removed.
563              */
564             LOG.warn("Exception while changing ops : " + e);
565           }
566         }
567       }
568     }
569 
570     //
571     // Remove calls that have been pending in the responseQueue
572     // for a long time.
573     //
574     private void doPurge(Call call, long now) {
575       synchronized (call.connection.responseQueue) {
576         Iterator<Call> iter = call.connection.responseQueue.listIterator(0);
577         while (iter.hasNext()) {
578           Call nextCall = iter.next();
579           if (now > nextCall.timestamp + PURGE_INTERVAL) {
580             closeConnection(nextCall.connection);
581             break;
582           }
583         }
584       }
585     }
586 
587     // Processes one response. Returns true if there are no more pending
588     // data for this channel.
589     //
590     @SuppressWarnings({"ConstantConditions"})
591     private boolean processResponse(final LinkedList<Call> responseQueue,
592                                     boolean inHandler) throws IOException {
593       boolean error = true;
594       boolean done = false;       // there is more data for this channel.
595       int numElements;
596       Call call = null;
597       try {
598         //noinspection SynchronizationOnLocalVariableOrMethodParameter
599         synchronized (responseQueue) {
600           //
601           // If there are no items for this channel, then we are done
602           //
603           numElements = responseQueue.size();
604           if (numElements == 0) {
605             error = false;
606             return true;              // no more data for this channel.
607           }
608           //
609           // Extract the first call
610           //
611           call = responseQueue.removeFirst();
612           SocketChannel channel = call.connection.channel;
613           if (LOG.isDebugEnabled()) {
614             LOG.debug(getName() + ": responding to #" + call.id + " from " +
615                       call.connection);
616           }
617           //
618           // Send as much data as we can in the non-blocking fashion
619           //
620           int numBytes = channelWrite(channel, call.response);
621           if (numBytes < 0) {
622             return true;
623           }
624           if (!call.response.hasRemaining()) {
625             call.connection.decRpcCount();
626             //noinspection RedundantIfStatement
627             if (numElements == 1) {    // last call fully processes.
628               done = true;             // no more data for this channel.
629             } else {
630               done = false;            // more calls pending to be sent.
631             }
632             if (LOG.isDebugEnabled()) {
633               LOG.debug(getName() + ": responding to #" + call.id + " from " +
634                         call.connection + " Wrote " + numBytes + " bytes.");
635             }
636           } else {
637             //
638             // If we were unable to write the entire response out, then
639             // insert in Selector queue.
640             //
641             call.connection.responseQueue.addFirst(call);
642 
643             if (inHandler) {
644               // set the serve time when the response has to be sent later
645               call.timestamp = System.currentTimeMillis();
646 
647               incPending();
648               try {
649                 // Wakeup the thread blocked on select, only then can the call
650                 // to channel.register() complete.
651                 writeSelector.wakeup();
652                 channel.register(writeSelector, SelectionKey.OP_WRITE, call);
653               } catch (ClosedChannelException e) {
654                 //Its ok. channel might be closed else where.
655                 done = true;
656               } finally {
657                 decPending();
658               }
659             }
660             if (LOG.isDebugEnabled()) {
661               LOG.debug(getName() + ": responding to #" + call.id + " from " +
662                         call.connection + " Wrote partial " + numBytes +
663                         " bytes.");
664             }
665           }
666           error = false;              // everything went off well
667         }
668       } finally {
669         if (error && call != null) {
670           LOG.warn(getName()+", call " + call + ": output error");
671           done = true;               // error. no more data for this channel.
672           closeConnection(call.connection);
673         }
674       }
675       return done;
676     }
677 
678     //
679     // Enqueue a response from the application.
680     //
681     void doRespond(Call call) throws IOException {
682       synchronized (call.connection.responseQueue) {
683         call.connection.responseQueue.addLast(call);
684         if (call.connection.responseQueue.size() == 1) {
685           processResponse(call.connection.responseQueue, true);
686         }
687       }
688     }
689 
690     private synchronized void incPending() {   // call waiting to be enqueued.
691       pending++;
692     }
693 
694     private synchronized void decPending() { // call done enqueueing.
695       pending--;
696       notify();
697     }
698 
699     private synchronized void waitPending() throws InterruptedException {
700       while (pending > 0) {
701         wait();
702       }
703     }
704   }
705 
706   /** Reads calls from a connection and queues them for handling. */
707   private class Connection {
708     private boolean versionRead = false; //if initial signature and
709                                          //version are read
710     private boolean headerRead = false;  //if the connection header that
711                                          //follows version is read.
712     protected SocketChannel channel;
713     private ByteBuffer data;
714     private ByteBuffer dataLengthBuffer;
715     protected final LinkedList<Call> responseQueue;
716     private volatile int rpcCount = 0; // number of outstanding rpcs
717     private long lastContact;
718     private int dataLength;
719     protected Socket socket;
720     // Cache the remote host & port info so that even if the socket is
721     // disconnected, we can say where it used to connect to.
722     private String hostAddress;
723     private int remotePort;
724     protected UserGroupInformation ticket = null;
725 
726     public Connection(SocketChannel channel, long lastContact) {
727       this.channel = channel;
728       this.lastContact = lastContact;
729       this.data = null;
730       this.dataLengthBuffer = ByteBuffer.allocate(4);
731       this.socket = channel.socket();
732       InetAddress addr = socket.getInetAddress();
733       if (addr == null) {
734         this.hostAddress = "*Unknown*";
735       } else {
736         this.hostAddress = addr.getHostAddress();
737       }
738       this.remotePort = socket.getPort();
739       this.responseQueue = new LinkedList<Call>();
740       if (socketSendBufferSize != 0) {
741         try {
742           socket.setSendBufferSize(socketSendBufferSize);
743         } catch (IOException e) {
744           LOG.warn("Connection: unable to set socket send buffer size to " +
745                    socketSendBufferSize);
746         }
747       }
748     }
749 
750     @Override
751     public String toString() {
752       return getHostAddress() + ":" + remotePort;
753     }
754 
755     public String getHostAddress() {
756       return hostAddress;
757     }
758 
759     public void setLastContact(long lastContact) {
760       this.lastContact = lastContact;
761     }
762 
763     public long getLastContact() {
764       return lastContact;
765     }
766 
767     /* Return true if the connection has no outstanding rpc */
768     private boolean isIdle() {
769       return rpcCount == 0;
770     }
771 
772     /* Decrement the outstanding RPC count */
773     protected void decRpcCount() {
774       rpcCount--;
775     }
776 
777     /* Increment the outstanding RPC count */
778     private void incRpcCount() {
779       rpcCount++;
780     }
781 
782     protected boolean timedOut(long currentTime) {
783       return isIdle() && currentTime - lastContact > maxIdleTime;
784     }
785 
786     public int readAndProcess() throws IOException, InterruptedException {
787       while (true) {
788         /* Read at most one RPC. If the header is not read completely yet
789          * then iterate until we read first RPC or until there is no data left.
790          */
791         int count;
792         if (dataLengthBuffer.remaining() > 0) {
793           count = channelRead(channel, dataLengthBuffer);
794           if (count < 0 || dataLengthBuffer.remaining() > 0)
795             return count;
796         }
797 
798         if (!versionRead) {
799           //Every connection is expected to send the header.
800           ByteBuffer versionBuffer = ByteBuffer.allocate(1);
801           count = channelRead(channel, versionBuffer);
802           if (count <= 0) {
803             return count;
804           }
805           int version = versionBuffer.get(0);
806 
807           dataLengthBuffer.flip();
808           if (!HEADER.equals(dataLengthBuffer) || version != CURRENT_VERSION) {
809             //Warning is ok since this is not supposed to happen.
810             LOG.warn("Incorrect header or version mismatch from " +
811                      hostAddress + ":" + remotePort +
812                      " got version " + version +
813                      " expected version " + CURRENT_VERSION);
814             return -1;
815           }
816           dataLengthBuffer.clear();
817           versionRead = true;
818           continue;
819         }
820 
821         if (data == null) {
822           dataLengthBuffer.flip();
823           dataLength = dataLengthBuffer.getInt();
824 
825           if (dataLength == HBaseClient.PING_CALL_ID) {
826             dataLengthBuffer.clear();
827             return 0;  //ping message
828           }
829           data = ByteBuffer.allocate(dataLength);
830           incRpcCount();  // Increment the rpc count
831         }
832 
833         count = channelRead(channel, data);
834 
835         if (data.remaining() == 0) {
836           dataLengthBuffer.clear();
837           data.flip();
838           if (headerRead) {
839             processData();
840             data = null;
841             return count;
842           }
843           processHeader();
844           headerRead = true;
845           data = null;
846           continue;
847         }
848         return count;
849       }
850     }
851 
852     /// Reads the header following version
853     private void processHeader() throws IOException {
854       /* In the current version, it is just a ticket.
855        * Later we could introduce a "ConnectionHeader" class.
856        */
857       DataInputStream in =
858         new DataInputStream(new ByteArrayInputStream(data.array()));
859       ticket = (UserGroupInformation) ObjectWritable.readObject(in, conf);
860     }
861 
862     private void processData() throws  IOException, InterruptedException {
863       DataInputStream dis =
864         new DataInputStream(new ByteArrayInputStream(data.array()));
865       int id = dis.readInt();                    // try to read an id
866 
867       if (LOG.isDebugEnabled())
868         LOG.debug(" got #" + id);
869 
870       Writable param = ReflectionUtils.newInstance(paramClass, conf);           // read param
871       param.readFields(dis);
872 
873       Call call = new Call(id, param, this);
874       callQueue.put(call);              // queue the call; maybe blocked here
875     }
876 
877     protected synchronized void close() {
878       data = null;
879       dataLengthBuffer = null;
880       if (!channel.isOpen())
881         return;
882       try {socket.shutdownOutput();} catch(Exception ignored) {} // FindBugs DE_MIGHT_IGNORE
883       if (channel.isOpen()) {
884         try {channel.close();} catch(Exception ignored) {}
885       }
886       try {socket.close();} catch(Exception ignored) {}
887     }
888   }
889 
890   /** Handles queued calls . */
891   private class Handler extends Thread {
892     public Handler(int instanceNumber) {
893       this.setDaemon(true);
894       this.setName("IPC Server handler "+ instanceNumber + " on " + port);
895     }
896 
897     @Override
898     public void run() {
899       LOG.info(getName() + ": starting");
900       SERVER.set(HBaseServer.this);
901       final int buffersize = 16 * 1024;
902       ByteArrayOutputStream buf = new ByteArrayOutputStream(buffersize);
903       while (running) {
904         try {
905           Call call = callQueue.take(); // pop the queue; maybe blocked here
906 
907           if (LOG.isDebugEnabled())
908             LOG.debug(getName() + ": has #" + call.id + " from " +
909                       call.connection);
910 
911           String errorClass = null;
912           String error = null;
913           Writable value = null;
914 
915           CurCall.set(call);
916           UserGroupInformation previous = UserGroupInformation.getCurrentUGI();
917           UserGroupInformation.setCurrentUser(call.connection.ticket);
918           try {
919             value = call(call.param, call.timestamp);             // make the call
920           } catch (Throwable e) {
921             LOG.debug(getName()+", call "+call+": error: " + e, e);
922             errorClass = e.getClass().getName();
923             error = StringUtils.stringifyException(e);
924           }
925           UserGroupInformation.setCurrentUser(previous);
926           CurCall.set(null);
927 
928           if (buf.size() > buffersize) {
929             // Allocate a new BAOS as reset only moves size back to zero but
930             // keeps the buffer of whatever the largest write was -- see
931             // hbase-900.
932             buf = new ByteArrayOutputStream(buffersize);
933           } else {
934             buf.reset();
935           }
936           DataOutputStream out = new DataOutputStream(buf);
937           out.writeInt(call.id);                // write call id
938           out.writeBoolean(error != null);      // write error flag
939 
940           if (error == null) {
941             value.write(out);
942           } else {
943             WritableUtils.writeString(out, errorClass);
944             WritableUtils.writeString(out, error);
945           }
946           call.setResponse(ByteBuffer.wrap(buf.toByteArray()));
947           responder.doRespond(call);
948         } catch (InterruptedException e) {
949           if (running) {                          // unexpected -- log it
950             LOG.info(getName() + " caught: " +
951                      StringUtils.stringifyException(e));
952           }
953         } catch (OutOfMemoryError e) {
954           if (errorHandler != null) {
955             if (errorHandler.checkOOME(e)) {
956               LOG.info(getName() + ": exiting on OOME");
957               return;
958             }
959           } else {
960             // rethrow if no handler
961             throw e;
962           }
963         } catch (Exception e) {
964           LOG.info(getName() + " caught: " +
965                    StringUtils.stringifyException(e));
966         }
967       }
968       LOG.info(getName() + ": exiting");
969     }
970 
971   }
972 
973   protected HBaseServer(String bindAddress, int port,
974                   Class<? extends Writable> paramClass, int handlerCount,
975                   Configuration conf)
976     throws IOException
977   {
978     this(bindAddress, port, paramClass, handlerCount,  conf, Integer.toString(port));
979   }
980   /* Constructs a server listening on the named port and address.  Parameters passed must
981    * be of the named class.  The <code>handlerCount</handlerCount> determines
982    * the number of handler threads that will be used to process calls.
983    *
984    */
985   protected HBaseServer(String bindAddress, int port,
986                   Class<? extends Writable> paramClass, int handlerCount,
987                   Configuration conf, String serverName)
988     throws IOException {
989     this.bindAddress = bindAddress;
990     this.conf = conf;
991     this.port = port;
992     this.paramClass = paramClass;
993     this.handlerCount = handlerCount;
994     this.socketSendBufferSize = 0;
995     this.maxQueueSize = handlerCount * MAX_QUEUE_SIZE_PER_HANDLER;
996     this.callQueue  = new LinkedBlockingQueue<Call>(maxQueueSize);
997     this.maxIdleTime = 2*conf.getInt("ipc.client.connection.maxidletime", 1000);
998     this.maxConnectionsToNuke = conf.getInt("ipc.client.kill.max", 10);
999     this.thresholdIdleConnections = conf.getInt("ipc.client.idlethreshold", 4000);
1000 
1001     // Start the listener here and let it bind to the port
1002     listener = new Listener();
1003     this.port = listener.getAddress().getPort();
1004     this.rpcMetrics = new HBaseRpcMetrics(serverName,
1005                           Integer.toString(this.port));
1006     this.tcpNoDelay = conf.getBoolean("ipc.server.tcpnodelay", false);
1007     this.tcpKeepAlive = conf.getBoolean("ipc.server.tcpkeepalive", true);
1008 
1009     // Create the responder here
1010     responder = new Responder();
1011   }
1012 
1013   protected void closeConnection(Connection connection) {
1014     synchronized (connectionList) {
1015       if (connectionList.remove(connection))
1016         numConnections--;
1017     }
1018     connection.close();
1019   }
1020 
1021   /** Sets the socket buffer size used for responding to RPCs.
1022    * @param size send size
1023    */
1024   public void setSocketSendBufSize(int size) { this.socketSendBufferSize = size; }
1025 
1026   /** Starts the service.  Must be called before any calls will be handled. */
1027   public synchronized void start() {
1028     responder.start();
1029     listener.start();
1030     handlers = new Handler[handlerCount];
1031 
1032     for (int i = 0; i < handlerCount; i++) {
1033       handlers[i] = new Handler(i);
1034       handlers[i].start();
1035     }
1036   }
1037 
1038   /** Stops the service.  No new calls will be handled after this is called. */
1039   public synchronized void stop() {
1040     LOG.info("Stopping server on " + port);
1041     running = false;
1042     if (handlers != null) {
1043       for (int i = 0; i < handlerCount; i++) {
1044         if (handlers[i] != null) {
1045           handlers[i].interrupt();
1046         }
1047       }
1048     }
1049     listener.interrupt();
1050     listener.doStop();
1051     responder.interrupt();
1052     notifyAll();
1053     if (this.rpcMetrics != null) {
1054       this.rpcMetrics.shutdown();
1055     }
1056   }
1057 
1058   /** Wait for the server to be stopped.
1059    * Does not wait for all subthreads to finish.
1060    *  See {@link #stop()}.
1061    * @throws InterruptedException e
1062    */
1063   public synchronized void join() throws InterruptedException {
1064     while (running) {
1065       wait();
1066     }
1067   }
1068 
1069   /**
1070    * Return the socket (ip+port) on which the RPC server is listening to.
1071    * @return the socket (ip+port) on which the RPC server is listening to.
1072    */
1073   public synchronized InetSocketAddress getListenerAddress() {
1074     return listener.getAddress();
1075   }
1076 
1077   /** Called for each call.
1078    * @param param writable parameter
1079    * @param receiveTime time
1080    * @return Writable
1081    * @throws IOException e
1082    */
1083   public abstract Writable call(Writable param, long receiveTime)
1084                                                 throws IOException;
1085 
1086   /**
1087    * The number of open RPC conections
1088    * @return the number of open rpc connections
1089    */
1090   public int getNumOpenConnections() {
1091     return numConnections;
1092   }
1093 
1094   /**
1095    * The number of rpc calls in the queue.
1096    * @return The number of rpc calls in the queue.
1097    */
1098   public int getCallQueueLen() {
1099     return callQueue.size();
1100   }
1101 
1102   /**
1103    * Set the handler for calling out of RPC for error conditions.
1104    * @param handler the handler implementation
1105    */
1106   public void setErrorHandler(HBaseRPCErrorHandler handler) {
1107     this.errorHandler = handler;
1108   }
1109 
1110   /**
1111    * When the read or write buffer size is larger than this limit, i/o will be
1112    * done in chunks of this size. Most RPC requests and responses would be
1113    * be smaller.
1114    */
1115   private static int NIO_BUFFER_LIMIT = 8*1024; //should not be more than 64KB.
1116 
1117   /**
1118    * This is a wrapper around {@link WritableByteChannel#write(ByteBuffer)}.
1119    * If the amount of data is large, it writes to channel in smaller chunks.
1120    * This is to avoid jdk from creating many direct buffers as the size of
1121    * buffer increases. This also minimizes extra copies in NIO layer
1122    * as a result of multiple write operations required to write a large
1123    * buffer.
1124    *
1125    * @param channel writable byte channel to write to
1126    * @param buffer buffer to write
1127    * @return number of bytes written
1128    * @throws java.io.IOException e
1129    * @see WritableByteChannel#write(ByteBuffer)
1130    */
1131   protected static int channelWrite(WritableByteChannel channel,
1132                                     ByteBuffer buffer) throws IOException {
1133     return (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
1134            channel.write(buffer) : channelIO(null, channel, buffer);
1135   }
1136 
1137   /**
1138    * This is a wrapper around {@link ReadableByteChannel#read(ByteBuffer)}.
1139    * If the amount of data is large, it writes to channel in smaller chunks.
1140    * This is to avoid jdk from creating many direct buffers as the size of
1141    * ByteBuffer increases. There should not be any performance degredation.
1142    *
1143    * @param channel writable byte channel to write on
1144    * @param buffer buffer to write
1145    * @return number of bytes written
1146    * @throws java.io.IOException e
1147    * @see ReadableByteChannel#read(ByteBuffer)
1148    */
1149   protected static int channelRead(ReadableByteChannel channel,
1150                                    ByteBuffer buffer) throws IOException {
1151     return (buffer.remaining() <= NIO_BUFFER_LIMIT) ?
1152            channel.read(buffer) : channelIO(channel, null, buffer);
1153   }
1154 
1155   /**
1156    * Helper for {@link #channelRead(ReadableByteChannel, ByteBuffer)}
1157    * and {@link #channelWrite(WritableByteChannel, ByteBuffer)}. Only
1158    * one of readCh or writeCh should be non-null.
1159    *
1160    * @param readCh read channel
1161    * @param writeCh write channel
1162    * @param buf buffer to read or write into/out of
1163    * @return bytes written
1164    * @throws java.io.IOException e
1165    * @see #channelRead(ReadableByteChannel, ByteBuffer)
1166    * @see #channelWrite(WritableByteChannel, ByteBuffer)
1167    */
1168   private static int channelIO(ReadableByteChannel readCh,
1169                                WritableByteChannel writeCh,
1170                                ByteBuffer buf) throws IOException {
1171 
1172     int originalLimit = buf.limit();
1173     int initialRemaining = buf.remaining();
1174     int ret = 0;
1175 
1176     while (buf.remaining() > 0) {
1177       try {
1178         int ioSize = Math.min(buf.remaining(), NIO_BUFFER_LIMIT);
1179         buf.limit(buf.position() + ioSize);
1180 
1181         ret = (readCh == null) ? writeCh.write(buf) : readCh.read(buf);
1182 
1183         if (ret < ioSize) {
1184           break;
1185         }
1186 
1187       } finally {
1188         buf.limit(originalLimit);
1189       }
1190     }
1191 
1192     int nBytes = initialRemaining - buf.remaining();
1193     return (nBytes > 0) ? nBytes : ret;
1194   }
1195 }