View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.ipc;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configuration;
26  import org.apache.hadoop.io.DataOutputBuffer;
27  import org.apache.hadoop.io.IOUtils;
28  import org.apache.hadoop.io.ObjectWritable;
29  import org.apache.hadoop.io.Writable;
30  import org.apache.hadoop.io.WritableUtils;
31  import org.apache.hadoop.ipc.RemoteException;
32  import org.apache.hadoop.net.NetUtils;
33  import org.apache.hadoop.security.UserGroupInformation;
34  import org.apache.hadoop.util.ReflectionUtils;
35  
36  import javax.net.SocketFactory;
37  import java.io.BufferedInputStream;
38  import java.io.BufferedOutputStream;
39  import java.io.DataInputStream;
40  import java.io.DataOutputStream;
41  import java.io.FilterInputStream;
42  import java.io.IOException;
43  import java.io.InputStream;
44  import java.net.ConnectException;
45  import java.net.InetSocketAddress;
46  import java.net.Socket;
47  import java.net.SocketTimeoutException;
48  import java.net.UnknownHostException;
49  import java.util.Hashtable;
50  import java.util.Iterator;
51  import java.util.Map.Entry;
52  import java.util.concurrent.atomic.AtomicBoolean;
53  import java.util.concurrent.atomic.AtomicLong;
54  
55  /** A client for an IPC service.  IPC calls take a single {@link Writable} as a
56   * parameter, and return a {@link Writable} as their value.  A service runs on
57   * a port and is defined by a parameter class and a value class.
58   *
59   * <p>This is the org.apache.hadoop.ipc.Client renamed as HBaseClient and
60   * moved into this package so can access package-private methods.
61   *
62   * @see HBaseServer
63   */
64  public class HBaseClient {
65  
66    private static final Log LOG =
67      LogFactory.getLog("org.apache.hadoop.ipc.HBaseClient");
68    protected final Hashtable<ConnectionId, Connection> connections =
69      new Hashtable<ConnectionId, Connection>();
70  
71    protected final Class<? extends Writable> valueClass;   // class of call values
72    protected int counter;                            // counter for call ids
73    protected final AtomicBoolean running = new AtomicBoolean(true); // if client runs
74    final protected Configuration conf;
75    final protected int maxIdleTime; // connections will be culled if it was idle for
76                             // maxIdleTime microsecs
77    final protected int maxRetries; //the max. no. of retries for socket connections
78    final protected long failureSleep; // Time to sleep before retry on failure.
79    protected final boolean tcpNoDelay; // if T then disable Nagle's Algorithm
80    protected final boolean tcpKeepAlive; // if T then use keepalives
81    protected final int pingInterval; // how often sends ping to the server in msecs
82  
83    protected final SocketFactory socketFactory;           // how to create sockets
84    private int refCount = 1;
85  
86    final private static String PING_INTERVAL_NAME = "ipc.ping.interval";
87    final static int DEFAULT_PING_INTERVAL = 60000; // 1 min
88    final static int PING_CALL_ID = -1;
89  
90    /**
91     * set the ping interval value in configuration
92     *
93     * @param conf Configuration
94     * @param pingInterval the ping interval
95     */
96    @SuppressWarnings({"UnusedDeclaration"})
97    public static void setPingInterval(Configuration conf, int pingInterval) {
98      conf.setInt(PING_INTERVAL_NAME, pingInterval);
99    }
100 
101   /**
102    * Get the ping interval from configuration;
103    * If not set in the configuration, return the default value.
104    *
105    * @param conf Configuration
106    * @return the ping interval
107    */
108   static int getPingInterval(Configuration conf) {
109     return conf.getInt(PING_INTERVAL_NAME, DEFAULT_PING_INTERVAL);
110   }
111 
112   /**
113    * Increment this client's reference count
114    *
115    */
116   synchronized void incCount() {
117     refCount++;
118   }
119 
120   /**
121    * Decrement this client's reference count
122    *
123    */
124   synchronized void decCount() {
125     refCount--;
126   }
127 
128   /**
129    * Return if this client has no reference
130    *
131    * @return true if this client has no reference; false otherwise
132    */
133   synchronized boolean isZeroReference() {
134     return refCount==0;
135   }
136 
137   /** A call waiting for a value. */
138   private class Call {
139     final int id;                                       // call id
140     final Writable param;                               // parameter
141     Writable value;                               // value, null if error
142     IOException error;                            // exception, null if value
143     boolean done;                                 // true when call is done
144 
145     protected Call(Writable param) {
146       this.param = param;
147       synchronized (HBaseClient.this) {
148         this.id = counter++;
149       }
150     }
151 
152     /** Indicate when the call is complete and the
153      * value or error are available.  Notifies by default.  */
154     protected synchronized void callComplete() {
155       this.done = true;
156       notify();                                 // notify caller
157     }
158 
159     /** Set the exception when there is an error.
160      * Notify the caller the call is done.
161      *
162      * @param error exception thrown by the call; either local or remote
163      */
164     public synchronized void setException(IOException error) {
165       this.error = error;
166       callComplete();
167     }
168 
169     /** Set the return value when there is no error.
170      * Notify the caller the call is done.
171      *
172      * @param value return value of the call.
173      */
174     public synchronized void setValue(Writable value) {
175       this.value = value;
176       callComplete();
177     }
178   }
179 
180   /** Thread that reads responses and notifies callers.  Each connection owns a
181    * socket connected to a remote address.  Calls are multiplexed through this
182    * socket: responses may be delivered out of order. */
183   private class Connection extends Thread {
184     private ConnectionId remoteId;
185     private Socket socket = null;                 // connected socket
186     private DataInputStream in;
187     private DataOutputStream out;
188 
189     // currently active calls
190     private final Hashtable<Integer, Call> calls = new Hashtable<Integer, Call>();
191     private final AtomicLong lastActivity = new AtomicLong();// last I/O activity time
192     protected final AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
193     private IOException closeException; // close reason
194 
195     public Connection(InetSocketAddress address) throws IOException {
196       this(new ConnectionId(address, null));
197     }
198 
199     public Connection(ConnectionId remoteId) throws IOException {
200       if (remoteId.getAddress().isUnresolved()) {
201         throw new UnknownHostException("unknown host: " +
202                                        remoteId.getAddress().getHostName());
203       }
204       this.remoteId = remoteId;
205       UserGroupInformation ticket = remoteId.getTicket();
206       this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
207         remoteId.getAddress().toString() +
208         ((ticket==null)?" from an unknown user": (" from " + ticket.getUserName())));
209       this.setDaemon(true);
210     }
211 
212     /** Update lastActivity with the current time. */
213     private void touch() {
214       lastActivity.set(System.currentTimeMillis());
215     }
216 
217     /**
218      * Add a call to this connection's call queue and notify
219      * a listener; synchronized.
220      * Returns false if called during shutdown.
221      * @param call to add
222      * @return true if the call was added.
223      */
224     protected synchronized boolean addCall(Call call) {
225       if (shouldCloseConnection.get())
226         return false;
227       calls.put(call.id, call);
228       notify();
229       return true;
230     }
231 
232     /** This class sends a ping to the remote side when timeout on
233      * reading. If no failure is detected, it retries until at least
234      * a byte is read.
235      */
236     private class PingInputStream extends FilterInputStream {
237       /* constructor */
238       protected PingInputStream(InputStream in) {
239         super(in);
240       }
241 
242       /* Process timeout exception
243        * if the connection is not going to be closed, send a ping.
244        * otherwise, throw the timeout exception.
245        */
246       private void handleTimeout(SocketTimeoutException e) throws IOException {
247         if (shouldCloseConnection.get() || !running.get()) {
248           throw e;
249         }
250         sendPing();
251       }
252 
253       /** Read a byte from the stream.
254        * Send a ping if timeout on read. Retries if no failure is detected
255        * until a byte is read.
256        * @throws IOException for any IO problem other than socket timeout
257        */
258       @Override
259       public int read() throws IOException {
260         do {
261           try {
262             return super.read();
263           } catch (SocketTimeoutException e) {
264             handleTimeout(e);
265           }
266         } while (true);
267       }
268 
269       /** Read bytes into a buffer starting from offset <code>off</code>
270        * Send a ping if timeout on read. Retries if no failure is detected
271        * until a byte is read.
272        *
273        * @return the total number of bytes read; -1 if the connection is closed.
274        */
275       @Override
276       public int read(byte[] buf, int off, int len) throws IOException {
277         do {
278           try {
279             return super.read(buf, off, len);
280           } catch (SocketTimeoutException e) {
281             handleTimeout(e);
282           }
283         } while (true);
284       }
285     }
286 
287     /** Connect to the server and set up the I/O streams. It then sends
288      * a header to the server and starts
289      * the connection thread that waits for responses.
290      * @throws java.io.IOException e
291      */
292     protected synchronized void setupIOstreams() throws IOException {
293       if (socket != null || shouldCloseConnection.get()) {
294         return;
295       }
296 
297       short ioFailures = 0;
298       short timeoutFailures = 0;
299       try {
300         if (LOG.isDebugEnabled()) {
301           LOG.debug("Connecting to "+remoteId.getAddress());
302         }
303         while (true) {
304           try {
305             this.socket = socketFactory.createSocket();
306             this.socket.setTcpNoDelay(tcpNoDelay);
307             this.socket.setKeepAlive(tcpKeepAlive);
308             // connection time out is 20s
309             NetUtils.connect(this.socket, remoteId.getAddress(), 20000);
310             this.socket.setSoTimeout(pingInterval);
311             break;
312           } catch (SocketTimeoutException toe) {
313             handleConnectionFailure(timeoutFailures++, maxRetries, toe);
314           } catch (IOException ie) {
315             handleConnectionFailure(ioFailures++, maxRetries, ie);
316           }
317         }
318         this.in = new DataInputStream(new BufferedInputStream
319             (new PingInputStream(NetUtils.getInputStream(socket))));
320         this.out = new DataOutputStream
321             (new BufferedOutputStream(NetUtils.getOutputStream(socket)));
322         writeHeader();
323 
324         // update last activity time
325         touch();
326 
327         // start the receiver thread after the socket connection has been set up
328         start();
329       } catch (IOException e) {
330         markClosed(e);
331         close();
332 
333         throw e;
334       }
335     }
336 
337     /* Handle connection failures
338      *
339      * If the current number of retries is equal to the max number of retries,
340      * stop retrying and throw the exception; Otherwise backoff N seconds and
341      * try connecting again.
342      *
343      * This Method is only called from inside setupIOstreams(), which is
344      * synchronized. Hence the sleep is synchronized; the locks will be retained.
345      *
346      * @param curRetries current number of retries
347      * @param maxRetries max number of retries allowed
348      * @param ioe failure reason
349      * @throws IOException if max number of retries is reached
350      */
351     private void handleConnectionFailure(
352         int curRetries, int maxRetries, IOException ioe) throws IOException {
353       // close the current connection
354       if (socket != null) { // could be null if the socket creation failed
355         try {
356           socket.close();
357         } catch (IOException e) {
358           LOG.warn("Not able to close a socket", e);
359         }
360       }
361       // set socket to null so that the next call to setupIOstreams
362       // can start the process of connect all over again.
363       socket = null;
364 
365       // throw the exception if the maximum number of retries is reached
366       if (curRetries >= maxRetries) {
367         throw ioe;
368       }
369 
370       // otherwise back off and retry
371       try {
372         Thread.sleep(failureSleep);
373       } catch (InterruptedException ignored) {}
374 
375       LOG.info("Retrying connect to server: " + remoteId.getAddress() +
376         " after sleeping " + failureSleep + "ms. Already tried " + curRetries +
377         " time(s).");
378     }
379 
380     /* Write the header for each connection
381      * Out is not synchronized because only the first thread does this.
382      */
383     private void writeHeader() throws IOException {
384       out.write(HBaseServer.HEADER.array());
385       out.write(HBaseServer.CURRENT_VERSION);
386       //When there are more fields we can have ConnectionHeader Writable.
387       DataOutputBuffer buf = new DataOutputBuffer();
388       ObjectWritable.writeObject(buf, remoteId.getTicket(),
389                                  UserGroupInformation.class, conf);
390       int bufLen = buf.getLength();
391       out.writeInt(bufLen);
392       out.write(buf.getData(), 0, bufLen);
393     }
394 
395     /* wait till someone signals us to start reading RPC response or
396      * it is idle too long, it is marked as to be closed,
397      * or the client is marked as not running.
398      *
399      * Return true if it is time to read a response; false otherwise.
400      */
401     @SuppressWarnings({"ThrowableInstanceNeverThrown"})
402     private synchronized boolean waitForWork() {
403       if (calls.isEmpty() && !shouldCloseConnection.get()  && running.get())  {
404         long timeout = maxIdleTime-
405               (System.currentTimeMillis()-lastActivity.get());
406         if (timeout>0) {
407           try {
408             wait(timeout);
409           } catch (InterruptedException ignored) {}
410         }
411       }
412 
413       if (!calls.isEmpty() && !shouldCloseConnection.get() && running.get()) {
414         return true;
415       } else if (shouldCloseConnection.get()) {
416         return false;
417       } else if (calls.isEmpty()) { // idle connection closed or stopped
418         markClosed(null);
419         return false;
420       } else { // get stopped but there are still pending requests
421         markClosed((IOException)new IOException().initCause(
422             new InterruptedException()));
423         return false;
424       }
425     }
426 
427     public InetSocketAddress getRemoteAddress() {
428       return remoteId.getAddress();
429     }
430 
431     /* Send a ping to the server if the time elapsed
432      * since last I/O activity is equal to or greater than the ping interval
433      */
434     protected synchronized void sendPing() throws IOException {
435       long curTime = System.currentTimeMillis();
436       if ( curTime - lastActivity.get() >= pingInterval) {
437         lastActivity.set(curTime);
438         //noinspection SynchronizeOnNonFinalField
439         synchronized (this.out) {
440           out.writeInt(PING_CALL_ID);
441           out.flush();
442         }
443       }
444     }
445 
446     @Override
447     public void run() {
448       if (LOG.isDebugEnabled())
449         LOG.debug(getName() + ": starting, having connections "
450             + connections.size());
451 
452       try {
453         while (waitForWork()) {//wait here for work - read or close connection
454           receiveResponse();
455         }
456       } catch (Throwable t) {
457         LOG.warn("Unexpected exception receiving call responses", t);
458         markClosed(new IOException("Unexpected exception receiving call responses", t));
459       }
460 
461       close();
462 
463       if (LOG.isDebugEnabled())
464         LOG.debug(getName() + ": stopped, remaining connections "
465             + connections.size());
466     }
467 
468     /* Initiates a call by sending the parameter to the remote server.
469      * Note: this is not called from the Connection thread, but by other
470      * threads.
471      */
472     protected void sendParam(Call call) {
473       if (shouldCloseConnection.get()) {
474         return;
475       }
476 
477       DataOutputBuffer d=null;
478       try {
479         //noinspection SynchronizeOnNonFinalField
480         synchronized (this.out) { // FindBugs IS2_INCONSISTENT_SYNC
481           if (LOG.isDebugEnabled())
482             LOG.debug(getName() + " sending #" + call.id);
483 
484           //for serializing the
485           //data to be written
486           d = new DataOutputBuffer();
487           d.writeInt(call.id);
488           call.param.write(d);
489           byte[] data = d.getData();
490           int dataLength = d.getLength();
491           out.writeInt(dataLength);      //first put the data length
492           out.write(data, 0, dataLength);//write the data
493           out.flush();
494         }
495       } catch(IOException e) {
496         markClosed(e);
497       } finally {
498         //the buffer is just an in-memory buffer, but it is still polite to
499         // close early
500         IOUtils.closeStream(d);
501       }
502     }
503 
504     /* Receive a response.
505      * Because only one receiver, so no synchronization on in.
506      */
507     private void receiveResponse() {
508       if (shouldCloseConnection.get()) {
509         return;
510       }
511       touch();
512 
513       try {
514         int id = in.readInt();                    // try to read an id
515 
516         if (LOG.isDebugEnabled())
517           LOG.debug(getName() + " got value #" + id);
518 
519         Call call = calls.get(id);
520 
521         boolean isError = in.readBoolean();     // read if error
522         if (isError) {
523           //noinspection ThrowableInstanceNeverThrown
524           call.setException(new RemoteException( WritableUtils.readString(in),
525               WritableUtils.readString(in)));
526           calls.remove(id);
527         } else {
528           Writable value = ReflectionUtils.newInstance(valueClass, conf);
529           value.readFields(in);                 // read value
530           call.setValue(value);
531           calls.remove(id);
532         }
533       } catch (IOException e) {
534         markClosed(e);
535       }
536     }
537 
538     private synchronized void markClosed(IOException e) {
539       if (shouldCloseConnection.compareAndSet(false, true)) {
540         closeException = e;
541         notifyAll();
542       }
543     }
544 
545     /** Close the connection. */
546     private synchronized void close() {
547       if (!shouldCloseConnection.get()) {
548         LOG.error("The connection is not in the closed state");
549         return;
550       }
551 
552       // release the resources
553       // first thing to do;take the connection out of the connection list
554       synchronized (connections) {
555         if (connections.get(remoteId) == this) {
556           connections.remove(remoteId);
557         }
558       }
559 
560       // close the streams and therefore the socket
561       IOUtils.closeStream(out);
562       IOUtils.closeStream(in);
563 
564       // clean up all calls
565       if (closeException == null) {
566         if (!calls.isEmpty()) {
567           LOG.warn(
568               "A connection is closed for no cause and calls are not empty");
569 
570           // clean up calls anyway
571           closeException = new IOException("Unexpected closed connection");
572           cleanupCalls();
573         }
574       } else {
575         // log the info
576         if (LOG.isDebugEnabled()) {
577           LOG.debug("closing ipc connection to " + remoteId.address + ": " +
578               closeException.getMessage(),closeException);
579         }
580 
581         // cleanup calls
582         cleanupCalls();
583       }
584       if (LOG.isDebugEnabled())
585         LOG.debug(getName() + ": closed");
586     }
587 
588     /* Cleanup all calls and mark them as done */
589     private void cleanupCalls() {
590       Iterator<Entry<Integer, Call>> itor = calls.entrySet().iterator() ;
591       while (itor.hasNext()) {
592         Call c = itor.next().getValue();
593         c.setException(closeException); // local exception
594         itor.remove();
595       }
596     }
597   }
598 
599   /** Call implementation used for parallel calls. */
600   private class ParallelCall extends Call {
601     private final ParallelResults results;
602     protected final int index;
603 
604     public ParallelCall(Writable param, ParallelResults results, int index) {
605       super(param);
606       this.results = results;
607       this.index = index;
608     }
609 
610     /** Deliver result to result collector. */
611     @Override
612     protected void callComplete() {
613       results.callComplete(this);
614     }
615   }
616 
617   /** Result collector for parallel calls. */
618   private static class ParallelResults {
619     protected final Writable[] values;
620     protected int size;
621     protected int count;
622 
623     public ParallelResults(int size) {
624       this.values = new Writable[size];
625       this.size = size;
626     }
627 
628     /*
629      * Collect a result.
630      */
631     synchronized void callComplete(ParallelCall call) {
632       // FindBugs IS2_INCONSISTENT_SYNC
633       values[call.index] = call.value;            // store the value
634       count++;                                    // count it
635       if (count == size)                          // if all values are in
636         notify();                                 // then notify waiting caller
637     }
638   }
639 
640   /**
641    * Construct an IPC client whose values are of the given {@link Writable}
642    * class.
643    * @param valueClass value class
644    * @param conf configuration
645    * @param factory socket factory
646    */
647   public HBaseClient(Class<? extends Writable> valueClass, Configuration conf,
648       SocketFactory factory) {
649     this.valueClass = valueClass;
650     this.maxIdleTime =
651       conf.getInt("hbase.ipc.client.connection.maxidletime", 10000); //10s
652     this.maxRetries = conf.getInt("hbase.ipc.client.connect.max.retries", 0);
653     this.failureSleep = conf.getInt("hbase.client.pause", 2000);
654     this.tcpNoDelay = conf.getBoolean("hbase.ipc.client.tcpnodelay", false);
655     this.tcpKeepAlive = conf.getBoolean("hbase.ipc.client.tcpkeepalive", true);
656     this.pingInterval = getPingInterval(conf);
657     if (LOG.isDebugEnabled()) {
658       LOG.debug("The ping interval is" + this.pingInterval + "ms.");
659     }
660     this.conf = conf;
661     this.socketFactory = factory;
662   }
663 
664   /**
665    * Construct an IPC client with the default SocketFactory
666    * @param valueClass value class
667    * @param conf configuration
668    */
669   public HBaseClient(Class<? extends Writable> valueClass, Configuration conf) {
670     this(valueClass, conf, NetUtils.getDefaultSocketFactory(conf));
671   }
672 
673   /** Return the socket factory of this client
674    *
675    * @return this client's socket factory
676    */
677   SocketFactory getSocketFactory() {
678     return socketFactory;
679   }
680 
681   /** Stop all threads related to this client.  No further calls may be made
682    * using this client. */
683   public void stop() {
684     if (LOG.isDebugEnabled()) {
685       LOG.debug("Stopping client");
686     }
687 
688     if (!running.compareAndSet(true, false)) {
689       return;
690     }
691 
692     // wake up all connections
693     synchronized (connections) {
694       for (Connection conn : connections.values()) {
695         conn.interrupt();
696       }
697     }
698 
699     // wait until all connections are closed
700     while (!connections.isEmpty()) {
701       try {
702         Thread.sleep(100);
703       } catch (InterruptedException ignored) {
704       }
705     }
706   }
707 
708   /** Make a call, passing <code>param</code>, to the IPC server running at
709    * <code>address</code>, returning the value.  Throws exceptions if there are
710    * network problems or if the remote code threw an exception.
711    * @param param writable parameter
712    * @param address network address
713    * @return Writable
714    * @throws IOException e
715    */
716   public Writable call(Writable param, InetSocketAddress address)
717   throws IOException {
718       return call(param, address, null);
719   }
720 
721   public Writable call(Writable param, InetSocketAddress addr,
722                        UserGroupInformation ticket)
723                        throws IOException {
724     Call call = new Call(param);
725     Connection connection = getConnection(addr, ticket, call);
726     connection.sendParam(call);                 // send the parameter
727     boolean interrupted = false;
728     //noinspection SynchronizationOnLocalVariableOrMethodParameter
729     synchronized (call) {
730       while (!call.done) {
731         try {
732           call.wait();                           // wait for the result
733         } catch (InterruptedException ignored) {
734           // save the fact that we were interrupted
735           interrupted = true;
736         }
737       }
738 
739       if (interrupted) {
740         // set the interrupt flag now that we are done waiting
741         Thread.currentThread().interrupt();
742       }
743 
744       if (call.error != null) {
745         if (call.error instanceof RemoteException) {
746           call.error.fillInStackTrace();
747           throw call.error;
748         }
749         // local exception
750         throw wrapException(addr, call.error);
751       }
752       return call.value;
753     }
754   }
755 
756   /**
757    * Take an IOException and the address we were trying to connect to
758    * and return an IOException with the input exception as the cause.
759    * The new exception provides the stack trace of the place where
760    * the exception is thrown and some extra diagnostics information.
761    * If the exception is ConnectException or SocketTimeoutException,
762    * return a new one of the same type; Otherwise return an IOException.
763    *
764    * @param addr target address
765    * @param exception the relevant exception
766    * @return an exception to throw
767    */
768   @SuppressWarnings({"ThrowableInstanceNeverThrown"})
769   private IOException wrapException(InetSocketAddress addr,
770                                          IOException exception) {
771     if (exception instanceof ConnectException) {
772       //connection refused; include the host:port in the error
773       return (ConnectException)new ConnectException(
774            "Call to " + addr + " failed on connection exception: " + exception)
775                     .initCause(exception);
776     } else if (exception instanceof SocketTimeoutException) {
777       return (SocketTimeoutException)new SocketTimeoutException(
778            "Call to " + addr + " failed on socket timeout exception: "
779                       + exception).initCause(exception);
780     } else {
781       return (IOException)new IOException(
782            "Call to " + addr + " failed on local exception: " + exception)
783                                  .initCause(exception);
784 
785     }
786   }
787 
788   /** Makes a set of calls in parallel.  Each parameter is sent to the
789    * corresponding address.  When all values are available, or have timed out
790    * or errored, the collected results are returned in an array.  The array
791    * contains nulls for calls that timed out or errored.
792    * @param params writable parameters
793    * @param addresses socket addresses
794    * @return  Writable[]
795    * @throws IOException e
796    */
797   public Writable[] call(Writable[] params, InetSocketAddress[] addresses)
798     throws IOException {
799     if (addresses.length == 0) return new Writable[0];
800 
801     ParallelResults results = new ParallelResults(params.length);
802     // TODO this synchronization block doesnt make any sense, we should possibly fix it
803     //noinspection SynchronizationOnLocalVariableOrMethodParameter
804     synchronized (results) {
805       for (int i = 0; i < params.length; i++) {
806         ParallelCall call = new ParallelCall(params[i], results, i);
807         try {
808           Connection connection = getConnection(addresses[i], null, call);
809           connection.sendParam(call);             // send each parameter
810         } catch (IOException e) {
811           // log errors
812           LOG.info("Calling "+addresses[i]+" caught: " +
813                    e.getMessage(),e);
814           results.size--;                         //  wait for one fewer result
815         }
816       }
817       while (results.count != results.size) {
818         try {
819           results.wait();                    // wait for all results
820         } catch (InterruptedException ignored) {}
821       }
822 
823       return results.values;
824     }
825   }
826 
827   /* Get a connection from the pool, or create a new one and add it to the
828    * pool.  Connections to a given host/port are reused. */
829   private Connection getConnection(InetSocketAddress addr,
830                                    UserGroupInformation ticket,
831                                    Call call)
832                                    throws IOException {
833     if (!running.get()) {
834       // the client is stopped
835       throw new IOException("The client is stopped");
836     }
837     Connection connection;
838     /* we could avoid this allocation for each RPC by having a
839      * connectionsId object and with set() method. We need to manage the
840      * refs for keys in HashMap properly. For now its ok.
841      */
842     ConnectionId remoteId = new ConnectionId(addr, ticket);
843     do {
844       synchronized (connections) {
845         connection = connections.get(remoteId);
846         if (connection == null) {
847           connection = new Connection(remoteId);
848           connections.put(remoteId, connection);
849         }
850       }
851     } while (!connection.addCall(call));
852 
853     //we don't invoke the method below inside "synchronized (connections)"
854     //block above. The reason for that is if the server happens to be slow,
855     //it will take longer to establish a connection and that will slow the
856     //entire system down.
857     connection.setupIOstreams();
858     return connection;
859   }
860 
861   /**
862    * This class holds the address and the user ticket. The client connections
863    * to servers are uniquely identified by <remoteAddress, ticket>
864    */
865   private static class ConnectionId {
866     final InetSocketAddress address;
867     final UserGroupInformation ticket;
868 
869     ConnectionId(InetSocketAddress address, UserGroupInformation ticket) {
870       this.address = address;
871       this.ticket = ticket;
872     }
873 
874     InetSocketAddress getAddress() {
875       return address;
876     }
877     UserGroupInformation getTicket() {
878       return ticket;
879     }
880 
881     @Override
882     public boolean equals(Object obj) {
883      if (obj instanceof ConnectionId) {
884        ConnectionId id = (ConnectionId) obj;
885        return address.equals(id.address) && ticket == id.ticket;
886        //Note : ticket is a ref comparision.
887      }
888      return false;
889     }
890 
891     @Override
892     public int hashCode() {
893       return address.hashCode() ^ System.identityHashCode(ticket);
894     }
895   }
896 }