View Javadoc

1   /**
2    * Copyright 2010 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.ipc;
22  
23  import org.apache.commons.logging.Log;
24  import org.apache.commons.logging.LogFactory;
25  import org.apache.hadoop.conf.Configurable;
26  import org.apache.hadoop.conf.Configuration;
27  import org.apache.hadoop.hbase.client.RetriesExhaustedException;
28  import org.apache.hadoop.hbase.io.HbaseObjectWritable;
29  import org.apache.hadoop.io.Writable;
30  import org.apache.hadoop.ipc.VersionedProtocol;
31  import org.apache.hadoop.net.NetUtils;
32  import org.apache.hadoop.security.UserGroupInformation;
33  
34  import javax.net.SocketFactory;
35  import java.io.DataInput;
36  import java.io.DataOutput;
37  import java.io.IOException;
38  import java.lang.reflect.Array;
39  import java.lang.reflect.InvocationHandler;
40  import java.lang.reflect.InvocationTargetException;
41  import java.lang.reflect.Method;
42  import java.lang.reflect.Proxy;
43  import java.net.ConnectException;
44  import java.net.InetSocketAddress;
45  import java.net.SocketTimeoutException;
46  import java.util.HashMap;
47  import java.util.Map;
48  
49  /** A simple RPC mechanism.
50   *
51   * This is a local hbase copy of the hadoop RPC so we can do things like
52   * address HADOOP-414 for hbase-only and try other hbase-specific
53   * optimizations like using our own version of ObjectWritable.  Class has been
54   * renamed to avoid confusing it w/ hadoop versions.
55   * <p>
56   *
57   *
58   * A <i>protocol</i> is a Java interface.  All parameters and return types must
59   * be one of:
60   *
61   * <ul> <li>a primitive type, <code>boolean</code>, <code>byte</code>,
62   * <code>char</code>, <code>short</code>, <code>int</code>, <code>long</code>,
63   * <code>float</code>, <code>double</code>, or <code>void</code>; or</li>
64   *
65   * <li>a {@link String}; or</li>
66   *
67   * <li>a {@link Writable}; or</li>
68   *
69   * <li>an array of the above types</li> </ul>
70   *
71   * All methods in the protocol should throw only IOException.  No field data of
72   * the protocol instance is transmitted.
73   */
74  public class HBaseRPC {
75    // Leave this out in the hadoop ipc package but keep class name.  Do this
76    // so that we dont' get the logging of this class's invocations by doing our
77    // blanket enabling DEBUG on the o.a.h.h. package.
78    protected static final Log LOG =
79      LogFactory.getLog("org.apache.hadoop.ipc.HbaseRPC");
80  
81    private HBaseRPC() {
82      super();
83    }                                  // no public ctor
84  
85    /** A method invocation, including the method name and its parameters.*/
86    private static class Invocation implements Writable, Configurable {
87      private String methodName;
88      @SuppressWarnings("unchecked")
89      private Class[] parameterClasses;
90      private Object[] parameters;
91      private Configuration conf;
92  
93      /** default constructor */
94      public Invocation() {
95        super();
96      }
97  
98      /**
99       * @param method method to call
100      * @param parameters parameters of call
101      */
102     public Invocation(Method method, Object[] parameters) {
103       this.methodName = method.getName();
104       this.parameterClasses = method.getParameterTypes();
105       this.parameters = parameters;
106     }
107 
108     /** @return The name of the method invoked. */
109     public String getMethodName() { return methodName; }
110 
111     /** @return The parameter classes. */
112     @SuppressWarnings("unchecked")
113     public Class[] getParameterClasses() { return parameterClasses; }
114 
115     /** @return The parameter instances. */
116     public Object[] getParameters() { return parameters; }
117 
118     public void readFields(DataInput in) throws IOException {
119       methodName = in.readUTF();
120       parameters = new Object[in.readInt()];
121       parameterClasses = new Class[parameters.length];
122       HbaseObjectWritable objectWritable = new HbaseObjectWritable();
123       for (int i = 0; i < parameters.length; i++) {
124         parameters[i] = HbaseObjectWritable.readObject(in, objectWritable,
125           this.conf);
126         parameterClasses[i] = objectWritable.getDeclaredClass();
127       }
128     }
129 
130     public void write(DataOutput out) throws IOException {
131       out.writeUTF(this.methodName);
132       out.writeInt(parameterClasses.length);
133       for (int i = 0; i < parameterClasses.length; i++) {
134         HbaseObjectWritable.writeObject(out, parameters[i], parameterClasses[i],
135                                    conf);
136       }
137     }
138 
139     @Override
140     public String toString() {
141       StringBuilder buffer = new StringBuilder(256);
142       buffer.append(methodName);
143       buffer.append("(");
144       for (int i = 0; i < parameters.length; i++) {
145         if (i != 0)
146           buffer.append(", ");
147         buffer.append(parameters[i]);
148       }
149       buffer.append(")");
150       return buffer.toString();
151     }
152 
153     public void setConf(Configuration conf) {
154       this.conf = conf;
155     }
156 
157     public Configuration getConf() {
158       return this.conf;
159     }
160   }
161 
162   /* Cache a client using its socket factory as the hash key */
163   static private class ClientCache {
164     private Map<SocketFactory, HBaseClient> clients =
165       new HashMap<SocketFactory, HBaseClient>();
166 
167     protected ClientCache() {}
168 
169     /**
170      * Construct & cache an IPC client with the user-provided SocketFactory
171      * if no cached client exists.
172      *
173      * @param conf Configuration
174      * @param factory socket factory
175      * @return an IPC client
176      */
177     protected synchronized HBaseClient getClient(Configuration conf,
178         SocketFactory factory) {
179       // Construct & cache client.  The configuration is only used for timeout,
180       // and Clients have connection pools.  So we can either (a) lose some
181       // connection pooling and leak sockets, or (b) use the same timeout for all
182       // configurations.  Since the IPC is usually intended globally, not
183       // per-job, we choose (a).
184       HBaseClient client = clients.get(factory);
185       if (client == null) {
186         // Make an hbase client instead of hadoop Client.
187         client = new HBaseClient(HbaseObjectWritable.class, conf, factory);
188         clients.put(factory, client);
189       } else {
190         client.incCount();
191       }
192       return client;
193     }
194 
195     /**
196      * Construct & cache an IPC client with the default SocketFactory
197      * if no cached client exists.
198      *
199      * @param conf Configuration
200      * @return an IPC client
201      */
202     protected synchronized HBaseClient getClient(Configuration conf) {
203       return getClient(conf, SocketFactory.getDefault());
204     }
205 
206     /**
207      * Stop a RPC client connection
208      * A RPC client is closed only when its reference count becomes zero.
209      * @param client client to stop
210      */
211     protected void stopClient(HBaseClient client) {
212       synchronized (this) {
213         client.decCount();
214         if (client.isZeroReference()) {
215           clients.remove(client.getSocketFactory());
216         }
217       }
218       if (client.isZeroReference()) {
219         client.stop();
220       }
221     }
222   }
223 
224   protected final static ClientCache CLIENTS = new ClientCache();
225 
226   private static class Invoker implements InvocationHandler {
227     private InetSocketAddress address;
228     private UserGroupInformation ticket;
229     private HBaseClient client;
230     private boolean isClosed = false;
231 
232     /**
233      * @param address address for invoker
234      * @param ticket ticket
235      * @param conf configuration
236      * @param factory socket factory
237      */
238     public Invoker(InetSocketAddress address, UserGroupInformation ticket,
239                    Configuration conf, SocketFactory factory) {
240       this.address = address;
241       this.ticket = ticket;
242       this.client = CLIENTS.getClient(conf, factory);
243     }
244 
245     public Object invoke(Object proxy, Method method, Object[] args)
246         throws Throwable {
247       final boolean logDebug = LOG.isDebugEnabled();
248       long startTime = 0;
249       if (logDebug) {
250         startTime = System.currentTimeMillis();
251       }
252       HbaseObjectWritable value = (HbaseObjectWritable)
253         client.call(new Invocation(method, args), address, ticket);
254       if (logDebug) {
255         long callTime = System.currentTimeMillis() - startTime;
256         LOG.debug("Call: " + method.getName() + " " + callTime);
257       }
258       return value.get();
259     }
260 
261     /* close the IPC client that's responsible for this invoker's RPCs */
262     synchronized protected void close() {
263       if (!isClosed) {
264         isClosed = true;
265         CLIENTS.stopClient(client);
266       }
267     }
268   }
269 
270   /**
271    * A version mismatch for the RPC protocol.
272    */
273   @SuppressWarnings("serial")
274   public static class VersionMismatch extends IOException {
275     private String interfaceName;
276     private long clientVersion;
277     private long serverVersion;
278 
279     /**
280      * Create a version mismatch exception
281      * @param interfaceName the name of the protocol mismatch
282      * @param clientVersion the client's version of the protocol
283      * @param serverVersion the server's version of the protocol
284      */
285     public VersionMismatch(String interfaceName, long clientVersion,
286                            long serverVersion) {
287       super("Protocol " + interfaceName + " version mismatch. (client = " +
288             clientVersion + ", server = " + serverVersion + ")");
289       this.interfaceName = interfaceName;
290       this.clientVersion = clientVersion;
291       this.serverVersion = serverVersion;
292     }
293 
294     /**
295      * Get the interface name
296      * @return the java class name
297      *          (eg. org.apache.hadoop.mapred.InterTrackerProtocol)
298      */
299     public String getInterfaceName() {
300       return interfaceName;
301     }
302 
303     /**
304      * @return the client's preferred version
305      */
306     public long getClientVersion() {
307       return clientVersion;
308     }
309 
310     /**
311      * @return the server's agreed to version.
312      */
313     public long getServerVersion() {
314       return serverVersion;
315     }
316   }
317 
318   /**
319    * @param protocol protocol interface
320    * @param clientVersion which client version we expect
321    * @param addr address of remote service
322    * @param conf configuration
323    * @param maxAttempts max attempts
324    * @param timeout timeout in milliseconds
325    * @return proxy
326    * @throws IOException e
327    */
328   @SuppressWarnings("unchecked")
329   public static VersionedProtocol waitForProxy(Class protocol,
330                                                long clientVersion,
331                                                InetSocketAddress addr,
332                                                Configuration conf,
333                                                int maxAttempts,
334                                                long timeout
335                                                ) throws IOException {
336     // HBase does limited number of reconnects which is different from hadoop.
337     long startTime = System.currentTimeMillis();
338     IOException ioe;
339     int reconnectAttempts = 0;
340     while (true) {
341       try {
342         return getProxy(protocol, clientVersion, addr, conf);
343       } catch(ConnectException se) {  // namenode has not been started
344         ioe = se;
345         if (maxAttempts >= 0 && ++reconnectAttempts >= maxAttempts) {
346           LOG.info("Server at " + addr + " could not be reached after " +
347             reconnectAttempts + " tries, giving up.");
348           throw new RetriesExhaustedException("Failed setting up proxy to " +
349             addr.toString() + " after attempts=" + reconnectAttempts);
350       }
351       } catch(SocketTimeoutException te) {  // namenode is busy
352         LOG.info("Problem connecting to server: " + addr);
353         ioe = te;
354       }
355       // check if timed out
356       if (System.currentTimeMillis()-timeout >= startTime) {
357         throw ioe;
358       }
359 
360       // wait for retry
361       try {
362         Thread.sleep(1000);
363       } catch (InterruptedException ie) {
364         // IGNORE
365       }
366     }
367   }
368 
369   /**
370    * Construct a client-side proxy object that implements the named protocol,
371    * talking to a server at the named address.
372    *
373    * @param protocol interface
374    * @param clientVersion version we are expecting
375    * @param addr remote address
376    * @param conf configuration
377    * @param factory socket factory
378    * @return proxy
379    * @throws IOException e
380    */
381   public static VersionedProtocol getProxy(Class<?> protocol,
382       long clientVersion, InetSocketAddress addr, Configuration conf,
383       SocketFactory factory) throws IOException {
384     return getProxy(protocol, clientVersion, addr, null, conf, factory);
385   }
386 
387   /**
388    * Construct a client-side proxy object that implements the named protocol,
389    * talking to a server at the named address.
390    *
391    * @param protocol interface
392    * @param clientVersion version we are expecting
393    * @param addr remote address
394    * @param ticket ticket
395    * @param conf configuration
396    * @param factory socket factory
397    * @return proxy
398    * @throws IOException e
399    */
400   public static VersionedProtocol getProxy(Class<?> protocol,
401       long clientVersion, InetSocketAddress addr, UserGroupInformation ticket,
402       Configuration conf, SocketFactory factory)
403   throws IOException {
404     VersionedProtocol proxy =
405         (VersionedProtocol) Proxy.newProxyInstance(
406             protocol.getClassLoader(), new Class[] { protocol },
407             new Invoker(addr, ticket, conf, factory));
408     long serverVersion = proxy.getProtocolVersion(protocol.getName(),
409                                                   clientVersion);
410     if (serverVersion == clientVersion) {
411       return proxy;
412     }
413     throw new VersionMismatch(protocol.getName(), clientVersion,
414                               serverVersion);
415   }
416 
417   /**
418    * Construct a client-side proxy object with the default SocketFactory
419    *
420    * @param protocol interface
421    * @param clientVersion version we are expecting
422    * @param addr remote address
423    * @param conf configuration
424    * @return a proxy instance
425    * @throws IOException e
426    */
427   public static VersionedProtocol getProxy(Class<?> protocol,
428       long clientVersion, InetSocketAddress addr, Configuration conf)
429       throws IOException {
430 
431     return getProxy(protocol, clientVersion, addr, conf, NetUtils
432         .getDefaultSocketFactory(conf));
433   }
434 
435   /**
436    * Stop this proxy and release its invoker's resource
437    * @param proxy the proxy to be stopped
438    */
439   public static void stopProxy(VersionedProtocol proxy) {
440     if (proxy!=null) {
441       ((Invoker)Proxy.getInvocationHandler(proxy)).close();
442     }
443   }
444 
445   /**
446    * Expert: Make multiple, parallel calls to a set of servers.
447    *
448    * @param method method to invoke
449    * @param params array of parameters
450    * @param addrs array of addresses
451    * @param conf configuration
452    * @return values
453    * @throws IOException e
454    */
455   public static Object[] call(Method method, Object[][] params,
456                               InetSocketAddress[] addrs, Configuration conf)
457     throws IOException {
458 
459     Invocation[] invocations = new Invocation[params.length];
460     for (int i = 0; i < params.length; i++)
461       invocations[i] = new Invocation(method, params[i]);
462     HBaseClient client = CLIENTS.getClient(conf);
463     try {
464     Writable[] wrappedValues = client.call(invocations, addrs);
465 
466     if (method.getReturnType() == Void.TYPE) {
467       return null;
468     }
469 
470     Object[] values =
471       (Object[])Array.newInstance(method.getReturnType(), wrappedValues.length);
472     for (int i = 0; i < values.length; i++)
473       if (wrappedValues[i] != null)
474         values[i] = ((HbaseObjectWritable)wrappedValues[i]).get();
475 
476     return values;
477     } finally {
478       CLIENTS.stopClient(client);
479     }
480   }
481 
482   /**
483    * Construct a server for a protocol implementation instance listening on a
484    * port and address.
485    *
486    * @param instance instance
487    * @param bindAddress bind address
488    * @param port port to bind to
489    * @param conf configuration
490    * @return Server
491    * @throws IOException e
492    */
493   public static Server getServer(final Object instance, final String bindAddress, final int port, Configuration conf)
494     throws IOException {
495     return getServer(instance, bindAddress, port, 1, false, conf);
496   }
497 
498   /**
499    * Construct a server for a protocol implementation instance listening on a
500    * port and address.
501    *
502    * @param instance instance
503    * @param bindAddress bind address
504    * @param port port to bind to
505    * @param numHandlers number of handlers to start
506    * @param verbose verbose flag
507    * @param conf configuration
508    * @return Server
509    * @throws IOException e
510    */
511   public static Server getServer(final Object instance, final String bindAddress, final int port,
512                                  final int numHandlers,
513                                  final boolean verbose, Configuration conf)
514     throws IOException {
515     return new Server(instance, conf, bindAddress, port, numHandlers, verbose);
516   }
517 
518   /** An RPC Server. */
519   public static class Server extends HBaseServer {
520     private Object instance;
521     private Class<?> implementation;
522     private boolean verbose;
523 
524     /**
525      * Construct an RPC server.
526      * @param instance the instance whose methods will be called
527      * @param conf the configuration to use
528      * @param bindAddress the address to bind on to listen for connection
529      * @param port the port to listen for connections on
530      * @throws IOException e
531      */
532     public Server(Object instance, Configuration conf, String bindAddress, int port)
533       throws IOException {
534       this(instance, conf,  bindAddress, port, 1, false);
535     }
536 
537     private static String classNameBase(String className) {
538       String[] names = className.split("\\.", -1);
539       if (names == null || names.length == 0) {
540         return className;
541       }
542       return names[names.length-1];
543     }
544 
545     /** Construct an RPC server.
546      * @param instance the instance whose methods will be called
547      * @param conf the configuration to use
548      * @param bindAddress the address to bind on to listen for connection
549      * @param port the port to listen for connections on
550      * @param numHandlers the number of method handler threads to run
551      * @param verbose whether each call should be logged
552      * @throws IOException e
553      */
554     public Server(Object instance, Configuration conf, String bindAddress,  int port,
555                   int numHandlers, boolean verbose) throws IOException {
556       super(bindAddress, port, Invocation.class, numHandlers, conf, classNameBase(instance.getClass().getName()));
557       this.instance = instance;
558       this.implementation = instance.getClass();
559       this.verbose = verbose;
560     }
561 
562     @Override
563     public Writable call(Writable param, long receivedTime) throws IOException {
564       try {
565         Invocation call = (Invocation)param;
566         if(call.getMethodName() == null) {
567           throw new IOException("Could not find requested method, the usual " +
568               "cause is a version mismatch between client and server.");
569         }
570         if (verbose) log("Call: " + call);
571         Method method =
572           implementation.getMethod(call.getMethodName(),
573                                    call.getParameterClasses());
574 
575         long startTime = System.currentTimeMillis();
576         Object value = method.invoke(instance, call.getParameters());
577         int processingTime = (int) (System.currentTimeMillis() - startTime);
578         int qTime = (int) (startTime-receivedTime);
579         if (LOG.isDebugEnabled()) {
580           LOG.debug("Served: " + call.getMethodName() +
581             " queueTime= " + qTime +
582             " procesingTime= " + processingTime);
583         }
584         rpcMetrics.rpcQueueTime.inc(qTime);
585         rpcMetrics.rpcProcessingTime.inc(processingTime);
586         rpcMetrics.inc(call.getMethodName(), processingTime);
587         if (verbose) log("Return: "+value);
588 
589         return new HbaseObjectWritable(method.getReturnType(), value);
590 
591       } catch (InvocationTargetException e) {
592         Throwable target = e.getTargetException();
593         if (target instanceof IOException) {
594           throw (IOException)target;
595         }
596         IOException ioe = new IOException(target.toString());
597         ioe.setStackTrace(target.getStackTrace());
598         throw ioe;
599       } catch (Throwable e) {
600         IOException ioe = new IOException(e.toString());
601         ioe.setStackTrace(e.getStackTrace());
602         throw ioe;
603       }
604     }
605   }
606 
607   protected static void log(String value) {
608     String v = value;
609     if (v != null && v.length() > 55)
610       v = v.substring(0, 55)+"...";
611     LOG.info(v);
612   }
613 }