View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.io;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.lang.reflect.Array;
25  import java.util.ArrayList;
26  import java.util.HashMap;
27  import java.util.List;
28  import java.util.Map;
29  import java.util.NavigableSet;
30  
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.apache.hadoop.conf.Configurable;
34  import org.apache.hadoop.conf.Configuration;
35  import org.apache.hadoop.conf.Configured;
36  import org.apache.hadoop.hbase.ClusterStatus;
37  import org.apache.hadoop.hbase.HColumnDescriptor;
38  import org.apache.hadoop.hbase.HConstants;
39  import org.apache.hadoop.hbase.HMsg;
40  import org.apache.hadoop.hbase.HRegionInfo;
41  import org.apache.hadoop.hbase.HServerAddress;
42  import org.apache.hadoop.hbase.HServerInfo;
43  import org.apache.hadoop.hbase.HTableDescriptor;
44  import org.apache.hadoop.hbase.KeyValue;
45  import org.apache.hadoop.hbase.client.Delete;
46  import org.apache.hadoop.hbase.client.Get;
47  import org.apache.hadoop.hbase.client.Increment;
48  import org.apache.hadoop.hbase.client.MultiPut;
49  import org.apache.hadoop.hbase.client.MultiPutResponse;
50  import org.apache.hadoop.hbase.client.MultiAction;
51  import org.apache.hadoop.hbase.client.Action;
52  import org.apache.hadoop.hbase.client.MultiResponse;
53  import org.apache.hadoop.hbase.client.Put;
54  import org.apache.hadoop.hbase.client.Result;
55  import org.apache.hadoop.hbase.client.Row;
56  import org.apache.hadoop.hbase.client.Scan;
57  import org.apache.hadoop.hbase.filter.BinaryComparator;
58  import org.apache.hadoop.hbase.filter.ColumnCountGetFilter;
59  import org.apache.hadoop.hbase.filter.ColumnPrefixFilter;
60  import org.apache.hadoop.hbase.filter.CompareFilter;
61  import org.apache.hadoop.hbase.filter.DependentColumnFilter;
62  import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
63  import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
64  import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
65  import org.apache.hadoop.hbase.filter.PageFilter;
66  import org.apache.hadoop.hbase.filter.PrefixFilter;
67  import org.apache.hadoop.hbase.filter.QualifierFilter;
68  import org.apache.hadoop.hbase.filter.RowFilter;
69  import org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter;
70  import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
71  import org.apache.hadoop.hbase.filter.SkipFilter;
72  import org.apache.hadoop.hbase.filter.ValueFilter;
73  import org.apache.hadoop.hbase.filter.WhileMatchFilter;
74  import org.apache.hadoop.hbase.filter.WritableByteArrayComparable;
75  import org.apache.hadoop.hbase.regionserver.HRegion;
76  import org.apache.hadoop.hbase.regionserver.wal.HLog;
77  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
78  import org.apache.hadoop.hbase.util.Bytes;
79  import org.apache.hadoop.io.MapWritable;
80  import org.apache.hadoop.io.ObjectWritable;
81  import org.apache.hadoop.io.Text;
82  import org.apache.hadoop.io.Writable;
83  import org.apache.hadoop.io.WritableFactories;
84  
85  /**
86   * This is a customized version of the polymorphic hadoop
87   * {@link ObjectWritable}.  It removes UTF8 (HADOOP-414).
88   * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing
89   * objects running a short sequentialWrite Performance Evaluation test just in
90   * ObjectWritable alone; more when we're doing randomRead-ing.  Other
91   * optimizations include our passing codes for classes instead of the
92   * actual class names themselves.  This makes it so this class needs amendment
93   * if non-Writable classes are introduced -- if passed a Writable for which we
94   * have no code, we just do the old-school passing of the class name, etc. --
95   * but passing codes the  savings are large particularly when cell
96   * data is small (If < a couple of kilobytes, the encoding/decoding of class
97   * name and reflection to instantiate class was costing in excess of the cell
98   * handling).
99   */
100 public class HbaseObjectWritable implements Writable, WritableWithSize, Configurable {
101   protected final static Log LOG = LogFactory.getLog(HbaseObjectWritable.class);
102 
103   // Here we maintain two static maps of classes to code and vice versa.
104   // Add new classes+codes as wanted or figure way to auto-generate these
105   // maps from the HMasterInterface.
106   static final Map<Byte, Class<?>> CODE_TO_CLASS =
107     new HashMap<Byte, Class<?>>();
108   static final Map<Class<?>, Byte> CLASS_TO_CODE =
109     new HashMap<Class<?>, Byte>();
110   // Special code that means 'not-encoded'; in this case we do old school
111   // sending of the class name using reflection, etc.
112   private static final byte NOT_ENCODED = 0;
113   static {
114     byte code = NOT_ENCODED + 1;
115     // Primitive types.
116     addToMap(Boolean.TYPE, code++);
117     addToMap(Byte.TYPE, code++);
118     addToMap(Character.TYPE, code++);
119     addToMap(Short.TYPE, code++);
120     addToMap(Integer.TYPE, code++);
121     addToMap(Long.TYPE, code++);
122     addToMap(Float.TYPE, code++);
123     addToMap(Double.TYPE, code++);
124     addToMap(Void.TYPE, code++);
125 
126     // Other java types
127     addToMap(String.class, code++);
128     addToMap(byte [].class, code++);
129     addToMap(byte [][].class, code++);
130 
131     // Hadoop types
132     addToMap(Text.class, code++);
133     addToMap(Writable.class, code++);
134     addToMap(Writable [].class, code++);
135     addToMap(HbaseMapWritable.class, code++);
136     addToMap(NullInstance.class, code++);
137 
138     // Hbase types
139     addToMap(HColumnDescriptor.class, code++);
140     addToMap(HConstants.Modify.class, code++);
141     addToMap(HMsg.class, code++);
142     addToMap(HMsg[].class, code++);
143     addToMap(HRegion.class, code++);
144     addToMap(HRegion[].class, code++);
145     addToMap(HRegionInfo.class, code++);
146     addToMap(HRegionInfo[].class, code++);
147     addToMap(HServerAddress.class, code++);
148     addToMap(HServerInfo.class, code++);
149     addToMap(HTableDescriptor.class, code++);
150     addToMap(MapWritable.class, code++);
151 
152     //
153     // HBASE-880
154     //
155     addToMap(ClusterStatus.class, code++);
156     addToMap(Delete.class, code++);
157     addToMap(Get.class, code++);
158     addToMap(KeyValue.class, code++);
159     addToMap(KeyValue[].class, code++);
160     addToMap(Put.class, code++);
161     addToMap(Put[].class, code++);
162     addToMap(Result.class, code++);
163     addToMap(Result[].class, code++);
164     addToMap(Scan.class, code++);
165 
166     addToMap(WhileMatchFilter.class, code++);
167     addToMap(PrefixFilter.class, code++);
168     addToMap(PageFilter.class, code++);
169     addToMap(InclusiveStopFilter.class, code++);
170     addToMap(ColumnCountGetFilter.class, code++);
171     addToMap(SingleColumnValueFilter.class, code++);
172     addToMap(SingleColumnValueExcludeFilter.class, code++);
173     addToMap(BinaryComparator.class, code++);
174     addToMap(CompareFilter.class, code++);
175     addToMap(RowFilter.class, code++);
176     addToMap(ValueFilter.class, code++);
177     addToMap(QualifierFilter.class, code++);
178     addToMap(SkipFilter.class, code++);
179     addToMap(WritableByteArrayComparable.class, code++);
180     addToMap(FirstKeyOnlyFilter.class, code++);
181     addToMap(DependentColumnFilter.class, code++);
182 
183     addToMap(Delete [].class, code++);
184 
185     addToMap(MultiPut.class, code++);
186     addToMap(MultiPutResponse.class, code++);
187 
188     addToMap(HLog.Entry.class, code++);
189     addToMap(HLog.Entry[].class, code++);
190     addToMap(HLogKey.class, code++);
191 
192     addToMap(List.class, code++);
193 
194     addToMap(NavigableSet.class, code++);
195     addToMap(ColumnPrefixFilter.class, code++);
196 
197     // Multi
198     addToMap(Row.class, code++);
199     addToMap(Action.class, code++);
200     addToMap(MultiAction.class, code++);
201     addToMap(MultiResponse.class, code++);
202 
203     addToMap(Increment.class, code++);
204 
205     addToMap(KeyOnlyFilter.class, code++);
206 
207   }
208 
209   private Class<?> declaredClass;
210   private Object instance;
211   private Configuration conf;
212 
213   /** default constructor for writable */
214   public HbaseObjectWritable() {
215     super();
216   }
217 
218   /**
219    * @param instance
220    */
221   public HbaseObjectWritable(Object instance) {
222     set(instance);
223   }
224 
225   /**
226    * @param declaredClass
227    * @param instance
228    */
229   public HbaseObjectWritable(Class<?> declaredClass, Object instance) {
230     this.declaredClass = declaredClass;
231     this.instance = instance;
232   }
233 
234   /** @return the instance, or null if none. */
235   public Object get() { return instance; }
236 
237   /** @return the class this is meant to be. */
238   public Class<?> getDeclaredClass() { return declaredClass; }
239 
240   /**
241    * Reset the instance.
242    * @param instance
243    */
244   public void set(Object instance) {
245     this.declaredClass = instance.getClass();
246     this.instance = instance;
247   }
248 
249   /**
250    * @see java.lang.Object#toString()
251    */
252   @Override
253   public String toString() {
254     return "OW[class=" + declaredClass + ",value=" + instance + "]";
255   }
256 
257 
258   public void readFields(DataInput in) throws IOException {
259     readObject(in, this, this.conf);
260   }
261 
262   public void write(DataOutput out) throws IOException {
263     writeObject(out, instance, declaredClass, conf);
264   }
265 
266   public long getWritableSize() {
267     return getWritableSize(instance, declaredClass, conf);
268   }
269 
270   private static class NullInstance extends Configured implements Writable {
271     Class<?> declaredClass;
272     /** default constructor for writable */
273     @SuppressWarnings("unused")
274     public NullInstance() { super(null); }
275 
276     /**
277      * @param declaredClass
278      * @param conf
279      */
280     public NullInstance(Class<?> declaredClass, Configuration conf) {
281       super(conf);
282       this.declaredClass = declaredClass;
283     }
284 
285     public void readFields(DataInput in) throws IOException {
286       this.declaredClass = CODE_TO_CLASS.get(in.readByte());
287     }
288 
289     public void write(DataOutput out) throws IOException {
290       writeClassCode(out, this.declaredClass);
291     }
292   }
293 
294   /**
295    * Write out the code byte for passed Class.
296    * @param out
297    * @param c
298    * @throws IOException
299    */
300   static void writeClassCode(final DataOutput out, final Class<?> c)
301   throws IOException {
302     Byte code = CLASS_TO_CODE.get(c);
303     if (code == null ) {
304       if ( List.class.isAssignableFrom(c)) {
305         code = CLASS_TO_CODE.get(List.class);
306       }
307       else if (Writable.class.isAssignableFrom(c)) {
308         code = CLASS_TO_CODE.get(Writable.class);
309       }
310     }
311     if (code == null) {
312       LOG.error("Unsupported type " + c);
313       StackTraceElement[] els = new Exception().getStackTrace();
314       for(StackTraceElement elem : els) {
315         LOG.error(elem.getMethodName());
316       }
317 //          new Exception().getStackTrace()[0].getMethodName());
318 //      throw new IOException(new Exception().getStackTrace()[0].getMethodName());
319       throw new UnsupportedOperationException("No code for unexpected " + c);
320     }
321     out.writeByte(code);
322   }
323 
324 
325   public static long getWritableSize(Object instance, Class declaredClass,
326                                      Configuration conf) {
327     long size = Bytes.SIZEOF_BYTE; // code
328     if (instance == null) {
329       return 0L;
330     }
331 
332     if (declaredClass.isArray()) {
333       if (declaredClass.equals(Result[].class)) {
334 
335         return size + Result.getWriteArraySize((Result[])instance);
336       }
337     }
338     if (declaredClass.equals(Result.class)) {
339       Result r = (Result) instance;
340       // one extra class code for writable instance.
341       return r.getWritableSize() + size + Bytes.SIZEOF_BYTE;
342     }
343     return 0L; // no hint is the default.
344   }
345   /**
346    * Write a {@link Writable}, {@link String}, primitive type, or an array of
347    * the preceding.
348    * @param out
349    * @param instance
350    * @param declaredClass
351    * @param conf
352    * @throws IOException
353    */
354   @SuppressWarnings("unchecked")
355   public static void writeObject(DataOutput out, Object instance,
356                                  Class declaredClass,
357                                  Configuration conf)
358   throws IOException {
359 
360     Object instanceObj = instance;
361     Class declClass = declaredClass;
362 
363     if (instanceObj == null) {                       // null
364       instanceObj = new NullInstance(declClass, conf);
365       declClass = Writable.class;
366     }
367     writeClassCode(out, declClass);
368     if (declClass.isArray()) {                // array
369       // If bytearray, just dump it out -- avoid the recursion and
370       // byte-at-a-time we were previously doing.
371       if (declClass.equals(byte [].class)) {
372         Bytes.writeByteArray(out, (byte [])instanceObj);
373       } else if(declClass.equals(Result [].class)) {
374         Result.writeArray(out, (Result [])instanceObj);
375       } else {
376         int length = Array.getLength(instanceObj);
377         out.writeInt(length);
378         for (int i = 0; i < length; i++) {
379           writeObject(out, Array.get(instanceObj, i),
380                     declClass.getComponentType(), conf);
381         }
382       }
383     } else if (List.class.isAssignableFrom(declClass)) {
384       List list = (List)instanceObj;
385       int length = list.size();
386       out.writeInt(length);
387       for (int i = 0; i < length; i++) {
388         writeObject(out, list.get(i),
389                   list.get(i).getClass(), conf);
390       }
391     } else if (declClass == String.class) {   // String
392       Text.writeString(out, (String)instanceObj);
393     } else if (declClass.isPrimitive()) {     // primitive type
394       if (declClass == Boolean.TYPE) {        // boolean
395         out.writeBoolean(((Boolean)instanceObj).booleanValue());
396       } else if (declClass == Character.TYPE) { // char
397         out.writeChar(((Character)instanceObj).charValue());
398       } else if (declClass == Byte.TYPE) {    // byte
399         out.writeByte(((Byte)instanceObj).byteValue());
400       } else if (declClass == Short.TYPE) {   // short
401         out.writeShort(((Short)instanceObj).shortValue());
402       } else if (declClass == Integer.TYPE) { // int
403         out.writeInt(((Integer)instanceObj).intValue());
404       } else if (declClass == Long.TYPE) {    // long
405         out.writeLong(((Long)instanceObj).longValue());
406       } else if (declClass == Float.TYPE) {   // float
407         out.writeFloat(((Float)instanceObj).floatValue());
408       } else if (declClass == Double.TYPE) {  // double
409         out.writeDouble(((Double)instanceObj).doubleValue());
410       } else if (declClass == Void.TYPE) {    // void
411       } else {
412         throw new IllegalArgumentException("Not a primitive: "+declClass);
413       }
414     } else if (declClass.isEnum()) {         // enum
415       Text.writeString(out, ((Enum)instanceObj).name());
416     } else if (Writable.class.isAssignableFrom(declClass)) { // Writable
417       Class <?> c = instanceObj.getClass();
418       Byte code = CLASS_TO_CODE.get(c);
419       if (code == null) {
420         out.writeByte(NOT_ENCODED);
421         Text.writeString(out, c.getName());
422       } else {
423         writeClassCode(out, c);
424       }
425       ((Writable)instanceObj).write(out);
426     } else {
427       throw new IOException("Can't write: "+instanceObj+" as "+declClass);
428     }
429   }
430 
431 
432   /**
433    * Read a {@link Writable}, {@link String}, primitive type, or an array of
434    * the preceding.
435    * @param in
436    * @param conf
437    * @return the object
438    * @throws IOException
439    */
440   public static Object readObject(DataInput in, Configuration conf)
441     throws IOException {
442     return readObject(in, null, conf);
443   }
444 
445   /**
446    * Read a {@link Writable}, {@link String}, primitive type, or an array of
447    * the preceding.
448    * @param in
449    * @param objectWritable
450    * @param conf
451    * @return the object
452    * @throws IOException
453    */
454   @SuppressWarnings("unchecked")
455   public static Object readObject(DataInput in,
456       HbaseObjectWritable objectWritable, Configuration conf)
457   throws IOException {
458     Class<?> declaredClass = CODE_TO_CLASS.get(in.readByte());
459     Object instance;
460     if (declaredClass.isPrimitive()) {            // primitive types
461       if (declaredClass == Boolean.TYPE) {             // boolean
462         instance = Boolean.valueOf(in.readBoolean());
463       } else if (declaredClass == Character.TYPE) {    // char
464         instance = Character.valueOf(in.readChar());
465       } else if (declaredClass == Byte.TYPE) {         // byte
466         instance = Byte.valueOf(in.readByte());
467       } else if (declaredClass == Short.TYPE) {        // short
468         instance = Short.valueOf(in.readShort());
469       } else if (declaredClass == Integer.TYPE) {      // int
470         instance = Integer.valueOf(in.readInt());
471       } else if (declaredClass == Long.TYPE) {         // long
472         instance = Long.valueOf(in.readLong());
473       } else if (declaredClass == Float.TYPE) {        // float
474         instance = Float.valueOf(in.readFloat());
475       } else if (declaredClass == Double.TYPE) {       // double
476         instance = Double.valueOf(in.readDouble());
477       } else if (declaredClass == Void.TYPE) {         // void
478         instance = null;
479       } else {
480         throw new IllegalArgumentException("Not a primitive: "+declaredClass);
481       }
482     } else if (declaredClass.isArray()) {              // array
483       if (declaredClass.equals(byte [].class)) {
484         instance = Bytes.readByteArray(in);
485       } else if(declaredClass.equals(Result [].class)) {
486         instance = Result.readArray(in);
487       } else {
488         int length = in.readInt();
489         instance = Array.newInstance(declaredClass.getComponentType(), length);
490         for (int i = 0; i < length; i++) {
491           Array.set(instance, i, readObject(in, conf));
492         }
493       }
494     } else if (List.class.isAssignableFrom(declaredClass)) {              // List
495       int length = in.readInt();
496       instance = new ArrayList(length);
497       for (int i = 0; i < length; i++) {
498         ((ArrayList)instance).add(readObject(in, conf));
499       }
500     } else if (declaredClass == String.class) {        // String
501       instance = Text.readString(in);
502     } else if (declaredClass.isEnum()) {         // enum
503       instance = Enum.valueOf((Class<? extends Enum>) declaredClass,
504         Text.readString(in));
505     } else {                                      // Writable
506       Class instanceClass = null;
507       Byte b = in.readByte();
508       if (b.byteValue() == NOT_ENCODED) {
509         String className = Text.readString(in);
510         try {
511           instanceClass = getClassByName(conf, className);
512         } catch (ClassNotFoundException e) {
513           LOG.error("Can't find class " + className, e);
514           throw new IOException("Can't find class " + className, e);
515         }
516       } else {
517         instanceClass = CODE_TO_CLASS.get(b);
518       }
519       Writable writable = WritableFactories.newInstance(instanceClass, conf);
520       try {
521         writable.readFields(in);
522       } catch (Exception e) {
523         LOG.error("Error in readFields", e);
524         throw new IOException("Error in readFields" , e);
525       }
526       instance = writable;
527       if (instanceClass == NullInstance.class) {  // null
528         declaredClass = ((NullInstance)instance).declaredClass;
529         instance = null;
530       }
531     }
532     if (objectWritable != null) {                 // store values
533       objectWritable.declaredClass = declaredClass;
534       objectWritable.instance = instance;
535     }
536     return instance;
537   }
538 
539   @SuppressWarnings("unchecked")
540   private static Class getClassByName(Configuration conf, String className)
541   throws ClassNotFoundException {
542     if(conf != null) {
543       return conf.getClassByName(className);
544     }
545     ClassLoader cl = Thread.currentThread().getContextClassLoader();
546     if(cl == null) {
547       cl = HbaseObjectWritable.class.getClassLoader();
548     }
549     return Class.forName(className, true, cl);
550   }
551 
552   private static void addToMap(final Class<?> clazz, final byte code) {
553     CLASS_TO_CODE.put(clazz, code);
554     CODE_TO_CLASS.put(code, clazz);
555   }
556 
557   public void setConf(Configuration conf) {
558     this.conf = conf;
559   }
560 
561   public Configuration getConf() {
562     return this.conf;
563   }
564 }