View Javadoc

1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  
19  package org.apache.hadoop.hbase.io;
20  
21  import java.io.DataInput;
22  import java.io.DataOutput;
23  import java.io.IOException;
24  import java.lang.reflect.Array;
25  import java.util.ArrayList;
26  import java.util.Collection;
27  import java.util.HashMap;
28  import java.util.Iterator;
29  import java.util.List;
30  import java.util.Map;
31  
32  import org.apache.commons.logging.Log;
33  import org.apache.commons.logging.LogFactory;
34  import org.apache.hadoop.conf.Configurable;
35  import org.apache.hadoop.conf.Configuration;
36  import org.apache.hadoop.conf.Configured;
37  import org.apache.hadoop.hbase.ClusterStatus;
38  import org.apache.hadoop.hbase.HColumnDescriptor;
39  import org.apache.hadoop.hbase.HConstants;
40  import org.apache.hadoop.hbase.HMsg;
41  import org.apache.hadoop.hbase.HRegionInfo;
42  import org.apache.hadoop.hbase.HServerAddress;
43  import org.apache.hadoop.hbase.HServerInfo;
44  import org.apache.hadoop.hbase.HTableDescriptor;
45  import org.apache.hadoop.hbase.KeyValue;
46  import org.apache.hadoop.hbase.client.Delete;
47  import org.apache.hadoop.hbase.client.Get;
48  import org.apache.hadoop.hbase.client.Put;
49  import org.apache.hadoop.hbase.client.Result;
50  import org.apache.hadoop.hbase.client.Scan;
51  import org.apache.hadoop.hbase.client.MultiPutResponse;
52  import org.apache.hadoop.hbase.client.MultiPut;
53  import org.apache.hadoop.hbase.filter.*;
54  import org.apache.hadoop.hbase.io.HbaseMapWritable;
55  import org.apache.hadoop.io.MapWritable;
56  import org.apache.hadoop.io.ObjectWritable;
57  import org.apache.hadoop.io.Text;
58  import org.apache.hadoop.io.Writable;
59  import org.apache.hadoop.io.WritableFactories;
60  import org.apache.hadoop.hbase.regionserver.HRegion;
61  import org.apache.hadoop.hbase.regionserver.wal.HLog;
62  import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
63  import org.apache.hadoop.hbase.util.Bytes;
64  
65  /**
66   * This is a customized version of the polymorphic hadoop
67   * {@link ObjectWritable}.  It removes UTF8 (HADOOP-414).
68   * Using {@link Text} intead of UTF-8 saves ~2% CPU between reading and writing
69   * objects running a short sequentialWrite Performance Evaluation test just in
70   * ObjectWritable alone; more when we're doing randomRead-ing.  Other
71   * optimizations include our passing codes for classes instead of the
72   * actual class names themselves.  This makes it so this class needs amendment
73   * if non-Writable classes are introduced -- if passed a Writable for which we
74   * have no code, we just do the old-school passing of the class name, etc. --
75   * but passing codes the  savings are large particularly when cell
76   * data is small (If < a couple of kilobytes, the encoding/decoding of class
77   * name and reflection to instantiate class was costing in excess of the cell
78   * handling).
79   */
80  public class HbaseObjectWritable implements Writable, Configurable {
81    protected final static Log LOG = LogFactory.getLog(HbaseObjectWritable.class);
82  
83    // Here we maintain two static maps of classes to code and vice versa.
84    // Add new classes+codes as wanted or figure way to auto-generate these
85    // maps from the HMasterInterface.
86    static final Map<Byte, Class<?>> CODE_TO_CLASS =
87      new HashMap<Byte, Class<?>>();
88    static final Map<Class<?>, Byte> CLASS_TO_CODE =
89      new HashMap<Class<?>, Byte>();
90    // Special code that means 'not-encoded'; in this case we do old school
91    // sending of the class name using reflection, etc.
92    private static final byte NOT_ENCODED = 0;
93    static {
94      byte code = NOT_ENCODED + 1;
95      // Primitive types.
96      addToMap(Boolean.TYPE, code++);
97      addToMap(Byte.TYPE, code++);
98      addToMap(Character.TYPE, code++);
99      addToMap(Short.TYPE, code++);
100     addToMap(Integer.TYPE, code++);
101     addToMap(Long.TYPE, code++);
102     addToMap(Float.TYPE, code++);
103     addToMap(Double.TYPE, code++);
104     addToMap(Void.TYPE, code++);
105 
106     // Other java types
107     addToMap(String.class, code++);
108     addToMap(byte [].class, code++);
109     addToMap(byte [][].class, code++);
110 
111     // Hadoop types
112     addToMap(Text.class, code++);
113     addToMap(Writable.class, code++);
114     addToMap(Writable [].class, code++);
115     addToMap(HbaseMapWritable.class, code++);
116     addToMap(NullInstance.class, code++);
117 
118     // Hbase types
119     addToMap(HColumnDescriptor.class, code++);
120     addToMap(HConstants.Modify.class, code++);
121     addToMap(HMsg.class, code++);
122     addToMap(HMsg[].class, code++);
123     addToMap(HRegion.class, code++);
124     addToMap(HRegion[].class, code++);
125     addToMap(HRegionInfo.class, code++);
126     addToMap(HRegionInfo[].class, code++);
127     addToMap(HServerAddress.class, code++);
128     addToMap(HServerInfo.class, code++);
129     addToMap(HTableDescriptor.class, code++);
130     addToMap(MapWritable.class, code++);
131 
132     //
133     // HBASE-880
134     //
135     addToMap(ClusterStatus.class, code++);
136     addToMap(Delete.class, code++);
137     addToMap(Get.class, code++);
138     addToMap(KeyValue.class, code++);
139     addToMap(KeyValue[].class, code++);
140     addToMap(Put.class, code++);
141     addToMap(Put[].class, code++);
142     addToMap(Result.class, code++);
143     addToMap(Result[].class, code++);
144     addToMap(Scan.class, code++);
145 
146     addToMap(WhileMatchFilter.class, code++);
147     addToMap(PrefixFilter.class, code++);
148     addToMap(PageFilter.class, code++);
149     addToMap(InclusiveStopFilter.class, code++);
150     addToMap(ColumnCountGetFilter.class, code++);
151     addToMap(SingleColumnValueFilter.class, code++);
152     addToMap(SingleColumnValueExcludeFilter.class, code++);
153     addToMap(BinaryComparator.class, code++);
154     addToMap(CompareFilter.class, code++);
155     addToMap(RowFilter.class, code++);
156     addToMap(ValueFilter.class, code++);
157     addToMap(QualifierFilter.class, code++);
158     addToMap(SkipFilter.class, code++);
159     addToMap(WritableByteArrayComparable.class, code++);
160     addToMap(FirstKeyOnlyFilter.class, code++);
161     addToMap(DependentColumnFilter.class, code++);
162 
163     addToMap(Delete [].class, code++);
164 
165     addToMap(MultiPut.class, code++);
166     addToMap(MultiPutResponse.class, code++);
167 
168     addToMap(HLog.Entry.class, code++);
169     addToMap(HLog.Entry[].class, code++);
170     addToMap(HLogKey.class, code++);
171 
172     // List
173     addToMap(List.class, code++);
174     addToMap(ColumnPrefixFilter.class, code++);
175   }
176 
177   private Class<?> declaredClass;
178   private Object instance;
179   private Configuration conf;
180 
181   /** default constructor for writable */
182   public HbaseObjectWritable() {
183     super();
184   }
185 
186   /**
187    * @param instance
188    */
189   public HbaseObjectWritable(Object instance) {
190     set(instance);
191   }
192 
193   /**
194    * @param declaredClass
195    * @param instance
196    */
197   public HbaseObjectWritable(Class<?> declaredClass, Object instance) {
198     this.declaredClass = declaredClass;
199     this.instance = instance;
200   }
201 
202   /** @return the instance, or null if none. */
203   public Object get() { return instance; }
204 
205   /** @return the class this is meant to be. */
206   public Class<?> getDeclaredClass() { return declaredClass; }
207 
208   /**
209    * Reset the instance.
210    * @param instance
211    */
212   public void set(Object instance) {
213     this.declaredClass = instance.getClass();
214     this.instance = instance;
215   }
216 
217   /**
218    * @see java.lang.Object#toString()
219    */
220   @Override
221   public String toString() {
222     return "OW[class=" + declaredClass + ",value=" + instance + "]";
223   }
224 
225 
226   public void readFields(DataInput in) throws IOException {
227     readObject(in, this, this.conf);
228   }
229 
230   public void write(DataOutput out) throws IOException {
231     writeObject(out, instance, declaredClass, conf);
232   }
233 
234   private static class NullInstance extends Configured implements Writable {
235     Class<?> declaredClass;
236     /** default constructor for writable */
237     @SuppressWarnings("unused")
238     public NullInstance() { super(null); }
239 
240     /**
241      * @param declaredClass
242      * @param conf
243      */
244     public NullInstance(Class<?> declaredClass, Configuration conf) {
245       super(conf);
246       this.declaredClass = declaredClass;
247     }
248 
249     public void readFields(DataInput in) throws IOException {
250       this.declaredClass = CODE_TO_CLASS.get(in.readByte());
251     }
252 
253     public void write(DataOutput out) throws IOException {
254       writeClassCode(out, this.declaredClass);
255     }
256   }
257 
258   /**
259    * Write out the code byte for passed Class.
260    * @param out
261    * @param c
262    * @throws IOException
263    */
264   static void writeClassCode(final DataOutput out, final Class<?> c)
265   throws IOException {
266     Byte code = CLASS_TO_CODE.get(c);
267     if (code == null ) {
268       if ( List.class.isAssignableFrom(c)) {
269         code = CLASS_TO_CODE.get(List.class);
270       }
271     }
272     if (code == null) {
273       LOG.error("Unsupported type " + c);
274       StackTraceElement[] els = new Exception().getStackTrace();
275       for(StackTraceElement elem : els) {
276         LOG.error(elem.getMethodName());
277       }
278 //          new Exception().getStackTrace()[0].getMethodName());
279 //      throw new IOException(new Exception().getStackTrace()[0].getMethodName());
280       throw new UnsupportedOperationException("No code for unexpected " + c);
281     }
282     out.writeByte(code);
283   }
284 
285   /**
286    * Write a {@link Writable}, {@link String}, primitive type, or an array of
287    * the preceding.
288    * @param out
289    * @param instance
290    * @param declaredClass
291    * @param conf
292    * @throws IOException
293    */
294   @SuppressWarnings("unchecked")
295   public static void writeObject(DataOutput out, Object instance,
296                                  Class declaredClass,
297                                  Configuration conf)
298   throws IOException {
299 
300     Object instanceObj = instance;
301     Class declClass = declaredClass;
302 
303     if (instanceObj == null) {                       // null
304       instanceObj = new NullInstance(declClass, conf);
305       declClass = Writable.class;
306     }
307     writeClassCode(out, declClass);
308     if (declClass.isArray()) {                // array
309       // If bytearray, just dump it out -- avoid the recursion and
310       // byte-at-a-time we were previously doing.
311       if (declClass.equals(byte [].class)) {
312         Bytes.writeByteArray(out, (byte [])instanceObj);
313       } else if(declClass.equals(Result [].class)) {
314         Result.writeArray(out, (Result [])instanceObj);
315       } else {
316         int length = Array.getLength(instanceObj);
317         out.writeInt(length);
318         for (int i = 0; i < length; i++) {
319           writeObject(out, Array.get(instanceObj, i),
320                     declClass.getComponentType(), conf);
321         }
322       }
323     } else if (List.class.isAssignableFrom(declClass)) {
324       List list = (List)instanceObj;
325       int length = list.size();
326       out.writeInt(length);
327       for (int i = 0; i < length; i++) {
328         writeObject(out, list.get(i),
329                   list.get(i).getClass(), conf);
330       }
331     } else if (declClass == String.class) {   // String
332       Text.writeString(out, (String)instanceObj);
333     } else if (declClass.isPrimitive()) {     // primitive type
334       if (declClass == Boolean.TYPE) {        // boolean
335         out.writeBoolean(((Boolean)instanceObj).booleanValue());
336       } else if (declClass == Character.TYPE) { // char
337         out.writeChar(((Character)instanceObj).charValue());
338       } else if (declClass == Byte.TYPE) {    // byte
339         out.writeByte(((Byte)instanceObj).byteValue());
340       } else if (declClass == Short.TYPE) {   // short
341         out.writeShort(((Short)instanceObj).shortValue());
342       } else if (declClass == Integer.TYPE) { // int
343         out.writeInt(((Integer)instanceObj).intValue());
344       } else if (declClass == Long.TYPE) {    // long
345         out.writeLong(((Long)instanceObj).longValue());
346       } else if (declClass == Float.TYPE) {   // float
347         out.writeFloat(((Float)instanceObj).floatValue());
348       } else if (declClass == Double.TYPE) {  // double
349         out.writeDouble(((Double)instanceObj).doubleValue());
350       } else if (declClass == Void.TYPE) {    // void
351       } else {
352         throw new IllegalArgumentException("Not a primitive: "+declClass);
353       }
354     } else if (declClass.isEnum()) {         // enum
355       Text.writeString(out, ((Enum)instanceObj).name());
356     } else if (Writable.class.isAssignableFrom(declClass)) { // Writable
357       Class <?> c = instanceObj.getClass();
358       Byte code = CLASS_TO_CODE.get(c);
359       if (code == null) {
360         out.writeByte(NOT_ENCODED);
361         Text.writeString(out, c.getName());
362       } else {
363         writeClassCode(out, c);
364       }
365       ((Writable)instanceObj).write(out);
366     } else {
367       throw new IOException("Can't write: "+instanceObj+" as "+declClass);
368     }
369   }
370 
371 
372   /**
373    * Read a {@link Writable}, {@link String}, primitive type, or an array of
374    * the preceding.
375    * @param in
376    * @param conf
377    * @return the object
378    * @throws IOException
379    */
380   public static Object readObject(DataInput in, Configuration conf)
381     throws IOException {
382     return readObject(in, null, conf);
383   }
384 
385   /**
386    * Read a {@link Writable}, {@link String}, primitive type, or an array of
387    * the preceding.
388    * @param in
389    * @param objectWritable
390    * @param conf
391    * @return the object
392    * @throws IOException
393    */
394   @SuppressWarnings("unchecked")
395   public static Object readObject(DataInput in,
396       HbaseObjectWritable objectWritable, Configuration conf)
397   throws IOException {
398     Class<?> declaredClass = CODE_TO_CLASS.get(in.readByte());
399     Object instance;
400     if (declaredClass.isPrimitive()) {            // primitive types
401       if (declaredClass == Boolean.TYPE) {             // boolean
402         instance = Boolean.valueOf(in.readBoolean());
403       } else if (declaredClass == Character.TYPE) {    // char
404         instance = Character.valueOf(in.readChar());
405       } else if (declaredClass == Byte.TYPE) {         // byte
406         instance = Byte.valueOf(in.readByte());
407       } else if (declaredClass == Short.TYPE) {        // short
408         instance = Short.valueOf(in.readShort());
409       } else if (declaredClass == Integer.TYPE) {      // int
410         instance = Integer.valueOf(in.readInt());
411       } else if (declaredClass == Long.TYPE) {         // long
412         instance = Long.valueOf(in.readLong());
413       } else if (declaredClass == Float.TYPE) {        // float
414         instance = Float.valueOf(in.readFloat());
415       } else if (declaredClass == Double.TYPE) {       // double
416         instance = Double.valueOf(in.readDouble());
417       } else if (declaredClass == Void.TYPE) {         // void
418         instance = null;
419       } else {
420         throw new IllegalArgumentException("Not a primitive: "+declaredClass);
421       }
422     } else if (declaredClass.isArray()) {              // array
423       if (declaredClass.equals(byte [].class)) {
424         instance = Bytes.readByteArray(in);
425       } else if(declaredClass.equals(Result [].class)) {
426         instance = Result.readArray(in);
427       } else {
428         int length = in.readInt();
429         instance = Array.newInstance(declaredClass.getComponentType(), length);
430         for (int i = 0; i < length; i++) {
431           Array.set(instance, i, readObject(in, conf));
432         }
433       }
434     } else if (List.class.isAssignableFrom(declaredClass)) {              // List
435       int length = in.readInt();
436       instance = new ArrayList(length);
437       for (int i = 0; i < length; i++) {
438         ((ArrayList)instance).add(readObject(in, conf));
439       }
440     } else if (declaredClass == String.class) {        // String
441       instance = Text.readString(in);
442     } else if (declaredClass.isEnum()) {         // enum
443       instance = Enum.valueOf((Class<? extends Enum>) declaredClass,
444         Text.readString(in));
445     } else {                                      // Writable
446       Class instanceClass = null;
447       Byte b = in.readByte();
448       if (b.byteValue() == NOT_ENCODED) {
449         String className = Text.readString(in);
450         try {
451           instanceClass = getClassByName(conf, className);
452         } catch (ClassNotFoundException e) {
453           LOG.error("Can't find class " + className, e);
454           throw new IOException("Can't find class " + className, e);
455         }
456       } else {
457         instanceClass = CODE_TO_CLASS.get(b);
458       }
459       Writable writable = WritableFactories.newInstance(instanceClass, conf);
460       try {
461         writable.readFields(in);
462       } catch (Exception e) {
463         LOG.error("Error in readFields", e);
464         throw new IOException("Error in readFields" , e);
465       }
466       instance = writable;
467       if (instanceClass == NullInstance.class) {  // null
468         declaredClass = ((NullInstance)instance).declaredClass;
469         instance = null;
470       }
471     }
472     if (objectWritable != null) {                 // store values
473       objectWritable.declaredClass = declaredClass;
474       objectWritable.instance = instance;
475     }
476     return instance;
477   }
478 
479   @SuppressWarnings("unchecked")
480   private static Class getClassByName(Configuration conf, String className)
481   throws ClassNotFoundException {
482     if(conf != null) {
483       return conf.getClassByName(className);
484     }
485     ClassLoader cl = Thread.currentThread().getContextClassLoader();
486     if(cl == null) {
487       cl = HbaseObjectWritable.class.getClassLoader();
488     }
489     return Class.forName(className, true, cl);
490   }
491 
492   private static void addToMap(final Class<?> clazz, final byte code) {
493     CLASS_TO_CODE.put(clazz, code);
494     CODE_TO_CLASS.put(code, clazz);
495   }
496 
497   public void setConf(Configuration conf) {
498     this.conf = conf;
499   }
500 
501   public Configuration getConf() {
502     return this.conf;
503   }
504 }