1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase;
21  
22  
23  import java.io.ByteArrayOutputStream;
24  import java.io.DataOutputStream;
25  import java.util.ArrayList;
26  import java.util.List;
27  import java.util.Map;
28  import java.util.NavigableSet;
29  import java.util.Set;
30  
31  import org.apache.hadoop.hbase.client.Delete;
32  import org.apache.hadoop.hbase.client.Get;
33  import org.apache.hadoop.hbase.client.Put;
34  import org.apache.hadoop.hbase.client.Result;
35  import org.apache.hadoop.hbase.client.RowLock;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.filter.BinaryComparator;
38  import org.apache.hadoop.hbase.filter.Filter;
39  import org.apache.hadoop.hbase.filter.PrefixFilter;
40  import org.apache.hadoop.hbase.filter.RowFilter;
41  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
42  import org.apache.hadoop.hbase.io.HbaseMapWritable;
43  import org.apache.hadoop.hbase.io.TimeRange;
44  import org.apache.hadoop.hbase.util.Bytes;
45  import org.apache.hadoop.hbase.util.Writables;
46  import org.apache.hadoop.io.DataInputBuffer;
47  
48  /**
49   * Test HBase Writables serializations
50   */
51  public class TestSerialization extends HBaseTestCase {
52  
53    @Override
54    protected void setUp() throws Exception {
55      super.setUp();
56    }
57  
58    @Override
59    protected void tearDown() throws Exception {
60      super.tearDown();
61    }
62  
63    public void testCompareFilter() throws Exception {
64      Filter f = new RowFilter(CompareOp.EQUAL,
65        new BinaryComparator(Bytes.toBytes("testRowOne-2")));
66      byte [] bytes = Writables.getBytes(f);
67      Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
68      assertNotNull(ff);
69    }
70  
71    public void testKeyValue() throws Exception {
72      byte [] row = Bytes.toBytes(getName());
73      byte [] family = Bytes.toBytes(getName());
74      byte [] qualifier = Bytes.toBytes(getName());
75      KeyValue original = new KeyValue(row, family, qualifier);
76      byte [] bytes = Writables.getBytes(original);
77      KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
78      assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
79    }
80  
81    @SuppressWarnings("unchecked")
82    public void testHbaseMapWritable() throws Exception {
83      HbaseMapWritable<byte [], byte []> hmw =
84        new HbaseMapWritable<byte[], byte[]>();
85      hmw.put("key".getBytes(), "value".getBytes());
86      byte [] bytes = Writables.getBytes(hmw);
87      hmw = (HbaseMapWritable<byte[], byte[]>)
88        Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
89      assertTrue(hmw.size() == 1);
90      assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
91    }
92  
93    public void testHMsg() throws Exception {
94      HMsg  m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE);
95      byte [] mb = Writables.getBytes(m);
96      HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
97      assertTrue(m.equals(deserializedHMsg));
98      m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE,
99        new HRegionInfo(new HTableDescriptor(getName()),
100         HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
101         "Some message".getBytes());
102     mb = Writables.getBytes(m);
103     deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
104     assertTrue(m.equals(deserializedHMsg));
105   }
106 
107   public void testTableDescriptor() throws Exception {
108     HTableDescriptor htd = createTableDescriptor(getName());
109     byte [] mb = Writables.getBytes(htd);
110     HTableDescriptor deserializedHtd =
111       (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
112     assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
113   }
114 
115   /**
116    * Test RegionInfo serialization
117    * @throws Exception
118    */
119   public void testRegionInfo() throws Exception {
120     HTableDescriptor htd = new HTableDescriptor(getName());
121     String [] families = new String [] {"info", "anchor"};
122     for (int i = 0; i < families.length; i++) {
123       htd.addFamily(new HColumnDescriptor(families[i]));
124     }
125     HRegionInfo hri = new HRegionInfo(htd,
126       HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
127     byte [] hrib = Writables.getBytes(hri);
128     HRegionInfo deserializedHri =
129       (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
130     assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
131     assertEquals(hri.getTableDesc().getFamilies().size(),
132       deserializedHri.getTableDesc().getFamilies().size());
133   }
134 
135   /**
136    * Test ServerInfo serialization
137    * @throws Exception
138    */
139   public void testServerInfo() throws Exception {
140     HServerInfo hsi = new HServerInfo(new HServerAddress("0.0.0.0:123"), -1,
141       1245, "default name");
142     byte [] b = Writables.getBytes(hsi);
143     HServerInfo deserializedHsi =
144       (HServerInfo)Writables.getWritable(b, new HServerInfo());
145     assertTrue(hsi.equals(deserializedHsi));
146   }
147 
148   public void testPut() throws Exception{
149     byte[] row = "row".getBytes();
150     byte[] fam = "fam".getBytes();
151     byte[] qf1 = "qf1".getBytes();
152     byte[] qf2 = "qf2".getBytes();
153     byte[] qf3 = "qf3".getBytes();
154     byte[] qf4 = "qf4".getBytes();
155     byte[] qf5 = "qf5".getBytes();
156     byte[] qf6 = "qf6".getBytes();
157     byte[] qf7 = "qf7".getBytes();
158     byte[] qf8 = "qf8".getBytes();
159 
160     long ts = System.currentTimeMillis();
161     byte[] val = "val".getBytes();
162 
163     Put put = new Put(row);
164     put.add(fam, qf1, ts, val);
165     put.add(fam, qf2, ts, val);
166     put.add(fam, qf3, ts, val);
167     put.add(fam, qf4, ts, val);
168     put.add(fam, qf5, ts, val);
169     put.add(fam, qf6, ts, val);
170     put.add(fam, qf7, ts, val);
171     put.add(fam, qf8, ts, val);
172 
173     byte[] sb = Writables.getBytes(put);
174     Put desPut = (Put)Writables.getWritable(sb, new Put());
175 
176     //Timing test
177 //    long start = System.nanoTime();
178 //    desPut = (Put)Writables.getWritable(sb, new Put());
179 //    long stop = System.nanoTime();
180 //    System.out.println("timer " +(stop-start));
181 
182     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
183     List<KeyValue> list = null;
184     List<KeyValue> desList = null;
185     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
186       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
187       list = entry.getValue();
188       desList = desPut.getFamilyMap().get(entry.getKey());
189       for(int i=0; i<list.size(); i++){
190         assertTrue(list.get(i).equals(desList.get(i)));
191       }
192     }
193   }
194 
195 
196   public void testPut2() throws Exception{
197     byte[] row = "testAbort,,1243116656250".getBytes();
198     byte[] fam = "historian".getBytes();
199     byte[] qf1 = "creation".getBytes();
200 
201     long ts = 9223372036854775807L;
202     byte[] val = "dont-care".getBytes();
203 
204     Put put = new Put(row);
205     put.add(fam, qf1, ts, val);
206 
207     byte[] sb = Writables.getBytes(put);
208     Put desPut = (Put)Writables.getWritable(sb, new Put());
209 
210     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
211     List<KeyValue> list = null;
212     List<KeyValue> desList = null;
213     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
214       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
215       list = entry.getValue();
216       desList = desPut.getFamilyMap().get(entry.getKey());
217       for(int i=0; i<list.size(); i++){
218         assertTrue(list.get(i).equals(desList.get(i)));
219       }
220     }
221   }
222 
223 
224   public void testDelete() throws Exception{
225     byte[] row = "row".getBytes();
226     byte[] fam = "fam".getBytes();
227     byte[] qf1 = "qf1".getBytes();
228 
229     long ts = System.currentTimeMillis();
230 
231     Delete delete = new Delete(row);
232     delete.deleteColumn(fam, qf1, ts);
233 
234     byte[] sb = Writables.getBytes(delete);
235     Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
236 
237     assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
238     List<KeyValue> list = null;
239     List<KeyValue> desList = null;
240     for(Map.Entry<byte[], List<KeyValue>> entry :
241         delete.getFamilyMap().entrySet()){
242       assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
243       list = entry.getValue();
244       desList = desDelete.getFamilyMap().get(entry.getKey());
245       for(int i=0; i<list.size(); i++){
246         assertTrue(list.get(i).equals(desList.get(i)));
247       }
248     }
249   }
250 
251   public void testGet() throws Exception{
252     byte[] row = "row".getBytes();
253     byte[] fam = "fam".getBytes();
254     byte[] qf1 = "qf1".getBytes();
255 
256     long ts = System.currentTimeMillis();
257     int maxVersions = 2;
258     long lockid = 5;
259     RowLock rowLock = new RowLock(lockid);
260 
261     Get get = new Get(row, rowLock);
262     get.addColumn(fam, qf1);
263     get.setTimeRange(ts, ts+1);
264     get.setMaxVersions(maxVersions);
265 
266     byte[] sb = Writables.getBytes(get);
267     Get desGet = (Get)Writables.getWritable(sb, new Get());
268 
269     assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
270     Set<byte[]> set = null;
271     Set<byte[]> desSet = null;
272 
273     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
274         get.getFamilyMap().entrySet()){
275       assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
276       set = entry.getValue();
277       desSet = desGet.getFamilyMap().get(entry.getKey());
278       for(byte [] qualifier : set){
279         assertTrue(desSet.contains(qualifier));
280       }
281     }
282 
283     assertEquals(get.getLockId(), desGet.getLockId());
284     assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
285     TimeRange tr = get.getTimeRange();
286     TimeRange desTr = desGet.getTimeRange();
287     assertEquals(tr.getMax(), desTr.getMax());
288     assertEquals(tr.getMin(), desTr.getMin());
289   }
290 
291 
292   public void testScan() throws Exception{
293     byte[] startRow = "startRow".getBytes();
294     byte[] stopRow  = "stopRow".getBytes();
295     byte[] fam = "fam".getBytes();
296     byte[] qf1 = "qf1".getBytes();
297 
298     long ts = System.currentTimeMillis();
299     int maxVersions = 2;
300 
301     Scan scan = new Scan(startRow, stopRow);
302     scan.addColumn(fam, qf1);
303     scan.setTimeRange(ts, ts+1);
304     scan.setMaxVersions(maxVersions);
305 
306     byte[] sb = Writables.getBytes(scan);
307     Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
308 
309     assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
310     assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
311     assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
312     Set<byte[]> set = null;
313     Set<byte[]> desSet = null;
314 
315     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
316         scan.getFamilyMap().entrySet()){
317       assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
318       set = entry.getValue();
319       desSet = desScan.getFamilyMap().get(entry.getKey());
320       for(byte[] column : set){
321         assertTrue(desSet.contains(column));
322       }
323 
324       // Test filters are serialized properly.
325       scan = new Scan(startRow);
326       byte [] prefix = Bytes.toBytes(getName());
327       scan.setFilter(new PrefixFilter(prefix));
328       sb = Writables.getBytes(scan);
329       desScan = (Scan)Writables.getWritable(sb, new Scan());
330       Filter f = desScan.getFilter();
331       assertTrue(f instanceof PrefixFilter);
332     }
333 
334     assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
335     TimeRange tr = scan.getTimeRange();
336     TimeRange desTr = desScan.getTimeRange();
337     assertEquals(tr.getMax(), desTr.getMax());
338     assertEquals(tr.getMin(), desTr.getMin());
339   }
340 
341   public void testResultEmpty() throws Exception {
342     List<KeyValue> keys = new ArrayList<KeyValue>();
343     Result r = new Result(keys);
344     assertTrue(r.isEmpty());
345     byte [] rb = Writables.getBytes(r);
346     Result deserializedR = (Result)Writables.getWritable(rb, new Result());
347     assertTrue(deserializedR.isEmpty());
348   }
349 
350 
351   public void testResult() throws Exception {
352     byte [] rowA = Bytes.toBytes("rowA");
353     byte [] famA = Bytes.toBytes("famA");
354     byte [] qfA = Bytes.toBytes("qfA");
355     byte [] valueA = Bytes.toBytes("valueA");
356 
357     byte [] rowB = Bytes.toBytes("rowB");
358     byte [] famB = Bytes.toBytes("famB");
359     byte [] qfB = Bytes.toBytes("qfB");
360     byte [] valueB = Bytes.toBytes("valueB");
361 
362     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
363     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
364 
365     Result result = new Result(new KeyValue[]{kvA, kvB});
366 
367     byte [] rb = Writables.getBytes(result);
368     Result deResult = (Result)Writables.getWritable(rb, new Result());
369 
370     assertTrue("results are not equivalent, first key mismatch",
371         result.sorted()[0].equals(deResult.sorted()[0]));
372 
373     assertTrue("results are not equivalent, second key mismatch",
374         result.sorted()[1].equals(deResult.sorted()[1]));
375 
376     // Test empty Result
377     Result r = new Result();
378     byte [] b = Writables.getBytes(r);
379     Result deserialized = (Result)Writables.getWritable(b, new Result());
380     assertEquals(r.size(), deserialized.size());
381   }
382 
383   public void testResultDynamicBuild() throws Exception {
384     byte [] rowA = Bytes.toBytes("rowA");
385     byte [] famA = Bytes.toBytes("famA");
386     byte [] qfA = Bytes.toBytes("qfA");
387     byte [] valueA = Bytes.toBytes("valueA");
388 
389     byte [] rowB = Bytes.toBytes("rowB");
390     byte [] famB = Bytes.toBytes("famB");
391     byte [] qfB = Bytes.toBytes("qfB");
392     byte [] valueB = Bytes.toBytes("valueB");
393 
394     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
395     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
396 
397     Result result = new Result(new KeyValue[]{kvA, kvB});
398 
399     byte [] rb = Writables.getBytes(result);
400 
401 
402     // Call getRow() first
403     Result deResult = (Result)Writables.getWritable(rb, new Result());
404     byte [] row = deResult.getRow();
405     assertTrue(Bytes.equals(row, rowA));
406 
407     // Call sorted() first
408     deResult = (Result)Writables.getWritable(rb, new Result());
409     assertTrue("results are not equivalent, first key mismatch",
410         result.sorted()[0].equals(deResult.sorted()[0]));
411     assertTrue("results are not equivalent, second key mismatch",
412         result.sorted()[1].equals(deResult.sorted()[1]));
413 
414     // Call raw() first
415     deResult = (Result)Writables.getWritable(rb, new Result());
416     assertTrue("results are not equivalent, first key mismatch",
417         result.raw()[0].equals(deResult.raw()[0]));
418     assertTrue("results are not equivalent, second key mismatch",
419         result.raw()[1].equals(deResult.raw()[1]));
420 
421 
422   }
423 
424   public void testResultArray() throws Exception {
425     byte [] rowA = Bytes.toBytes("rowA");
426     byte [] famA = Bytes.toBytes("famA");
427     byte [] qfA = Bytes.toBytes("qfA");
428     byte [] valueA = Bytes.toBytes("valueA");
429 
430     byte [] rowB = Bytes.toBytes("rowB");
431     byte [] famB = Bytes.toBytes("famB");
432     byte [] qfB = Bytes.toBytes("qfB");
433     byte [] valueB = Bytes.toBytes("valueB");
434 
435     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
436     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
437 
438 
439     Result result1 = new Result(new KeyValue[]{kvA, kvB});
440     Result result2 = new Result(new KeyValue[]{kvB});
441     Result result3 = new Result(new KeyValue[]{kvB});
442 
443     Result [] results = new Result [] {result1, result2, result3};
444 
445     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
446     DataOutputStream out = new DataOutputStream(byteStream);
447     Result.writeArray(out, results);
448 
449     byte [] rb = byteStream.toByteArray();
450 
451     DataInputBuffer in = new DataInputBuffer();
452     in.reset(rb, 0, rb.length);
453 
454     Result [] deResults = Result.readArray(in);
455 
456     assertTrue(results.length == deResults.length);
457 
458     for(int i=0;i<results.length;i++) {
459       KeyValue [] keysA = results[i].sorted();
460       KeyValue [] keysB = deResults[i].sorted();
461       assertTrue(keysA.length == keysB.length);
462       for(int j=0;j<keysA.length;j++) {
463         assertTrue("Expected equivalent keys but found:\n" +
464             "KeyA : " + keysA[j].toString() + "\n" +
465             "KeyB : " + keysB[j].toString() + "\n" +
466             keysA.length + " total keys, " + i + "th so far"
467             ,keysA[j].equals(keysB[j]));
468       }
469     }
470 
471   }
472 
473   public void testResultArrayEmpty() throws Exception {
474     List<KeyValue> keys = new ArrayList<KeyValue>();
475     Result r = new Result(keys);
476     Result [] results = new Result [] {r};
477 
478     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
479     DataOutputStream out = new DataOutputStream(byteStream);
480 
481     Result.writeArray(out, results);
482 
483     results = null;
484 
485     byteStream = new ByteArrayOutputStream();
486     out = new DataOutputStream(byteStream);
487     Result.writeArray(out, results);
488 
489     byte [] rb = byteStream.toByteArray();
490 
491     DataInputBuffer in = new DataInputBuffer();
492     in.reset(rb, 0, rb.length);
493 
494     Result [] deResults = Result.readArray(in);
495 
496     assertTrue(deResults.length == 0);
497 
498     results = new Result[0];
499 
500     byteStream = new ByteArrayOutputStream();
501     out = new DataOutputStream(byteStream);
502     Result.writeArray(out, results);
503 
504     rb = byteStream.toByteArray();
505 
506     in = new DataInputBuffer();
507     in.reset(rb, 0, rb.length);
508 
509     deResults = Result.readArray(in);
510 
511     assertTrue(deResults.length == 0);
512 
513   }
514 
515   public void testTimeRange(String[] args) throws Exception{
516     TimeRange tr = new TimeRange(0,5);
517     byte [] mb = Writables.getBytes(tr);
518     TimeRange deserializedTr =
519       (TimeRange)Writables.getWritable(mb, new TimeRange());
520 
521     assertEquals(tr.getMax(), deserializedTr.getMax());
522     assertEquals(tr.getMin(), deserializedTr.getMin());
523 
524   }
525 
526   public void testKeyValue2() throws Exception {
527     byte[] row = getName().getBytes();
528     byte[] fam = "fam".getBytes();
529     byte[] qf = "qf".getBytes();
530     long ts = System.currentTimeMillis();
531     byte[] val = "val".getBytes();
532 
533     KeyValue kv = new KeyValue(row, fam, qf, ts, val);
534 
535     byte [] mb = Writables.getBytes(kv);
536     KeyValue deserializedKv =
537       (KeyValue)Writables.getWritable(mb, new KeyValue());
538     assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
539     assertEquals(kv.getOffset(), deserializedKv.getOffset());
540     assertEquals(kv.getLength(), deserializedKv.getLength());
541   }
542 }