View Javadoc

1   /**
2    *
3    * Licensed to the Apache Software Foundation (ASF) under one
4    * or more contributor license agreements.  See the NOTICE file
5    * distributed with this work for additional information
6    * regarding copyright ownership.  The ASF licenses this file
7    * to you under the Apache License, Version 2.0 (the
8    * "License"); you may not use this file except in compliance
9    * with the License.  You may obtain a copy of the License at
10   *
11   *     http://www.apache.org/licenses/LICENSE-2.0
12   *
13   * Unless required by applicable law or agreed to in writing, software
14   * distributed under the License is distributed on an "AS IS" BASIS,
15   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16   * See the License for the specific language governing permissions and
17   * limitations under the License.
18   */
19  package org.apache.hadoop.hbase;
20  
21  
22  import static org.junit.Assert.assertEquals;
23  import static org.junit.Assert.assertNotNull;
24  import static org.junit.Assert.assertTrue;
25  
26  import java.io.ByteArrayInputStream;
27  import java.io.ByteArrayOutputStream;
28  import java.io.DataInputStream;
29  import java.io.DataOutputStream;
30  import java.util.List;
31  import java.util.Map;
32  import java.util.NavigableSet;
33  import java.util.Set;
34  
35  import org.apache.hadoop.hbase.client.Get;
36  import org.apache.hadoop.hbase.client.Scan;
37  import org.apache.hadoop.hbase.exceptions.DeserializationException;
38  import org.apache.hadoop.hbase.filter.BinaryComparator;
39  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
40  import org.apache.hadoop.hbase.filter.Filter;
41  import org.apache.hadoop.hbase.filter.PrefixFilter;
42  import org.apache.hadoop.hbase.filter.RowFilter;
43  import org.apache.hadoop.hbase.io.TimeRange;
44  import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
45  import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
46  import org.apache.hadoop.hbase.util.Bytes;
47  import org.apache.hadoop.hbase.util.Writables;
48  import org.apache.hadoop.io.DataInputBuffer;
49  import org.junit.Test;
50  import org.junit.experimental.categories.Category;
51  
52  /**
53   * Test HBase Writables serializations
54   */
55  @Category(SmallTests.class)
56  public class TestSerialization {
57    @Test public void testKeyValue() throws Exception {
58      final String name = "testKeyValue2";
59      byte[] row = name.getBytes();
60      byte[] fam = "fam".getBytes();
61      byte[] qf = "qf".getBytes();
62      long ts = System.currentTimeMillis();
63      byte[] val = "val".getBytes();
64      KeyValue kv = new KeyValue(row, fam, qf, ts, val);
65      ByteArrayOutputStream baos = new ByteArrayOutputStream();
66      DataOutputStream dos = new DataOutputStream(baos);
67      long l = KeyValue.write(kv, dos);
68      dos.close();
69      byte [] mb = baos.toByteArray();
70      ByteArrayInputStream bais = new ByteArrayInputStream(mb);
71      DataInputStream dis = new DataInputStream(bais);
72      KeyValue deserializedKv = KeyValue.create(dis);
73      assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
74      assertEquals(kv.getOffset(), deserializedKv.getOffset());
75      assertEquals(kv.getLength(), deserializedKv.getLength());
76    }
77    
78    @Test
79    public void testSplitLogTask() throws DeserializationException {
80      SplitLogTask slt = new SplitLogTask.Unassigned(ServerName.valueOf("mgr,1,1"));
81      byte [] bytes = slt.toByteArray();
82      SplitLogTask sltDeserialized = SplitLogTask.parseFrom(bytes);
83      assertTrue(slt.equals(sltDeserialized));
84    }
85  
86    @Test public void testCompareFilter() throws Exception {
87      Filter f = new RowFilter(CompareOp.EQUAL,
88        new BinaryComparator(Bytes.toBytes("testRowOne-2")));
89      byte [] bytes = f.toByteArray();
90      Filter ff = RowFilter.parseFrom(bytes);
91      assertNotNull(ff);
92    }
93  
94    @Test public void testTableDescriptor() throws Exception {
95      final String name = "testTableDescriptor";
96      HTableDescriptor htd = createTableDescriptor(name);
97      byte [] mb = Writables.getBytes(htd);
98      HTableDescriptor deserializedHtd =
99        (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
100     assertEquals(htd.getTableName(), deserializedHtd.getTableName());
101   }
102 
103   /**
104    * Test RegionInfo serialization
105    * @throws Exception
106    */
107   @Test public void testRegionInfo() throws Exception {
108     HRegionInfo hri = createRandomRegion("testRegionInfo");
109 
110     //test toByteArray()
111     byte [] hrib = hri.toByteArray();
112     HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib);
113     assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
114     assertEquals(hri, deserializedHri);
115 
116     //test toDelimitedByteArray()
117     hrib = hri.toDelimitedByteArray();
118     DataInputBuffer buf = new DataInputBuffer();
119     try {
120       buf.reset(hrib, hrib.length);
121       deserializedHri = HRegionInfo.parseFrom(buf);
122       assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
123       assertEquals(hri, deserializedHri);
124     } finally {
125       buf.close();
126     }
127   }
128 
129   @Test public void testRegionInfos() throws Exception {
130     HRegionInfo hri = createRandomRegion("testRegionInfos");
131     byte[] triple = HRegionInfo.toDelimitedByteArray(hri, hri, hri);
132     List<HRegionInfo> regions = HRegionInfo.parseDelimitedFrom(triple, 0, triple.length);
133     assertTrue(regions.size() == 3);
134     assertTrue(regions.get(0).equals(regions.get(1)));
135     assertTrue(regions.get(0).equals(regions.get(2)));
136   }
137 
138   private HRegionInfo createRandomRegion(final String name) {
139     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
140     String [] families = new String [] {"info", "anchor"};
141     for (int i = 0; i < families.length; i++) {
142       htd.addFamily(new HColumnDescriptor(families[i]));
143     }
144     return new HRegionInfo(htd.getTableName(), HConstants.EMPTY_START_ROW,
145       HConstants.EMPTY_END_ROW);
146   }
147 
148   /*
149    * TODO
150   @Test public void testPut() throws Exception{
151     byte[] row = "row".getBytes();
152     byte[] fam = "fam".getBytes();
153     byte[] qf1 = "qf1".getBytes();
154     byte[] qf2 = "qf2".getBytes();
155     byte[] qf3 = "qf3".getBytes();
156     byte[] qf4 = "qf4".getBytes();
157     byte[] qf5 = "qf5".getBytes();
158     byte[] qf6 = "qf6".getBytes();
159     byte[] qf7 = "qf7".getBytes();
160     byte[] qf8 = "qf8".getBytes();
161 
162     long ts = System.currentTimeMillis();
163     byte[] val = "val".getBytes();
164 
165     Put put = new Put(row);
166     put.setWriteToWAL(false);
167     put.add(fam, qf1, ts, val);
168     put.add(fam, qf2, ts, val);
169     put.add(fam, qf3, ts, val);
170     put.add(fam, qf4, ts, val);
171     put.add(fam, qf5, ts, val);
172     put.add(fam, qf6, ts, val);
173     put.add(fam, qf7, ts, val);
174     put.add(fam, qf8, ts, val);
175 
176     byte[] sb = Writables.getBytes(put);
177     Put desPut = (Put)Writables.getWritable(sb, new Put());
178 
179     //Timing test
180 //    long start = System.nanoTime();
181 //    desPut = (Put)Writables.getWritable(sb, new Put());
182 //    long stop = System.nanoTime();
183 //    System.out.println("timer " +(stop-start));
184 
185     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
186     List<KeyValue> list = null;
187     List<KeyValue> desList = null;
188     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
189       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
190       list = entry.getValue();
191       desList = desPut.getFamilyMap().get(entry.getKey());
192       for(int i=0; i<list.size(); i++){
193         assertTrue(list.get(i).equals(desList.get(i)));
194       }
195     }
196   }
197 
198 
199   @Test public void testPut2() throws Exception{
200     byte[] row = "testAbort,,1243116656250".getBytes();
201     byte[] fam = "historian".getBytes();
202     byte[] qf1 = "creation".getBytes();
203 
204     long ts = 9223372036854775807L;
205     byte[] val = "dont-care".getBytes();
206 
207     Put put = new Put(row);
208     put.add(fam, qf1, ts, val);
209 
210     byte[] sb = Writables.getBytes(put);
211     Put desPut = (Put)Writables.getWritable(sb, new Put());
212 
213     assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
214     List<KeyValue> list = null;
215     List<KeyValue> desList = null;
216     for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
217       assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
218       list = entry.getValue();
219       desList = desPut.getFamilyMap().get(entry.getKey());
220       for(int i=0; i<list.size(); i++){
221         assertTrue(list.get(i).equals(desList.get(i)));
222       }
223     }
224   }
225 
226 
227   @Test public void testDelete() throws Exception{
228     byte[] row = "row".getBytes();
229     byte[] fam = "fam".getBytes();
230     byte[] qf1 = "qf1".getBytes();
231 
232     long ts = System.currentTimeMillis();
233 
234     Delete delete = new Delete(row);
235     delete.deleteColumn(fam, qf1, ts);
236 
237     byte[] sb = Writables.getBytes(delete);
238     Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
239 
240     assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
241     List<KeyValue> list = null;
242     List<KeyValue> desList = null;
243     for(Map.Entry<byte[], List<KeyValue>> entry :
244         delete.getFamilyMap().entrySet()){
245       assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
246       list = entry.getValue();
247       desList = desDelete.getFamilyMap().get(entry.getKey());
248       for(int i=0; i<list.size(); i++){
249         assertTrue(list.get(i).equals(desList.get(i)));
250       }
251     }
252   }
253   */
254 
255   @Test public void testGet() throws Exception{
256     byte[] row = "row".getBytes();
257     byte[] fam = "fam".getBytes();
258     byte[] qf1 = "qf1".getBytes();
259 
260     long ts = System.currentTimeMillis();
261     int maxVersions = 2;
262 
263     Get get = new Get(row);
264     get.addColumn(fam, qf1);
265     get.setTimeRange(ts, ts+1);
266     get.setMaxVersions(maxVersions);
267 
268     ClientProtos.Get getProto = ProtobufUtil.toGet(get);
269     Get desGet = ProtobufUtil.toGet(getProto);
270 
271     assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
272     Set<byte[]> set = null;
273     Set<byte[]> desSet = null;
274 
275     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
276         get.getFamilyMap().entrySet()){
277       assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
278       set = entry.getValue();
279       desSet = desGet.getFamilyMap().get(entry.getKey());
280       for(byte [] qualifier : set){
281         assertTrue(desSet.contains(qualifier));
282       }
283     }
284 
285     assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
286     TimeRange tr = get.getTimeRange();
287     TimeRange desTr = desGet.getTimeRange();
288     assertEquals(tr.getMax(), desTr.getMax());
289     assertEquals(tr.getMin(), desTr.getMin());
290   }
291 
292 
293   @Test public void testScan() throws Exception {
294 
295     byte[] startRow = "startRow".getBytes();
296     byte[] stopRow  = "stopRow".getBytes();
297     byte[] fam = "fam".getBytes();
298     byte[] qf1 = "qf1".getBytes();
299 
300     long ts = System.currentTimeMillis();
301     int maxVersions = 2;
302 
303     Scan scan = new Scan(startRow, stopRow);
304     scan.addColumn(fam, qf1);
305     scan.setTimeRange(ts, ts+1);
306     scan.setMaxVersions(maxVersions);
307 
308     ClientProtos.Scan scanProto = ProtobufUtil.toScan(scan);
309     Scan desScan = ProtobufUtil.toScan(scanProto);
310 
311     assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
312     assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
313     assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
314     Set<byte[]> set = null;
315     Set<byte[]> desSet = null;
316 
317     for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
318         scan.getFamilyMap().entrySet()){
319       assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
320       set = entry.getValue();
321       desSet = desScan.getFamilyMap().get(entry.getKey());
322       for(byte[] column : set){
323         assertTrue(desSet.contains(column));
324       }
325 
326       // Test filters are serialized properly.
327       scan = new Scan(startRow);
328       final String name = "testScan";
329       byte [] prefix = Bytes.toBytes(name);
330       scan.setFilter(new PrefixFilter(prefix));
331       scanProto = ProtobufUtil.toScan(scan);
332       desScan = ProtobufUtil.toScan(scanProto);
333       Filter f = desScan.getFilter();
334       assertTrue(f instanceof PrefixFilter);
335     }
336 
337     assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
338     TimeRange tr = scan.getTimeRange();
339     TimeRange desTr = desScan.getTimeRange();
340     assertEquals(tr.getMax(), desTr.getMax());
341     assertEquals(tr.getMin(), desTr.getMin());
342   }
343 
344   /*
345    * TODO
346   @Test public void testResultEmpty() throws Exception {
347     List<KeyValue> keys = new ArrayList<KeyValue>();
348     Result r = Result.newResult(keys);
349     assertTrue(r.isEmpty());
350     byte [] rb = Writables.getBytes(r);
351     Result deserializedR = (Result)Writables.getWritable(rb, new Result());
352     assertTrue(deserializedR.isEmpty());
353   }
354 
355 
356   @Test public void testResult() throws Exception {
357     byte [] rowA = Bytes.toBytes("rowA");
358     byte [] famA = Bytes.toBytes("famA");
359     byte [] qfA = Bytes.toBytes("qfA");
360     byte [] valueA = Bytes.toBytes("valueA");
361 
362     byte [] rowB = Bytes.toBytes("rowB");
363     byte [] famB = Bytes.toBytes("famB");
364     byte [] qfB = Bytes.toBytes("qfB");
365     byte [] valueB = Bytes.toBytes("valueB");
366 
367     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
368     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
369 
370     Result result = Result.newResult(new KeyValue[]{kvA, kvB});
371 
372     byte [] rb = Writables.getBytes(result);
373     Result deResult = (Result)Writables.getWritable(rb, new Result());
374 
375     assertTrue("results are not equivalent, first key mismatch",
376         result.raw()[0].equals(deResult.raw()[0]));
377 
378     assertTrue("results are not equivalent, second key mismatch",
379         result.raw()[1].equals(deResult.raw()[1]));
380 
381     // Test empty Result
382     Result r = new Result();
383     byte [] b = Writables.getBytes(r);
384     Result deserialized = (Result)Writables.getWritable(b, new Result());
385     assertEquals(r.size(), deserialized.size());
386   }
387 
388   @Test public void testResultDynamicBuild() throws Exception {
389     byte [] rowA = Bytes.toBytes("rowA");
390     byte [] famA = Bytes.toBytes("famA");
391     byte [] qfA = Bytes.toBytes("qfA");
392     byte [] valueA = Bytes.toBytes("valueA");
393 
394     byte [] rowB = Bytes.toBytes("rowB");
395     byte [] famB = Bytes.toBytes("famB");
396     byte [] qfB = Bytes.toBytes("qfB");
397     byte [] valueB = Bytes.toBytes("valueB");
398 
399     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
400     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
401 
402     Result result = Result.newResult(new KeyValue[]{kvA, kvB});
403 
404     byte [] rb = Writables.getBytes(result);
405 
406 
407     // Call getRow() first
408     Result deResult = (Result)Writables.getWritable(rb, new Result());
409     byte [] row = deResult.getRow();
410     assertTrue(Bytes.equals(row, rowA));
411 
412     // Call sorted() first
413     deResult = (Result)Writables.getWritable(rb, new Result());
414     assertTrue("results are not equivalent, first key mismatch",
415         result.raw()[0].equals(deResult.raw()[0]));
416     assertTrue("results are not equivalent, second key mismatch",
417         result.raw()[1].equals(deResult.raw()[1]));
418 
419     // Call raw() first
420     deResult = (Result)Writables.getWritable(rb, new Result());
421     assertTrue("results are not equivalent, first key mismatch",
422         result.raw()[0].equals(deResult.raw()[0]));
423     assertTrue("results are not equivalent, second key mismatch",
424         result.raw()[1].equals(deResult.raw()[1]));
425 
426 
427   }
428 
429   @Test public void testResultArray() throws Exception {
430     byte [] rowA = Bytes.toBytes("rowA");
431     byte [] famA = Bytes.toBytes("famA");
432     byte [] qfA = Bytes.toBytes("qfA");
433     byte [] valueA = Bytes.toBytes("valueA");
434 
435     byte [] rowB = Bytes.toBytes("rowB");
436     byte [] famB = Bytes.toBytes("famB");
437     byte [] qfB = Bytes.toBytes("qfB");
438     byte [] valueB = Bytes.toBytes("valueB");
439 
440     KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
441     KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
442 
443 
444     Result result1 = Result.newResult(new KeyValue[]{kvA, kvB});
445     Result result2 = Result.newResult(new KeyValue[]{kvB});
446     Result result3 = Result.newResult(new KeyValue[]{kvB});
447 
448     Result [] results = new Result [] {result1, result2, result3};
449 
450     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
451     DataOutputStream out = new DataOutputStream(byteStream);
452     Result.writeArray(out, results);
453 
454     byte [] rb = byteStream.toByteArray();
455 
456     DataInputBuffer in = new DataInputBuffer();
457     in.reset(rb, 0, rb.length);
458 
459     Result [] deResults = Result.readArray(in);
460 
461     assertTrue(results.length == deResults.length);
462 
463     for(int i=0;i<results.length;i++) {
464       KeyValue [] keysA = results[i].raw();
465       KeyValue [] keysB = deResults[i].raw();
466       assertTrue(keysA.length == keysB.length);
467       for(int j=0;j<keysA.length;j++) {
468         assertTrue("Expected equivalent keys but found:\n" +
469             "KeyA : " + keysA[j].toString() + "\n" +
470             "KeyB : " + keysB[j].toString() + "\n" +
471             keysA.length + " total keys, " + i + "th so far"
472             ,keysA[j].equals(keysB[j]));
473       }
474     }
475 
476   }
477 
478   @Test public void testResultArrayEmpty() throws Exception {
479     List<KeyValue> keys = new ArrayList<KeyValue>();
480     Result r = Result.newResult(keys);
481     Result [] results = new Result [] {r};
482 
483     ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
484     DataOutputStream out = new DataOutputStream(byteStream);
485 
486     Result.writeArray(out, results);
487 
488     results = null;
489 
490     byteStream = new ByteArrayOutputStream();
491     out = new DataOutputStream(byteStream);
492     Result.writeArray(out, results);
493 
494     byte [] rb = byteStream.toByteArray();
495 
496     DataInputBuffer in = new DataInputBuffer();
497     in.reset(rb, 0, rb.length);
498 
499     Result [] deResults = Result.readArray(in);
500 
501     assertTrue(deResults.length == 0);
502 
503     results = new Result[0];
504 
505     byteStream = new ByteArrayOutputStream();
506     out = new DataOutputStream(byteStream);
507     Result.writeArray(out, results);
508 
509     rb = byteStream.toByteArray();
510 
511     in = new DataInputBuffer();
512     in.reset(rb, 0, rb.length);
513 
514     deResults = Result.readArray(in);
515 
516     assertTrue(deResults.length == 0);
517 
518   }
519   */
520 
521   protected static final int MAXVERSIONS = 3;
522   protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
523   protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
524   protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
525   protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
526 
527   /**
528    * Create a table of name <code>name</code> with {@link COLUMNS} for
529    * families.
530    * @param name Name to give table.
531    * @return Column descriptor.
532    */
533   protected HTableDescriptor createTableDescriptor(final String name) {
534     return createTableDescriptor(name, MAXVERSIONS);
535   }
536 
537   /**
538    * Create a table of name <code>name</code> with {@link COLUMNS} for
539    * families.
540    * @param name Name to give table.
541    * @param versions How many versions to allow per column.
542    * @return Column descriptor.
543    */
544   protected HTableDescriptor createTableDescriptor(final String name,
545       final int versions) {
546     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name));
547     htd.addFamily(new HColumnDescriptor(fam1)
548         .setMaxVersions(versions)
549         .setBlockCacheEnabled(false)
550     );
551     htd.addFamily(new HColumnDescriptor(fam2)
552         .setMaxVersions(versions)
553         .setBlockCacheEnabled(false)
554     );
555     htd.addFamily(new HColumnDescriptor(fam3)
556         .setMaxVersions(versions)
557         .setBlockCacheEnabled(false)
558     );
559     return htd;
560   }
561 }