1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22
23 import static org.junit.Assert.*;
24
25 import java.io.ByteArrayOutputStream;
26 import java.io.DataOutputStream;
27 import java.io.IOException;
28 import java.util.ArrayList;
29 import java.util.HashSet;
30 import java.util.List;
31 import java.util.Map;
32 import java.util.NavigableSet;
33 import java.util.Set;
34 import java.util.TreeMap;
35
36 import org.apache.hadoop.hbase.HServerLoad092.RegionLoad;
37 import org.apache.hadoop.hbase.client.Delete;
38 import org.apache.hadoop.hbase.client.Get;
39 import org.apache.hadoop.hbase.client.Put;
40 import org.apache.hadoop.hbase.client.Result;
41 import org.apache.hadoop.hbase.client.RowLock;
42 import org.apache.hadoop.hbase.client.Scan;
43 import org.apache.hadoop.hbase.filter.BinaryComparator;
44 import org.apache.hadoop.hbase.filter.Filter;
45 import org.apache.hadoop.hbase.filter.PrefixFilter;
46 import org.apache.hadoop.hbase.filter.RowFilter;
47 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
48 import org.apache.hadoop.hbase.io.HbaseMapWritable;
49 import org.apache.hadoop.hbase.io.TimeRange;
50 import org.apache.hadoop.hbase.util.Bytes;
51 import org.apache.hadoop.hbase.util.Writables;
52 import org.apache.hadoop.io.DataInputBuffer;
53 import org.junit.Test;
54 import org.junit.experimental.categories.Category;
55
56
57
58
59 @Category(SmallTests.class)
60 public class TestSerialization {
61 @Test
62 public void testHServerLoadVersioning() throws IOException {
63 Set<String> cps = new HashSet<String>(0);
64 Map<byte [], RegionLoad> regions = new TreeMap<byte [], RegionLoad>(Bytes.BYTES_COMPARATOR);
65 regions.put(HConstants.META_TABLE_NAME,
66 new HServerLoad092.RegionLoad(HConstants.META_TABLE_NAME, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, cps));
67 HServerLoad092 hsl092 = new HServerLoad092(0, 0, 0, 0, regions, cps);
68 byte [] hsl092bytes = Writables.getBytes(hsl092);
69 HServerLoad hsl = (HServerLoad)Writables.getWritable(hsl092bytes, new HServerLoad());
70
71 }
72
73 @Test public void testCompareFilter() throws Exception {
74 Filter f = new RowFilter(CompareOp.EQUAL,
75 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
76 byte [] bytes = Writables.getBytes(f);
77 Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
78 assertNotNull(ff);
79 }
80
81 @Test public void testKeyValue() throws Exception {
82 final String name = "testKeyValue";
83 byte [] row = Bytes.toBytes(name);
84 byte [] family = Bytes.toBytes(name);
85 byte [] qualifier = Bytes.toBytes(name);
86 KeyValue original = new KeyValue(row, family, qualifier);
87 byte [] bytes = Writables.getBytes(original);
88 KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
89 assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
90 }
91
92 @SuppressWarnings("unchecked")
93 @Test public void testHbaseMapWritable() throws Exception {
94 HbaseMapWritable<byte [], byte []> hmw =
95 new HbaseMapWritable<byte[], byte[]>();
96 hmw.put("key".getBytes(), "value".getBytes());
97 byte [] bytes = Writables.getBytes(hmw);
98 hmw = (HbaseMapWritable<byte[], byte[]>)
99 Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
100 assertTrue(hmw.size() == 1);
101 assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
102 }
103
104
105 @Test public void testTableDescriptor() throws Exception {
106 final String name = "testTableDescriptor";
107 HTableDescriptor htd = createTableDescriptor(name);
108 byte [] mb = Writables.getBytes(htd);
109 HTableDescriptor deserializedHtd =
110 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
111 assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
112 }
113
114
115
116
117
118 @Test public void testRegionInfo() throws Exception {
119 HRegionInfo hri = createRandomRegion("testRegionInfo");
120 byte [] hrib = Writables.getBytes(hri);
121 HRegionInfo deserializedHri =
122 (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
123 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
124
125
126 }
127
128 @Test public void testRegionInfos() throws Exception {
129 HRegionInfo hri = createRandomRegion("testRegionInfos");
130 byte [] hrib = Writables.getBytes(hri);
131 byte [] triple = new byte [3 * hrib.length];
132 System.arraycopy(hrib, 0, triple, 0, hrib.length);
133 System.arraycopy(hrib, 0, triple, hrib.length, hrib.length);
134 System.arraycopy(hrib, 0, triple, hrib.length * 2, hrib.length);
135 List<HRegionInfo> regions = Writables.getHRegionInfos(triple, 0, triple.length);
136 assertTrue(regions.size() == 3);
137 assertTrue(regions.get(0).equals(regions.get(1)));
138 assertTrue(regions.get(0).equals(regions.get(2)));
139 }
140
141 private HRegionInfo createRandomRegion(final String name) {
142 HTableDescriptor htd = new HTableDescriptor(name);
143 String [] families = new String [] {"info", "anchor"};
144 for (int i = 0; i < families.length; i++) {
145 htd.addFamily(new HColumnDescriptor(families[i]));
146 }
147 return new HRegionInfo(htd.getName(), HConstants.EMPTY_START_ROW,
148 HConstants.EMPTY_END_ROW);
149 }
150
151 @Test public void testPut() throws Exception{
152 byte[] row = "row".getBytes();
153 byte[] fam = "fam".getBytes();
154 byte[] qf1 = "qf1".getBytes();
155 byte[] qf2 = "qf2".getBytes();
156 byte[] qf3 = "qf3".getBytes();
157 byte[] qf4 = "qf4".getBytes();
158 byte[] qf5 = "qf5".getBytes();
159 byte[] qf6 = "qf6".getBytes();
160 byte[] qf7 = "qf7".getBytes();
161 byte[] qf8 = "qf8".getBytes();
162
163 long ts = System.currentTimeMillis();
164 byte[] val = "val".getBytes();
165
166 Put put = new Put(row);
167 put.setWriteToWAL(false);
168 put.add(fam, qf1, ts, val);
169 put.add(fam, qf2, ts, val);
170 put.add(fam, qf3, ts, val);
171 put.add(fam, qf4, ts, val);
172 put.add(fam, qf5, ts, val);
173 put.add(fam, qf6, ts, val);
174 put.add(fam, qf7, ts, val);
175 put.add(fam, qf8, ts, val);
176
177 byte[] sb = Writables.getBytes(put);
178 Put desPut = (Put)Writables.getWritable(sb, new Put());
179
180
181
182
183
184
185
186 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
187 List<KeyValue> list = null;
188 List<KeyValue> desList = null;
189 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
190 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
191 list = entry.getValue();
192 desList = desPut.getFamilyMap().get(entry.getKey());
193 for(int i=0; i<list.size(); i++){
194 assertTrue(list.get(i).equals(desList.get(i)));
195 }
196 }
197 }
198
199
200 @Test public void testPut2() throws Exception{
201 byte[] row = "testAbort,,1243116656250".getBytes();
202 byte[] fam = "historian".getBytes();
203 byte[] qf1 = "creation".getBytes();
204
205 long ts = 9223372036854775807L;
206 byte[] val = "dont-care".getBytes();
207
208 Put put = new Put(row);
209 put.add(fam, qf1, ts, val);
210
211 byte[] sb = Writables.getBytes(put);
212 Put desPut = (Put)Writables.getWritable(sb, new Put());
213
214 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
215 List<KeyValue> list = null;
216 List<KeyValue> desList = null;
217 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
218 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
219 list = entry.getValue();
220 desList = desPut.getFamilyMap().get(entry.getKey());
221 for(int i=0; i<list.size(); i++){
222 assertTrue(list.get(i).equals(desList.get(i)));
223 }
224 }
225 }
226
227
228 @Test public void testDelete() throws Exception{
229 byte[] row = "row".getBytes();
230 byte[] fam = "fam".getBytes();
231 byte[] qf1 = "qf1".getBytes();
232
233 long ts = System.currentTimeMillis();
234
235 Delete delete = new Delete(row);
236 delete.deleteColumn(fam, qf1, ts);
237
238 byte[] sb = Writables.getBytes(delete);
239 Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
240
241 assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
242 List<KeyValue> list = null;
243 List<KeyValue> desList = null;
244 for(Map.Entry<byte[], List<KeyValue>> entry :
245 delete.getFamilyMap().entrySet()){
246 assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
247 list = entry.getValue();
248 desList = desDelete.getFamilyMap().get(entry.getKey());
249 for(int i=0; i<list.size(); i++){
250 assertTrue(list.get(i).equals(desList.get(i)));
251 }
252 }
253 }
254
255 @Test public void testGet() throws Exception{
256 byte[] row = "row".getBytes();
257 byte[] fam = "fam".getBytes();
258 byte[] qf1 = "qf1".getBytes();
259
260 long ts = System.currentTimeMillis();
261 int maxVersions = 2;
262 long lockid = 5;
263 RowLock rowLock = new RowLock(lockid);
264
265 Get get = new Get(row, rowLock);
266 get.addColumn(fam, qf1);
267 get.setTimeRange(ts, ts+1);
268 get.setMaxVersions(maxVersions);
269
270 byte[] sb = Writables.getBytes(get);
271 Get desGet = (Get)Writables.getWritable(sb, new Get());
272
273 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
274 Set<byte[]> set = null;
275 Set<byte[]> desSet = null;
276
277 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
278 get.getFamilyMap().entrySet()){
279 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
280 set = entry.getValue();
281 desSet = desGet.getFamilyMap().get(entry.getKey());
282 for(byte [] qualifier : set){
283 assertTrue(desSet.contains(qualifier));
284 }
285 }
286
287 assertEquals(get.getLockId(), desGet.getLockId());
288 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
289 TimeRange tr = get.getTimeRange();
290 TimeRange desTr = desGet.getTimeRange();
291 assertEquals(tr.getMax(), desTr.getMax());
292 assertEquals(tr.getMin(), desTr.getMin());
293 }
294
295
296 @Test public void testScan() throws Exception {
297
298 byte[] startRow = "startRow".getBytes();
299 byte[] stopRow = "stopRow".getBytes();
300 byte[] fam = "fam".getBytes();
301 byte[] qf1 = "qf1".getBytes();
302
303 long ts = System.currentTimeMillis();
304 int maxVersions = 2;
305
306 Scan scan = new Scan(startRow, stopRow);
307 scan.addColumn(fam, qf1);
308 scan.setTimeRange(ts, ts+1);
309 scan.setMaxVersions(maxVersions);
310
311 byte[] sb = Writables.getBytes(scan);
312 Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
313
314 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
315 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
316 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
317 Set<byte[]> set = null;
318 Set<byte[]> desSet = null;
319
320 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
321 scan.getFamilyMap().entrySet()){
322 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
323 set = entry.getValue();
324 desSet = desScan.getFamilyMap().get(entry.getKey());
325 for(byte[] column : set){
326 assertTrue(desSet.contains(column));
327 }
328
329
330 scan = new Scan(startRow);
331 final String name = "testScan";
332 byte [] prefix = Bytes.toBytes(name);
333 scan.setFilter(new PrefixFilter(prefix));
334 sb = Writables.getBytes(scan);
335 desScan = (Scan)Writables.getWritable(sb, new Scan());
336 Filter f = desScan.getFilter();
337 assertTrue(f instanceof PrefixFilter);
338 }
339
340 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
341 TimeRange tr = scan.getTimeRange();
342 TimeRange desTr = desScan.getTimeRange();
343 assertEquals(tr.getMax(), desTr.getMax());
344 assertEquals(tr.getMin(), desTr.getMin());
345 }
346
347 @Test public void testResultEmpty() throws Exception {
348 List<KeyValue> keys = new ArrayList<KeyValue>();
349 Result r = new Result(keys);
350 assertTrue(r.isEmpty());
351 byte [] rb = Writables.getBytes(r);
352 Result deserializedR = (Result)Writables.getWritable(rb, new Result());
353 assertTrue(deserializedR.isEmpty());
354 }
355
356
357 @Test public void testResult() throws Exception {
358 byte [] rowA = Bytes.toBytes("rowA");
359 byte [] famA = Bytes.toBytes("famA");
360 byte [] qfA = Bytes.toBytes("qfA");
361 byte [] valueA = Bytes.toBytes("valueA");
362
363 byte [] rowB = Bytes.toBytes("rowB");
364 byte [] famB = Bytes.toBytes("famB");
365 byte [] qfB = Bytes.toBytes("qfB");
366 byte [] valueB = Bytes.toBytes("valueB");
367
368 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
369 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
370
371 Result result = new Result(new KeyValue[]{kvA, kvB});
372
373 byte [] rb = Writables.getBytes(result);
374 Result deResult = (Result)Writables.getWritable(rb, new Result());
375
376 assertTrue("results are not equivalent, first key mismatch",
377 result.raw()[0].equals(deResult.raw()[0]));
378
379 assertTrue("results are not equivalent, second key mismatch",
380 result.raw()[1].equals(deResult.raw()[1]));
381
382
383 Result r = new Result();
384 byte [] b = Writables.getBytes(r);
385 Result deserialized = (Result)Writables.getWritable(b, new Result());
386 assertEquals(r.size(), deserialized.size());
387 }
388
389 @Test public void testResultDynamicBuild() throws Exception {
390 byte [] rowA = Bytes.toBytes("rowA");
391 byte [] famA = Bytes.toBytes("famA");
392 byte [] qfA = Bytes.toBytes("qfA");
393 byte [] valueA = Bytes.toBytes("valueA");
394
395 byte [] rowB = Bytes.toBytes("rowB");
396 byte [] famB = Bytes.toBytes("famB");
397 byte [] qfB = Bytes.toBytes("qfB");
398 byte [] valueB = Bytes.toBytes("valueB");
399
400 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
401 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
402
403 Result result = new Result(new KeyValue[]{kvA, kvB});
404
405 byte [] rb = Writables.getBytes(result);
406
407
408
409 Result deResult = (Result)Writables.getWritable(rb, new Result());
410 byte [] row = deResult.getRow();
411 assertTrue(Bytes.equals(row, rowA));
412
413
414 deResult = (Result)Writables.getWritable(rb, new Result());
415 assertTrue("results are not equivalent, first key mismatch",
416 result.raw()[0].equals(deResult.raw()[0]));
417 assertTrue("results are not equivalent, second key mismatch",
418 result.raw()[1].equals(deResult.raw()[1]));
419
420
421 deResult = (Result)Writables.getWritable(rb, new Result());
422 assertTrue("results are not equivalent, first key mismatch",
423 result.raw()[0].equals(deResult.raw()[0]));
424 assertTrue("results are not equivalent, second key mismatch",
425 result.raw()[1].equals(deResult.raw()[1]));
426
427
428 }
429
430 @Test public void testResultArray() throws Exception {
431 byte [] rowA = Bytes.toBytes("rowA");
432 byte [] famA = Bytes.toBytes("famA");
433 byte [] qfA = Bytes.toBytes("qfA");
434 byte [] valueA = Bytes.toBytes("valueA");
435
436 byte [] rowB = Bytes.toBytes("rowB");
437 byte [] famB = Bytes.toBytes("famB");
438 byte [] qfB = Bytes.toBytes("qfB");
439 byte [] valueB = Bytes.toBytes("valueB");
440
441 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
442 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
443
444
445 Result result1 = new Result(new KeyValue[]{kvA, kvB});
446 Result result2 = new Result(new KeyValue[]{kvB});
447 Result result3 = new Result(new KeyValue[]{kvB});
448
449 Result [] results = new Result [] {result1, result2, result3};
450
451 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
452 DataOutputStream out = new DataOutputStream(byteStream);
453 Result.writeArray(out, results);
454
455 byte [] rb = byteStream.toByteArray();
456
457 DataInputBuffer in = new DataInputBuffer();
458 in.reset(rb, 0, rb.length);
459
460 Result [] deResults = Result.readArray(in);
461
462 assertTrue(results.length == deResults.length);
463
464 for(int i=0;i<results.length;i++) {
465 KeyValue [] keysA = results[i].raw();
466 KeyValue [] keysB = deResults[i].raw();
467 assertTrue(keysA.length == keysB.length);
468 for(int j=0;j<keysA.length;j++) {
469 assertTrue("Expected equivalent keys but found:\n" +
470 "KeyA : " + keysA[j].toString() + "\n" +
471 "KeyB : " + keysB[j].toString() + "\n" +
472 keysA.length + " total keys, " + i + "th so far"
473 ,keysA[j].equals(keysB[j]));
474 }
475 }
476
477 }
478
479 @Test public void testResultArrayEmpty() throws Exception {
480 List<KeyValue> keys = new ArrayList<KeyValue>();
481 Result r = new Result(keys);
482 Result [] results = new Result [] {r};
483
484 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
485 DataOutputStream out = new DataOutputStream(byteStream);
486
487 Result.writeArray(out, results);
488
489 results = null;
490
491 byteStream = new ByteArrayOutputStream();
492 out = new DataOutputStream(byteStream);
493 Result.writeArray(out, results);
494
495 byte [] rb = byteStream.toByteArray();
496
497 DataInputBuffer in = new DataInputBuffer();
498 in.reset(rb, 0, rb.length);
499
500 Result [] deResults = Result.readArray(in);
501
502 assertTrue(deResults.length == 0);
503
504 results = new Result[0];
505
506 byteStream = new ByteArrayOutputStream();
507 out = new DataOutputStream(byteStream);
508 Result.writeArray(out, results);
509
510 rb = byteStream.toByteArray();
511
512 in = new DataInputBuffer();
513 in.reset(rb, 0, rb.length);
514
515 deResults = Result.readArray(in);
516
517 assertTrue(deResults.length == 0);
518
519 }
520
521 @Test public void testTimeRange() throws Exception{
522 TimeRange tr = new TimeRange(0,5);
523 byte [] mb = Writables.getBytes(tr);
524 TimeRange deserializedTr =
525 (TimeRange)Writables.getWritable(mb, new TimeRange());
526
527 assertEquals(tr.getMax(), deserializedTr.getMax());
528 assertEquals(tr.getMin(), deserializedTr.getMin());
529
530 }
531
532 @Test public void testKeyValue2() throws Exception {
533 final String name = "testKeyValue2";
534 byte[] row = name.getBytes();
535 byte[] fam = "fam".getBytes();
536 byte[] qf = "qf".getBytes();
537 long ts = System.currentTimeMillis();
538 byte[] val = "val".getBytes();
539
540 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
541
542 byte [] mb = Writables.getBytes(kv);
543 KeyValue deserializedKv =
544 (KeyValue)Writables.getWritable(mb, new KeyValue());
545 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
546 assertEquals(kv.getOffset(), deserializedKv.getOffset());
547 assertEquals(kv.getLength(), deserializedKv.getLength());
548 }
549
550 protected static final int MAXVERSIONS = 3;
551 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
552 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
553 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
554 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
555
556
557
558
559
560
561
562 protected HTableDescriptor createTableDescriptor(final String name) {
563 return createTableDescriptor(name, MAXVERSIONS);
564 }
565
566
567
568
569
570
571
572
573 protected HTableDescriptor createTableDescriptor(final String name,
574 final int versions) {
575 HTableDescriptor htd = new HTableDescriptor(name);
576 htd.addFamily(new HColumnDescriptor(fam1)
577 .setMaxVersions(versions)
578 .setBlockCacheEnabled(false)
579 );
580 htd.addFamily(new HColumnDescriptor(fam2)
581 .setMaxVersions(versions)
582 .setBlockCacheEnabled(false)
583 );
584 htd.addFamily(new HColumnDescriptor(fam3)
585 .setMaxVersions(versions)
586 .setBlockCacheEnabled(false)
587 );
588 return htd;
589 }
590
591 @org.junit.Rule
592 public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
593 new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
594 }
595