1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22
23 import static org.junit.Assert.*;
24
25 import java.io.ByteArrayOutputStream;
26 import java.io.DataOutputStream;
27 import java.util.ArrayList;
28 import java.util.List;
29 import java.util.Map;
30 import java.util.NavigableSet;
31 import java.util.Set;
32
33 import org.apache.hadoop.hbase.client.Delete;
34 import org.apache.hadoop.hbase.client.Get;
35 import org.apache.hadoop.hbase.client.Put;
36 import org.apache.hadoop.hbase.client.Result;
37 import org.apache.hadoop.hbase.client.RowLock;
38 import org.apache.hadoop.hbase.client.Scan;
39 import org.apache.hadoop.hbase.filter.BinaryComparator;
40 import org.apache.hadoop.hbase.filter.Filter;
41 import org.apache.hadoop.hbase.filter.PrefixFilter;
42 import org.apache.hadoop.hbase.filter.RowFilter;
43 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
44 import org.apache.hadoop.hbase.io.HbaseMapWritable;
45 import org.apache.hadoop.hbase.io.TimeRange;
46 import org.apache.hadoop.hbase.util.Bytes;
47 import org.apache.hadoop.hbase.util.Writables;
48 import org.apache.hadoop.io.DataInputBuffer;
49 import org.junit.Test;
50
51
52
53
54 public class TestSerialization {
55
56 @Test public void testCompareFilter() throws Exception {
57 Filter f = new RowFilter(CompareOp.EQUAL,
58 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
59 byte [] bytes = Writables.getBytes(f);
60 Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
61 assertNotNull(ff);
62 }
63
64 @Test public void testKeyValue() throws Exception {
65 final String name = "testKeyValue";
66 byte [] row = Bytes.toBytes(name);
67 byte [] family = Bytes.toBytes(name);
68 byte [] qualifier = Bytes.toBytes(name);
69 KeyValue original = new KeyValue(row, family, qualifier);
70 byte [] bytes = Writables.getBytes(original);
71 KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
72 assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
73 }
74
75 @SuppressWarnings("unchecked")
76 @Test public void testHbaseMapWritable() throws Exception {
77 HbaseMapWritable<byte [], byte []> hmw =
78 new HbaseMapWritable<byte[], byte[]>();
79 hmw.put("key".getBytes(), "value".getBytes());
80 byte [] bytes = Writables.getBytes(hmw);
81 hmw = (HbaseMapWritable<byte[], byte[]>)
82 Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
83 assertTrue(hmw.size() == 1);
84 assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
85 }
86
87 @Test public void testHMsg() throws Exception {
88 final String name = "testHMsg";
89 HMsg m = new HMsg(HMsg.Type.STOP_REGIONSERVER);
90 byte [] mb = Writables.getBytes(m);
91 HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
92 assertTrue(m.equals(deserializedHMsg));
93 m = new HMsg(HMsg.Type.STOP_REGIONSERVER,
94 new HRegionInfo(new HTableDescriptor(name),
95 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
96 "Some message".getBytes());
97 mb = Writables.getBytes(m);
98 deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
99 assertTrue(m.equals(deserializedHMsg));
100 }
101
102 @Test public void testTableDescriptor() throws Exception {
103 final String name = "testTableDescriptor";
104 HTableDescriptor htd = createTableDescriptor(name);
105 byte [] mb = Writables.getBytes(htd);
106 HTableDescriptor deserializedHtd =
107 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
108 assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
109 }
110
111
112
113
114
115 @Test public void testRegionInfo() throws Exception {
116 final String name = "testRegionInfo";
117 HTableDescriptor htd = new HTableDescriptor(name);
118 String [] families = new String [] {"info", "anchor"};
119 for (int i = 0; i < families.length; i++) {
120 htd.addFamily(new HColumnDescriptor(families[i]));
121 }
122 HRegionInfo hri = new HRegionInfo(htd,
123 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
124 byte [] hrib = Writables.getBytes(hri);
125 HRegionInfo deserializedHri =
126 (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
127 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
128 assertEquals(hri.getTableDesc().getFamilies().size(),
129 deserializedHri.getTableDesc().getFamilies().size());
130 }
131
132
133
134
135
136 @Test public void testServerInfo() throws Exception {
137 HServerInfo hsi = new HServerInfo(new HServerAddress("0.0.0.0:123"), -1,
138 1245, "default name");
139 byte [] b = Writables.getBytes(hsi);
140 HServerInfo deserializedHsi =
141 (HServerInfo)Writables.getWritable(b, new HServerInfo());
142 assertTrue(hsi.equals(deserializedHsi));
143 }
144
145 @Test public void testPut() throws Exception{
146 byte[] row = "row".getBytes();
147 byte[] fam = "fam".getBytes();
148 byte[] qf1 = "qf1".getBytes();
149 byte[] qf2 = "qf2".getBytes();
150 byte[] qf3 = "qf3".getBytes();
151 byte[] qf4 = "qf4".getBytes();
152 byte[] qf5 = "qf5".getBytes();
153 byte[] qf6 = "qf6".getBytes();
154 byte[] qf7 = "qf7".getBytes();
155 byte[] qf8 = "qf8".getBytes();
156
157 long ts = System.currentTimeMillis();
158 byte[] val = "val".getBytes();
159
160 Put put = new Put(row);
161 put.add(fam, qf1, ts, val);
162 put.add(fam, qf2, ts, val);
163 put.add(fam, qf3, ts, val);
164 put.add(fam, qf4, ts, val);
165 put.add(fam, qf5, ts, val);
166 put.add(fam, qf6, ts, val);
167 put.add(fam, qf7, ts, val);
168 put.add(fam, qf8, ts, val);
169
170 byte[] sb = Writables.getBytes(put);
171 Put desPut = (Put)Writables.getWritable(sb, new Put());
172
173
174
175
176
177
178
179 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
180 List<KeyValue> list = null;
181 List<KeyValue> desList = null;
182 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
183 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
184 list = entry.getValue();
185 desList = desPut.getFamilyMap().get(entry.getKey());
186 for(int i=0; i<list.size(); i++){
187 assertTrue(list.get(i).equals(desList.get(i)));
188 }
189 }
190 }
191
192
193 @Test public void testPut2() throws Exception{
194 byte[] row = "testAbort,,1243116656250".getBytes();
195 byte[] fam = "historian".getBytes();
196 byte[] qf1 = "creation".getBytes();
197
198 long ts = 9223372036854775807L;
199 byte[] val = "dont-care".getBytes();
200
201 Put put = new Put(row);
202 put.add(fam, qf1, ts, val);
203
204 byte[] sb = Writables.getBytes(put);
205 Put desPut = (Put)Writables.getWritable(sb, new Put());
206
207 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
208 List<KeyValue> list = null;
209 List<KeyValue> desList = null;
210 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
211 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
212 list = entry.getValue();
213 desList = desPut.getFamilyMap().get(entry.getKey());
214 for(int i=0; i<list.size(); i++){
215 assertTrue(list.get(i).equals(desList.get(i)));
216 }
217 }
218 }
219
220
221 @Test public void testDelete() throws Exception{
222 byte[] row = "row".getBytes();
223 byte[] fam = "fam".getBytes();
224 byte[] qf1 = "qf1".getBytes();
225
226 long ts = System.currentTimeMillis();
227
228 Delete delete = new Delete(row);
229 delete.deleteColumn(fam, qf1, ts);
230
231 byte[] sb = Writables.getBytes(delete);
232 Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
233
234 assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
235 List<KeyValue> list = null;
236 List<KeyValue> desList = null;
237 for(Map.Entry<byte[], List<KeyValue>> entry :
238 delete.getFamilyMap().entrySet()){
239 assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
240 list = entry.getValue();
241 desList = desDelete.getFamilyMap().get(entry.getKey());
242 for(int i=0; i<list.size(); i++){
243 assertTrue(list.get(i).equals(desList.get(i)));
244 }
245 }
246 }
247
248 @Test public void testGet() throws Exception{
249 byte[] row = "row".getBytes();
250 byte[] fam = "fam".getBytes();
251 byte[] qf1 = "qf1".getBytes();
252
253 long ts = System.currentTimeMillis();
254 int maxVersions = 2;
255 long lockid = 5;
256 RowLock rowLock = new RowLock(lockid);
257
258 Get get = new Get(row, rowLock);
259 get.addColumn(fam, qf1);
260 get.setTimeRange(ts, ts+1);
261 get.setMaxVersions(maxVersions);
262
263 byte[] sb = Writables.getBytes(get);
264 Get desGet = (Get)Writables.getWritable(sb, new Get());
265
266 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
267 Set<byte[]> set = null;
268 Set<byte[]> desSet = null;
269
270 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
271 get.getFamilyMap().entrySet()){
272 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
273 set = entry.getValue();
274 desSet = desGet.getFamilyMap().get(entry.getKey());
275 for(byte [] qualifier : set){
276 assertTrue(desSet.contains(qualifier));
277 }
278 }
279
280 assertEquals(get.getLockId(), desGet.getLockId());
281 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
282 TimeRange tr = get.getTimeRange();
283 TimeRange desTr = desGet.getTimeRange();
284 assertEquals(tr.getMax(), desTr.getMax());
285 assertEquals(tr.getMin(), desTr.getMin());
286 }
287
288
289 @Test public void testScan() throws Exception {
290
291 byte[] startRow = "startRow".getBytes();
292 byte[] stopRow = "stopRow".getBytes();
293 byte[] fam = "fam".getBytes();
294 byte[] qf1 = "qf1".getBytes();
295
296 long ts = System.currentTimeMillis();
297 int maxVersions = 2;
298
299 Scan scan = new Scan(startRow, stopRow);
300 scan.addColumn(fam, qf1);
301 scan.setTimeRange(ts, ts+1);
302 scan.setMaxVersions(maxVersions);
303
304 byte[] sb = Writables.getBytes(scan);
305 Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
306
307 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
308 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
309 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
310 Set<byte[]> set = null;
311 Set<byte[]> desSet = null;
312
313 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
314 scan.getFamilyMap().entrySet()){
315 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
316 set = entry.getValue();
317 desSet = desScan.getFamilyMap().get(entry.getKey());
318 for(byte[] column : set){
319 assertTrue(desSet.contains(column));
320 }
321
322
323 scan = new Scan(startRow);
324 final String name = "testScan";
325 byte [] prefix = Bytes.toBytes(name);
326 scan.setFilter(new PrefixFilter(prefix));
327 sb = Writables.getBytes(scan);
328 desScan = (Scan)Writables.getWritable(sb, new Scan());
329 Filter f = desScan.getFilter();
330 assertTrue(f instanceof PrefixFilter);
331 }
332
333 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
334 TimeRange tr = scan.getTimeRange();
335 TimeRange desTr = desScan.getTimeRange();
336 assertEquals(tr.getMax(), desTr.getMax());
337 assertEquals(tr.getMin(), desTr.getMin());
338 }
339
340 @Test public void testResultEmpty() throws Exception {
341 List<KeyValue> keys = new ArrayList<KeyValue>();
342 Result r = new Result(keys);
343 assertTrue(r.isEmpty());
344 byte [] rb = Writables.getBytes(r);
345 Result deserializedR = (Result)Writables.getWritable(rb, new Result());
346 assertTrue(deserializedR.isEmpty());
347 }
348
349
350 @Test public void testResult() throws Exception {
351 byte [] rowA = Bytes.toBytes("rowA");
352 byte [] famA = Bytes.toBytes("famA");
353 byte [] qfA = Bytes.toBytes("qfA");
354 byte [] valueA = Bytes.toBytes("valueA");
355
356 byte [] rowB = Bytes.toBytes("rowB");
357 byte [] famB = Bytes.toBytes("famB");
358 byte [] qfB = Bytes.toBytes("qfB");
359 byte [] valueB = Bytes.toBytes("valueB");
360
361 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
362 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
363
364 Result result = new Result(new KeyValue[]{kvA, kvB});
365
366 byte [] rb = Writables.getBytes(result);
367 Result deResult = (Result)Writables.getWritable(rb, new Result());
368
369 assertTrue("results are not equivalent, first key mismatch",
370 result.sorted()[0].equals(deResult.sorted()[0]));
371
372 assertTrue("results are not equivalent, second key mismatch",
373 result.sorted()[1].equals(deResult.sorted()[1]));
374
375
376 Result r = new Result();
377 byte [] b = Writables.getBytes(r);
378 Result deserialized = (Result)Writables.getWritable(b, new Result());
379 assertEquals(r.size(), deserialized.size());
380 }
381
382 @Test public void testResultDynamicBuild() throws Exception {
383 byte [] rowA = Bytes.toBytes("rowA");
384 byte [] famA = Bytes.toBytes("famA");
385 byte [] qfA = Bytes.toBytes("qfA");
386 byte [] valueA = Bytes.toBytes("valueA");
387
388 byte [] rowB = Bytes.toBytes("rowB");
389 byte [] famB = Bytes.toBytes("famB");
390 byte [] qfB = Bytes.toBytes("qfB");
391 byte [] valueB = Bytes.toBytes("valueB");
392
393 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
394 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
395
396 Result result = new Result(new KeyValue[]{kvA, kvB});
397
398 byte [] rb = Writables.getBytes(result);
399
400
401
402 Result deResult = (Result)Writables.getWritable(rb, new Result());
403 byte [] row = deResult.getRow();
404 assertTrue(Bytes.equals(row, rowA));
405
406
407 deResult = (Result)Writables.getWritable(rb, new Result());
408 assertTrue("results are not equivalent, first key mismatch",
409 result.sorted()[0].equals(deResult.sorted()[0]));
410 assertTrue("results are not equivalent, second key mismatch",
411 result.sorted()[1].equals(deResult.sorted()[1]));
412
413
414 deResult = (Result)Writables.getWritable(rb, new Result());
415 assertTrue("results are not equivalent, first key mismatch",
416 result.raw()[0].equals(deResult.raw()[0]));
417 assertTrue("results are not equivalent, second key mismatch",
418 result.raw()[1].equals(deResult.raw()[1]));
419
420
421 }
422
423 @Test public void testResultArray() throws Exception {
424 byte [] rowA = Bytes.toBytes("rowA");
425 byte [] famA = Bytes.toBytes("famA");
426 byte [] qfA = Bytes.toBytes("qfA");
427 byte [] valueA = Bytes.toBytes("valueA");
428
429 byte [] rowB = Bytes.toBytes("rowB");
430 byte [] famB = Bytes.toBytes("famB");
431 byte [] qfB = Bytes.toBytes("qfB");
432 byte [] valueB = Bytes.toBytes("valueB");
433
434 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
435 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
436
437
438 Result result1 = new Result(new KeyValue[]{kvA, kvB});
439 Result result2 = new Result(new KeyValue[]{kvB});
440 Result result3 = new Result(new KeyValue[]{kvB});
441
442 Result [] results = new Result [] {result1, result2, result3};
443
444 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
445 DataOutputStream out = new DataOutputStream(byteStream);
446 Result.writeArray(out, results);
447
448 byte [] rb = byteStream.toByteArray();
449
450 DataInputBuffer in = new DataInputBuffer();
451 in.reset(rb, 0, rb.length);
452
453 Result [] deResults = Result.readArray(in);
454
455 assertTrue(results.length == deResults.length);
456
457 for(int i=0;i<results.length;i++) {
458 KeyValue [] keysA = results[i].sorted();
459 KeyValue [] keysB = deResults[i].sorted();
460 assertTrue(keysA.length == keysB.length);
461 for(int j=0;j<keysA.length;j++) {
462 assertTrue("Expected equivalent keys but found:\n" +
463 "KeyA : " + keysA[j].toString() + "\n" +
464 "KeyB : " + keysB[j].toString() + "\n" +
465 keysA.length + " total keys, " + i + "th so far"
466 ,keysA[j].equals(keysB[j]));
467 }
468 }
469
470 }
471
472 @Test public void testResultArrayEmpty() throws Exception {
473 List<KeyValue> keys = new ArrayList<KeyValue>();
474 Result r = new Result(keys);
475 Result [] results = new Result [] {r};
476
477 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
478 DataOutputStream out = new DataOutputStream(byteStream);
479
480 Result.writeArray(out, results);
481
482 results = null;
483
484 byteStream = new ByteArrayOutputStream();
485 out = new DataOutputStream(byteStream);
486 Result.writeArray(out, results);
487
488 byte [] rb = byteStream.toByteArray();
489
490 DataInputBuffer in = new DataInputBuffer();
491 in.reset(rb, 0, rb.length);
492
493 Result [] deResults = Result.readArray(in);
494
495 assertTrue(deResults.length == 0);
496
497 results = new Result[0];
498
499 byteStream = new ByteArrayOutputStream();
500 out = new DataOutputStream(byteStream);
501 Result.writeArray(out, results);
502
503 rb = byteStream.toByteArray();
504
505 in = new DataInputBuffer();
506 in.reset(rb, 0, rb.length);
507
508 deResults = Result.readArray(in);
509
510 assertTrue(deResults.length == 0);
511
512 }
513
514 @Test public void testTimeRange() throws Exception{
515 TimeRange tr = new TimeRange(0,5);
516 byte [] mb = Writables.getBytes(tr);
517 TimeRange deserializedTr =
518 (TimeRange)Writables.getWritable(mb, new TimeRange());
519
520 assertEquals(tr.getMax(), deserializedTr.getMax());
521 assertEquals(tr.getMin(), deserializedTr.getMin());
522
523 }
524
525 @Test public void testKeyValue2() throws Exception {
526 final String name = "testKeyValue2";
527 byte[] row = name.getBytes();
528 byte[] fam = "fam".getBytes();
529 byte[] qf = "qf".getBytes();
530 long ts = System.currentTimeMillis();
531 byte[] val = "val".getBytes();
532
533 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
534
535 byte [] mb = Writables.getBytes(kv);
536 KeyValue deserializedKv =
537 (KeyValue)Writables.getWritable(mb, new KeyValue());
538 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
539 assertEquals(kv.getOffset(), deserializedKv.getOffset());
540 assertEquals(kv.getLength(), deserializedKv.getLength());
541 }
542
543 protected static final int MAXVERSIONS = 3;
544 protected final static byte [] fam1 = Bytes.toBytes("colfamily1");
545 protected final static byte [] fam2 = Bytes.toBytes("colfamily2");
546 protected final static byte [] fam3 = Bytes.toBytes("colfamily3");
547 protected static final byte [][] COLUMNS = {fam1, fam2, fam3};
548
549
550
551
552
553
554
555 protected HTableDescriptor createTableDescriptor(final String name) {
556 return createTableDescriptor(name, MAXVERSIONS);
557 }
558
559
560
561
562
563
564
565
566 protected HTableDescriptor createTableDescriptor(final String name,
567 final int versions) {
568 HTableDescriptor htd = new HTableDescriptor(name);
569 htd.addFamily(new HColumnDescriptor(fam1, versions,
570 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
571 Integer.MAX_VALUE, HConstants.FOREVER,
572 HColumnDescriptor.DEFAULT_BLOOMFILTER,
573 HConstants.REPLICATION_SCOPE_LOCAL));
574 htd.addFamily(new HColumnDescriptor(fam2, versions,
575 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
576 Integer.MAX_VALUE, HConstants.FOREVER,
577 HColumnDescriptor.DEFAULT_BLOOMFILTER,
578 HConstants.REPLICATION_SCOPE_LOCAL));
579 htd.addFamily(new HColumnDescriptor(fam3, versions,
580 HColumnDescriptor.DEFAULT_COMPRESSION, false, false,
581 Integer.MAX_VALUE, HConstants.FOREVER,
582 HColumnDescriptor.DEFAULT_BLOOMFILTER,
583 HConstants.REPLICATION_SCOPE_LOCAL));
584 return htd;
585 }
586 }