1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.apache.hadoop.hbase;
21
22
23 import java.io.ByteArrayOutputStream;
24 import java.io.DataOutputStream;
25 import java.util.ArrayList;
26 import java.util.List;
27 import java.util.Map;
28 import java.util.NavigableSet;
29 import java.util.Set;
30
31 import org.apache.hadoop.hbase.client.Delete;
32 import org.apache.hadoop.hbase.client.Get;
33 import org.apache.hadoop.hbase.client.Put;
34 import org.apache.hadoop.hbase.client.Result;
35 import org.apache.hadoop.hbase.client.RowLock;
36 import org.apache.hadoop.hbase.client.Scan;
37 import org.apache.hadoop.hbase.filter.BinaryComparator;
38 import org.apache.hadoop.hbase.filter.Filter;
39 import org.apache.hadoop.hbase.filter.PrefixFilter;
40 import org.apache.hadoop.hbase.filter.RowFilter;
41 import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
42 import org.apache.hadoop.hbase.io.HbaseMapWritable;
43 import org.apache.hadoop.hbase.io.TimeRange;
44 import org.apache.hadoop.hbase.util.Bytes;
45 import org.apache.hadoop.hbase.util.Writables;
46 import org.apache.hadoop.io.DataInputBuffer;
47
48
49
50
51 public class TestSerialization extends HBaseTestCase {
52
53 @Override
54 protected void setUp() throws Exception {
55 super.setUp();
56 }
57
58 @Override
59 protected void tearDown() throws Exception {
60 super.tearDown();
61 }
62
63 public void testCompareFilter() throws Exception {
64 Filter f = new RowFilter(CompareOp.EQUAL,
65 new BinaryComparator(Bytes.toBytes("testRowOne-2")));
66 byte [] bytes = Writables.getBytes(f);
67 Filter ff = (Filter)Writables.getWritable(bytes, new RowFilter());
68 assertNotNull(ff);
69 }
70
71 public void testKeyValue() throws Exception {
72 byte [] row = Bytes.toBytes(getName());
73 byte [] family = Bytes.toBytes(getName());
74 byte [] qualifier = Bytes.toBytes(getName());
75 KeyValue original = new KeyValue(row, family, qualifier);
76 byte [] bytes = Writables.getBytes(original);
77 KeyValue newone = (KeyValue)Writables.getWritable(bytes, new KeyValue());
78 assertTrue(KeyValue.COMPARATOR.compare(original, newone) == 0);
79 }
80
81 @SuppressWarnings("unchecked")
82 public void testHbaseMapWritable() throws Exception {
83 HbaseMapWritable<byte [], byte []> hmw =
84 new HbaseMapWritable<byte[], byte[]>();
85 hmw.put("key".getBytes(), "value".getBytes());
86 byte [] bytes = Writables.getBytes(hmw);
87 hmw = (HbaseMapWritable<byte[], byte[]>)
88 Writables.getWritable(bytes, new HbaseMapWritable<byte [], byte []>());
89 assertTrue(hmw.size() == 1);
90 assertTrue(Bytes.equals("value".getBytes(), hmw.get("key".getBytes())));
91 }
92
93 public void testHMsg() throws Exception {
94 HMsg m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE);
95 byte [] mb = Writables.getBytes(m);
96 HMsg deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
97 assertTrue(m.equals(deserializedHMsg));
98 m = new HMsg(HMsg.Type.MSG_REGIONSERVER_QUIESCE,
99 new HRegionInfo(new HTableDescriptor(getName()),
100 HConstants.EMPTY_BYTE_ARRAY, HConstants.EMPTY_BYTE_ARRAY),
101 "Some message".getBytes());
102 mb = Writables.getBytes(m);
103 deserializedHMsg = (HMsg)Writables.getWritable(mb, new HMsg());
104 assertTrue(m.equals(deserializedHMsg));
105 }
106
107 public void testTableDescriptor() throws Exception {
108 HTableDescriptor htd = createTableDescriptor(getName());
109 byte [] mb = Writables.getBytes(htd);
110 HTableDescriptor deserializedHtd =
111 (HTableDescriptor)Writables.getWritable(mb, new HTableDescriptor());
112 assertEquals(htd.getNameAsString(), deserializedHtd.getNameAsString());
113 }
114
115
116
117
118
119 public void testRegionInfo() throws Exception {
120 HTableDescriptor htd = new HTableDescriptor(getName());
121 String [] families = new String [] {"info", "anchor"};
122 for (int i = 0; i < families.length; i++) {
123 htd.addFamily(new HColumnDescriptor(families[i]));
124 }
125 HRegionInfo hri = new HRegionInfo(htd,
126 HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
127 byte [] hrib = Writables.getBytes(hri);
128 HRegionInfo deserializedHri =
129 (HRegionInfo)Writables.getWritable(hrib, new HRegionInfo());
130 assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName());
131 assertEquals(hri.getTableDesc().getFamilies().size(),
132 deserializedHri.getTableDesc().getFamilies().size());
133 }
134
135
136
137
138
139 public void testServerInfo() throws Exception {
140 HServerInfo hsi = new HServerInfo(new HServerAddress("0.0.0.0:123"), -1,
141 1245, "default name");
142 byte [] b = Writables.getBytes(hsi);
143 HServerInfo deserializedHsi =
144 (HServerInfo)Writables.getWritable(b, new HServerInfo());
145 assertTrue(hsi.equals(deserializedHsi));
146 }
147
148 public void testPut() throws Exception{
149 byte[] row = "row".getBytes();
150 byte[] fam = "fam".getBytes();
151 byte[] qf1 = "qf1".getBytes();
152 byte[] qf2 = "qf2".getBytes();
153 byte[] qf3 = "qf3".getBytes();
154 byte[] qf4 = "qf4".getBytes();
155 byte[] qf5 = "qf5".getBytes();
156 byte[] qf6 = "qf6".getBytes();
157 byte[] qf7 = "qf7".getBytes();
158 byte[] qf8 = "qf8".getBytes();
159
160 long ts = System.currentTimeMillis();
161 byte[] val = "val".getBytes();
162
163 Put put = new Put(row);
164 put.add(fam, qf1, ts, val);
165 put.add(fam, qf2, ts, val);
166 put.add(fam, qf3, ts, val);
167 put.add(fam, qf4, ts, val);
168 put.add(fam, qf5, ts, val);
169 put.add(fam, qf6, ts, val);
170 put.add(fam, qf7, ts, val);
171 put.add(fam, qf8, ts, val);
172
173 byte[] sb = Writables.getBytes(put);
174 Put desPut = (Put)Writables.getWritable(sb, new Put());
175
176
177
178
179
180
181
182 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
183 List<KeyValue> list = null;
184 List<KeyValue> desList = null;
185 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
186 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
187 list = entry.getValue();
188 desList = desPut.getFamilyMap().get(entry.getKey());
189 for(int i=0; i<list.size(); i++){
190 assertTrue(list.get(i).equals(desList.get(i)));
191 }
192 }
193 }
194
195
196 public void testPut2() throws Exception{
197 byte[] row = "testAbort,,1243116656250".getBytes();
198 byte[] fam = "historian".getBytes();
199 byte[] qf1 = "creation".getBytes();
200
201 long ts = 9223372036854775807L;
202 byte[] val = "dont-care".getBytes();
203
204 Put put = new Put(row);
205 put.add(fam, qf1, ts, val);
206
207 byte[] sb = Writables.getBytes(put);
208 Put desPut = (Put)Writables.getWritable(sb, new Put());
209
210 assertTrue(Bytes.equals(put.getRow(), desPut.getRow()));
211 List<KeyValue> list = null;
212 List<KeyValue> desList = null;
213 for(Map.Entry<byte[], List<KeyValue>> entry : put.getFamilyMap().entrySet()){
214 assertTrue(desPut.getFamilyMap().containsKey(entry.getKey()));
215 list = entry.getValue();
216 desList = desPut.getFamilyMap().get(entry.getKey());
217 for(int i=0; i<list.size(); i++){
218 assertTrue(list.get(i).equals(desList.get(i)));
219 }
220 }
221 }
222
223
224 public void testDelete() throws Exception{
225 byte[] row = "row".getBytes();
226 byte[] fam = "fam".getBytes();
227 byte[] qf1 = "qf1".getBytes();
228
229 long ts = System.currentTimeMillis();
230
231 Delete delete = new Delete(row);
232 delete.deleteColumn(fam, qf1, ts);
233
234 byte[] sb = Writables.getBytes(delete);
235 Delete desDelete = (Delete)Writables.getWritable(sb, new Delete());
236
237 assertTrue(Bytes.equals(delete.getRow(), desDelete.getRow()));
238 List<KeyValue> list = null;
239 List<KeyValue> desList = null;
240 for(Map.Entry<byte[], List<KeyValue>> entry :
241 delete.getFamilyMap().entrySet()){
242 assertTrue(desDelete.getFamilyMap().containsKey(entry.getKey()));
243 list = entry.getValue();
244 desList = desDelete.getFamilyMap().get(entry.getKey());
245 for(int i=0; i<list.size(); i++){
246 assertTrue(list.get(i).equals(desList.get(i)));
247 }
248 }
249 }
250
251 public void testGet() throws Exception{
252 byte[] row = "row".getBytes();
253 byte[] fam = "fam".getBytes();
254 byte[] qf1 = "qf1".getBytes();
255
256 long ts = System.currentTimeMillis();
257 int maxVersions = 2;
258 long lockid = 5;
259 RowLock rowLock = new RowLock(lockid);
260
261 Get get = new Get(row, rowLock);
262 get.addColumn(fam, qf1);
263 get.setTimeRange(ts, ts+1);
264 get.setMaxVersions(maxVersions);
265
266 byte[] sb = Writables.getBytes(get);
267 Get desGet = (Get)Writables.getWritable(sb, new Get());
268
269 assertTrue(Bytes.equals(get.getRow(), desGet.getRow()));
270 Set<byte[]> set = null;
271 Set<byte[]> desSet = null;
272
273 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
274 get.getFamilyMap().entrySet()){
275 assertTrue(desGet.getFamilyMap().containsKey(entry.getKey()));
276 set = entry.getValue();
277 desSet = desGet.getFamilyMap().get(entry.getKey());
278 for(byte [] qualifier : set){
279 assertTrue(desSet.contains(qualifier));
280 }
281 }
282
283 assertEquals(get.getLockId(), desGet.getLockId());
284 assertEquals(get.getMaxVersions(), desGet.getMaxVersions());
285 TimeRange tr = get.getTimeRange();
286 TimeRange desTr = desGet.getTimeRange();
287 assertEquals(tr.getMax(), desTr.getMax());
288 assertEquals(tr.getMin(), desTr.getMin());
289 }
290
291
292 public void testScan() throws Exception{
293 byte[] startRow = "startRow".getBytes();
294 byte[] stopRow = "stopRow".getBytes();
295 byte[] fam = "fam".getBytes();
296 byte[] qf1 = "qf1".getBytes();
297
298 long ts = System.currentTimeMillis();
299 int maxVersions = 2;
300
301 Scan scan = new Scan(startRow, stopRow);
302 scan.addColumn(fam, qf1);
303 scan.setTimeRange(ts, ts+1);
304 scan.setMaxVersions(maxVersions);
305
306 byte[] sb = Writables.getBytes(scan);
307 Scan desScan = (Scan)Writables.getWritable(sb, new Scan());
308
309 assertTrue(Bytes.equals(scan.getStartRow(), desScan.getStartRow()));
310 assertTrue(Bytes.equals(scan.getStopRow(), desScan.getStopRow()));
311 assertEquals(scan.getCacheBlocks(), desScan.getCacheBlocks());
312 Set<byte[]> set = null;
313 Set<byte[]> desSet = null;
314
315 for(Map.Entry<byte[], NavigableSet<byte[]>> entry :
316 scan.getFamilyMap().entrySet()){
317 assertTrue(desScan.getFamilyMap().containsKey(entry.getKey()));
318 set = entry.getValue();
319 desSet = desScan.getFamilyMap().get(entry.getKey());
320 for(byte[] column : set){
321 assertTrue(desSet.contains(column));
322 }
323
324
325 scan = new Scan(startRow);
326 byte [] prefix = Bytes.toBytes(getName());
327 scan.setFilter(new PrefixFilter(prefix));
328 sb = Writables.getBytes(scan);
329 desScan = (Scan)Writables.getWritable(sb, new Scan());
330 Filter f = desScan.getFilter();
331 assertTrue(f instanceof PrefixFilter);
332 }
333
334 assertEquals(scan.getMaxVersions(), desScan.getMaxVersions());
335 TimeRange tr = scan.getTimeRange();
336 TimeRange desTr = desScan.getTimeRange();
337 assertEquals(tr.getMax(), desTr.getMax());
338 assertEquals(tr.getMin(), desTr.getMin());
339 }
340
341 public void testResultEmpty() throws Exception {
342 List<KeyValue> keys = new ArrayList<KeyValue>();
343 Result r = new Result(keys);
344 assertTrue(r.isEmpty());
345 byte [] rb = Writables.getBytes(r);
346 Result deserializedR = (Result)Writables.getWritable(rb, new Result());
347 assertTrue(deserializedR.isEmpty());
348 }
349
350
351 public void testResult() throws Exception {
352 byte [] rowA = Bytes.toBytes("rowA");
353 byte [] famA = Bytes.toBytes("famA");
354 byte [] qfA = Bytes.toBytes("qfA");
355 byte [] valueA = Bytes.toBytes("valueA");
356
357 byte [] rowB = Bytes.toBytes("rowB");
358 byte [] famB = Bytes.toBytes("famB");
359 byte [] qfB = Bytes.toBytes("qfB");
360 byte [] valueB = Bytes.toBytes("valueB");
361
362 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
363 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
364
365 Result result = new Result(new KeyValue[]{kvA, kvB});
366
367 byte [] rb = Writables.getBytes(result);
368 Result deResult = (Result)Writables.getWritable(rb, new Result());
369
370 assertTrue("results are not equivalent, first key mismatch",
371 result.sorted()[0].equals(deResult.sorted()[0]));
372
373 assertTrue("results are not equivalent, second key mismatch",
374 result.sorted()[1].equals(deResult.sorted()[1]));
375
376
377 Result r = new Result();
378 byte [] b = Writables.getBytes(r);
379 Result deserialized = (Result)Writables.getWritable(b, new Result());
380 assertEquals(r.size(), deserialized.size());
381 }
382
383 public void testResultDynamicBuild() throws Exception {
384 byte [] rowA = Bytes.toBytes("rowA");
385 byte [] famA = Bytes.toBytes("famA");
386 byte [] qfA = Bytes.toBytes("qfA");
387 byte [] valueA = Bytes.toBytes("valueA");
388
389 byte [] rowB = Bytes.toBytes("rowB");
390 byte [] famB = Bytes.toBytes("famB");
391 byte [] qfB = Bytes.toBytes("qfB");
392 byte [] valueB = Bytes.toBytes("valueB");
393
394 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
395 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
396
397 Result result = new Result(new KeyValue[]{kvA, kvB});
398
399 byte [] rb = Writables.getBytes(result);
400
401
402
403 Result deResult = (Result)Writables.getWritable(rb, new Result());
404 byte [] row = deResult.getRow();
405 assertTrue(Bytes.equals(row, rowA));
406
407
408 deResult = (Result)Writables.getWritable(rb, new Result());
409 assertTrue("results are not equivalent, first key mismatch",
410 result.sorted()[0].equals(deResult.sorted()[0]));
411 assertTrue("results are not equivalent, second key mismatch",
412 result.sorted()[1].equals(deResult.sorted()[1]));
413
414
415 deResult = (Result)Writables.getWritable(rb, new Result());
416 assertTrue("results are not equivalent, first key mismatch",
417 result.raw()[0].equals(deResult.raw()[0]));
418 assertTrue("results are not equivalent, second key mismatch",
419 result.raw()[1].equals(deResult.raw()[1]));
420
421
422 }
423
424 public void testResultArray() throws Exception {
425 byte [] rowA = Bytes.toBytes("rowA");
426 byte [] famA = Bytes.toBytes("famA");
427 byte [] qfA = Bytes.toBytes("qfA");
428 byte [] valueA = Bytes.toBytes("valueA");
429
430 byte [] rowB = Bytes.toBytes("rowB");
431 byte [] famB = Bytes.toBytes("famB");
432 byte [] qfB = Bytes.toBytes("qfB");
433 byte [] valueB = Bytes.toBytes("valueB");
434
435 KeyValue kvA = new KeyValue(rowA, famA, qfA, valueA);
436 KeyValue kvB = new KeyValue(rowB, famB, qfB, valueB);
437
438
439 Result result1 = new Result(new KeyValue[]{kvA, kvB});
440 Result result2 = new Result(new KeyValue[]{kvB});
441 Result result3 = new Result(new KeyValue[]{kvB});
442
443 Result [] results = new Result [] {result1, result2, result3};
444
445 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
446 DataOutputStream out = new DataOutputStream(byteStream);
447 Result.writeArray(out, results);
448
449 byte [] rb = byteStream.toByteArray();
450
451 DataInputBuffer in = new DataInputBuffer();
452 in.reset(rb, 0, rb.length);
453
454 Result [] deResults = Result.readArray(in);
455
456 assertTrue(results.length == deResults.length);
457
458 for(int i=0;i<results.length;i++) {
459 KeyValue [] keysA = results[i].sorted();
460 KeyValue [] keysB = deResults[i].sorted();
461 assertTrue(keysA.length == keysB.length);
462 for(int j=0;j<keysA.length;j++) {
463 assertTrue("Expected equivalent keys but found:\n" +
464 "KeyA : " + keysA[j].toString() + "\n" +
465 "KeyB : " + keysB[j].toString() + "\n" +
466 keysA.length + " total keys, " + i + "th so far"
467 ,keysA[j].equals(keysB[j]));
468 }
469 }
470
471 }
472
473 public void testResultArrayEmpty() throws Exception {
474 List<KeyValue> keys = new ArrayList<KeyValue>();
475 Result r = new Result(keys);
476 Result [] results = new Result [] {r};
477
478 ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
479 DataOutputStream out = new DataOutputStream(byteStream);
480
481 Result.writeArray(out, results);
482
483 results = null;
484
485 byteStream = new ByteArrayOutputStream();
486 out = new DataOutputStream(byteStream);
487 Result.writeArray(out, results);
488
489 byte [] rb = byteStream.toByteArray();
490
491 DataInputBuffer in = new DataInputBuffer();
492 in.reset(rb, 0, rb.length);
493
494 Result [] deResults = Result.readArray(in);
495
496 assertTrue(deResults.length == 0);
497
498 results = new Result[0];
499
500 byteStream = new ByteArrayOutputStream();
501 out = new DataOutputStream(byteStream);
502 Result.writeArray(out, results);
503
504 rb = byteStream.toByteArray();
505
506 in = new DataInputBuffer();
507 in.reset(rb, 0, rb.length);
508
509 deResults = Result.readArray(in);
510
511 assertTrue(deResults.length == 0);
512
513 }
514
515 public void testTimeRange(String[] args) throws Exception{
516 TimeRange tr = new TimeRange(0,5);
517 byte [] mb = Writables.getBytes(tr);
518 TimeRange deserializedTr =
519 (TimeRange)Writables.getWritable(mb, new TimeRange());
520
521 assertEquals(tr.getMax(), deserializedTr.getMax());
522 assertEquals(tr.getMin(), deserializedTr.getMin());
523
524 }
525
526 public void testKeyValue2() throws Exception {
527 byte[] row = getName().getBytes();
528 byte[] fam = "fam".getBytes();
529 byte[] qf = "qf".getBytes();
530 long ts = System.currentTimeMillis();
531 byte[] val = "val".getBytes();
532
533 KeyValue kv = new KeyValue(row, fam, qf, ts, val);
534
535 byte [] mb = Writables.getBytes(kv);
536 KeyValue deserializedKv =
537 (KeyValue)Writables.getWritable(mb, new KeyValue());
538 assertTrue(Bytes.equals(kv.getBuffer(), deserializedKv.getBuffer()));
539 assertEquals(kv.getOffset(), deserializedKv.getOffset());
540 assertEquals(kv.getLength(), deserializedKv.getLength());
541 }
542 }