1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.client;
20
21 import static org.junit.Assert.*;
22
23 import java.io.IOException;
24 import java.util.ArrayList;
25 import java.util.Arrays;
26 import java.util.List;
27
28 import org.apache.commons.logging.Log;
29 import org.apache.commons.logging.LogFactory;
30 import org.apache.hadoop.hbase.*;
31 import org.apache.hadoop.hbase.filter.Filter;
32 import org.apache.hadoop.hbase.filter.TimestampsFilter;
33 import org.apache.hadoop.hbase.util.Bytes;
34 import org.junit.After;
35 import org.junit.AfterClass;
36 import org.junit.Before;
37 import org.junit.BeforeClass;
38 import org.junit.Test;
39 import org.junit.experimental.categories.Category;
40
41
42
43
44
45
46 @Category(MediumTests.class)
47 public class TestTimestampsFilter {
48 final Log LOG = LogFactory.getLog(getClass());
49 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
50
51
52
53
54 @BeforeClass
55 public static void setUpBeforeClass() throws Exception {
56 TEST_UTIL.startMiniCluster();
57 }
58
59
60
61
62 @AfterClass
63 public static void tearDownAfterClass() throws Exception {
64 TEST_UTIL.shutdownMiniCluster();
65 }
66
67
68
69
70 @Before
71 public void setUp() throws Exception {
72
73 }
74
75
76
77
78 @After
79 public void tearDown() throws Exception {
80
81 }
82
83
84
85
86
87
88
89
90
91 @Test
92 public void testTimestampsFilter() throws Exception {
93 byte [] TABLE = Bytes.toBytes("testTimestampsFilter");
94 byte [] FAMILY = Bytes.toBytes("event_log");
95 byte [][] FAMILIES = new byte[][] { FAMILY };
96 KeyValue kvs[];
97
98
99 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
100
101 for (int rowIdx = 0; rowIdx < 5; rowIdx++) {
102 for (int colIdx = 0; colIdx < 5; colIdx++) {
103
104 putNVersions(ht, FAMILY, rowIdx, colIdx, 201, 300);
105
106 putNVersions(ht, FAMILY, rowIdx, colIdx, 1, 100);
107 }
108 }
109
110
111 verifyInsertedValues(ht, FAMILY);
112
113 TEST_UTIL.flush();
114
115
116 verifyInsertedValues(ht, FAMILY);
117
118
119
120 for (int rowIdx = 0; rowIdx < 5; rowIdx++) {
121 for (int colIdx = 0; colIdx < 5; colIdx++) {
122 putNVersions(ht, FAMILY, rowIdx, colIdx, 301, 400);
123 putNVersions(ht, FAMILY, rowIdx, colIdx, 101, 200);
124 }
125 }
126
127 for (int rowIdx = 0; rowIdx < 5; rowIdx++) {
128 for (int colIdx = 0; colIdx < 5; colIdx++) {
129 kvs = getNVersions(ht, FAMILY, rowIdx, colIdx,
130 Arrays.asList(505L, 5L, 105L, 305L, 205L));
131 assertEquals(4, kvs.length);
132 checkOneCell(kvs[0], FAMILY, rowIdx, colIdx, 305);
133 checkOneCell(kvs[1], FAMILY, rowIdx, colIdx, 205);
134 checkOneCell(kvs[2], FAMILY, rowIdx, colIdx, 105);
135 checkOneCell(kvs[3], FAMILY, rowIdx, colIdx, 5);
136 }
137 }
138
139
140
141 kvs = getNVersions(ht, FAMILY, 2, 2, new ArrayList<Long>());
142 assertEquals(0, kvs == null? 0: kvs.length);
143
144
145
146
147
148 Result[] results = scanNVersions(ht, FAMILY, 0, 4,
149 Arrays.asList(6L, 106L, 306L));
150 assertEquals("# of rows returned from scan", 5, results.length);
151 for (int rowIdx = 0; rowIdx < 5; rowIdx++) {
152 kvs = results[rowIdx].raw();
153
154
155 assertEquals("Number of KeyValues in result for row:" + rowIdx,
156 3*5, kvs.length);
157 for (int colIdx = 0; colIdx < 5; colIdx++) {
158 int offset = colIdx * 3;
159 checkOneCell(kvs[offset + 0], FAMILY, rowIdx, colIdx, 306);
160 checkOneCell(kvs[offset + 1], FAMILY, rowIdx, colIdx, 106);
161 checkOneCell(kvs[offset + 2], FAMILY, rowIdx, colIdx, 6);
162 }
163 }
164 ht.close();
165 }
166
167 @Test
168 public void testMultiColumns() throws Exception {
169 byte [] TABLE = Bytes.toBytes("testTimestampsFilterMultiColumns");
170 byte [] FAMILY = Bytes.toBytes("event_log");
171 byte [][] FAMILIES = new byte[][] { FAMILY };
172 KeyValue kvs[];
173
174
175 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
176
177 Put p = new Put(Bytes.toBytes("row"));
178 p.add(FAMILY, Bytes.toBytes("column0"), 3, Bytes.toBytes("value0-3"));
179 p.add(FAMILY, Bytes.toBytes("column1"), 3, Bytes.toBytes("value1-3"));
180 p.add(FAMILY, Bytes.toBytes("column2"), 1, Bytes.toBytes("value2-1"));
181 p.add(FAMILY, Bytes.toBytes("column2"), 2, Bytes.toBytes("value2-2"));
182 p.add(FAMILY, Bytes.toBytes("column2"), 3, Bytes.toBytes("value2-3"));
183 p.add(FAMILY, Bytes.toBytes("column3"), 2, Bytes.toBytes("value3-2"));
184 p.add(FAMILY, Bytes.toBytes("column4"), 1, Bytes.toBytes("value4-1"));
185 p.add(FAMILY, Bytes.toBytes("column4"), 2, Bytes.toBytes("value4-2"));
186 p.add(FAMILY, Bytes.toBytes("column4"), 3, Bytes.toBytes("value4-3"));
187 ht.put(p);
188
189 ArrayList timestamps = new ArrayList();
190 timestamps.add(new Long(3));
191 TimestampsFilter filter = new TimestampsFilter(timestamps);
192
193 Get g = new Get(Bytes.toBytes("row"));
194 g.setFilter(filter);
195 g.setMaxVersions();
196 g.addColumn(FAMILY, Bytes.toBytes("column2"));
197 g.addColumn(FAMILY, Bytes.toBytes("column4"));
198
199 Result result = ht.get(g);
200 for (KeyValue kv : result.list()) {
201 System.out.println("found row " + Bytes.toString(kv.getRow()) +
202 ", column " + Bytes.toString(kv.getQualifier()) + ", value "
203 + Bytes.toString(kv.getValue()));
204 }
205
206 assertEquals(result.list().size(), 2);
207 assertEquals(Bytes.toString(result.list().get(0).getValue()),
208 "value2-3");
209 assertEquals(Bytes.toString(result.list().get(1).getValue()),
210 "value4-3");
211
212 ht.close();
213 }
214
215
216
217
218
219
220 @Test
221 public void testWithVersionDeletes() throws Exception {
222
223
224 testWithVersionDeletes(false);
225
226
227 testWithVersionDeletes(true);
228 }
229
230 private void testWithVersionDeletes(boolean flushTables) throws IOException {
231 byte [] TABLE = Bytes.toBytes("testWithVersionDeletes_" +
232 (flushTables ? "flush" : "noflush"));
233 byte [] FAMILY = Bytes.toBytes("event_log");
234 byte [][] FAMILIES = new byte[][] { FAMILY };
235
236
237 HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
238
239
240 putNVersions(ht, FAMILY, 0, 0, 1, 5);
241
242
243 deleteOneVersion(ht, FAMILY, 0, 0, 4);
244
245 if (flushTables) {
246 TEST_UTIL.flush();
247 }
248
249
250
251 KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L, 4L, 5L));
252 assertEquals(3, kvs.length);
253 checkOneCell(kvs[0], FAMILY, 0, 0, 5);
254 checkOneCell(kvs[1], FAMILY, 0, 0, 3);
255 checkOneCell(kvs[2], FAMILY, 0, 0, 2);
256
257 ht.close();
258 }
259
260 private void verifyInsertedValues(HTable ht, byte[] cf) throws IOException {
261 for (int rowIdx = 0; rowIdx < 5; rowIdx++) {
262 for (int colIdx = 0; colIdx < 5; colIdx++) {
263
264 KeyValue[] kvs = getNVersions(ht, cf, rowIdx, colIdx,
265 Arrays.asList(5L, 300L, 6L, 80L));
266 assertEquals(4, kvs.length);
267 checkOneCell(kvs[0], cf, rowIdx, colIdx, 300);
268 checkOneCell(kvs[1], cf, rowIdx, colIdx, 80);
269 checkOneCell(kvs[2], cf, rowIdx, colIdx, 6);
270 checkOneCell(kvs[3], cf, rowIdx, colIdx, 5);
271
272
273 kvs = getNVersions(ht, cf, rowIdx, colIdx,
274 Arrays.asList(101L, 102L));
275 assertEquals(0, kvs == null? 0: kvs.length);
276
277
278 kvs = getNVersions(ht, cf, rowIdx, colIdx,
279 Arrays.asList(1L, 300L, 105L, 70L, 115L));
280 assertEquals(3, kvs.length);
281 checkOneCell(kvs[0], cf, rowIdx, colIdx, 300);
282 checkOneCell(kvs[1], cf, rowIdx, colIdx, 70);
283 checkOneCell(kvs[2], cf, rowIdx, colIdx, 1);
284 }
285 }
286 }
287
288
289
290
291
292 private void checkOneCell(KeyValue kv, byte[] cf,
293 int rowIdx, int colIdx, long ts) {
294
295 String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
296
297 assertEquals("Row mismatch which checking: " + ctx,
298 "row:"+ rowIdx, Bytes.toString(kv.getRow()));
299
300 assertEquals("ColumnFamily mismatch while checking: " + ctx,
301 Bytes.toString(cf), Bytes.toString(kv.getFamily()));
302
303 assertEquals("Column qualifier mismatch while checking: " + ctx,
304 "column:" + colIdx,
305 Bytes.toString(kv.getQualifier()));
306
307 assertEquals("Timestamp mismatch while checking: " + ctx,
308 ts, kv.getTimestamp());
309
310 assertEquals("Value mismatch while checking: " + ctx,
311 "value-version-" + ts, Bytes.toString(kv.getValue()));
312 }
313
314
315
316
317
318
319 private KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx,
320 int colIdx, List<Long> versions)
321 throws IOException {
322 byte row[] = Bytes.toBytes("row:" + rowIdx);
323 byte column[] = Bytes.toBytes("column:" + colIdx);
324 Filter filter = new TimestampsFilter(versions);
325 Get get = new Get(row);
326 get.addColumn(cf, column);
327 get.setFilter(filter);
328 get.setMaxVersions();
329 Result result = ht.get(get);
330
331 return result.raw();
332 }
333
334
335
336
337
338 private Result[] scanNVersions(HTable ht, byte[] cf, int startRowIdx,
339 int endRowIdx, List<Long> versions)
340 throws IOException {
341 byte startRow[] = Bytes.toBytes("row:" + startRowIdx);
342 byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1);
343 Filter filter = new TimestampsFilter(versions);
344 Scan scan = new Scan(startRow, endRow);
345 scan.setFilter(filter);
346 scan.setMaxVersions();
347 ResultScanner scanner = ht.getScanner(scan);
348 return scanner.next(endRowIdx - startRowIdx + 1);
349 }
350
351
352
353
354
355 private void putNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx,
356 long versionStart, long versionEnd)
357 throws IOException {
358 byte row[] = Bytes.toBytes("row:" + rowIdx);
359 byte column[] = Bytes.toBytes("column:" + colIdx);
360 Put put = new Put(row);
361 put.setDurability(Durability.SKIP_WAL);
362
363 for (long idx = versionStart; idx <= versionEnd; idx++) {
364 put.add(cf, column, idx, Bytes.toBytes("value-version-" + idx));
365 }
366
367 ht.put(put);
368 }
369
370
371
372
373
374 private void deleteOneVersion(HTable ht, byte[] cf, int rowIdx,
375 int colIdx, long version)
376 throws IOException {
377 byte row[] = Bytes.toBytes("row:" + rowIdx);
378 byte column[] = Bytes.toBytes("column:" + colIdx);
379 Delete del = new Delete(row);
380 del.deleteColumn(cf, column, version);
381 ht.delete(del);
382 }
383
384 }
385
386