1   /**
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.client;
21  
22  import static org.junit.Assert.*;
23  
24  import java.io.IOException;
25  import java.util.Arrays;
26  import java.util.Collections;
27  import java.util.List;
28  
29  import org.apache.commons.logging.Log;
30  import org.apache.commons.logging.LogFactory;
31  import org.apache.hadoop.hbase.*;
32  import org.apache.hadoop.hbase.util.Bytes;
33  import org.junit.After;
34  import org.junit.AfterClass;
35  import org.junit.Before;
36  import org.junit.BeforeClass;
37  import org.junit.Test;
38  import org.junit.experimental.categories.Category;
39  
40  /**
41   * Run tests related to {@link TimestampsFilter} using HBase client APIs.
42   * Sets up the HBase mini cluster once at start. Each creates a table
43   * named for the method and does its stuff against that.
44   */
45  @Category(LargeTests.class)
46  public class TestMultipleTimestamps {
47    final Log LOG = LogFactory.getLog(getClass());
48    private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
49  
50    /**
51     * @throws java.lang.Exception
52     */
53    @BeforeClass
54    public static void setUpBeforeClass() throws Exception {
55      TEST_UTIL.startMiniCluster();
56    }
57  
58    /**
59     * @throws java.lang.Exception
60     */
61    @AfterClass
62    public static void tearDownAfterClass() throws Exception {
63      TEST_UTIL.shutdownMiniCluster();
64    }
65  
66    /**
67     * @throws java.lang.Exception
68     */
69    @Before
70    public void setUp() throws Exception {
71      // Nothing to do.
72    }
73  
74    /**
75     * @throws java.lang.Exception
76     */
77    @After
78    public void tearDown() throws Exception {
79      // Nothing to do.
80    }
81  
82    @Test
83    public void testReseeksWithOneColumnMiltipleTimestamp() throws IOException {
84      byte [] TABLE = Bytes.toBytes("testReseeksWithOne" +
85      "ColumnMiltipleTimestamps");
86      byte [] FAMILY = Bytes.toBytes("event_log");
87      byte [][] FAMILIES = new byte[][] { FAMILY };
88  
89      // create table; set versions to max...
90      HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
91  
92      Integer[] putRows = new Integer[] {1, 3, 5, 7};
93      Integer[] putColumns = new Integer[] { 1, 3, 5};
94      Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
95  
96      Integer[] scanRows = new Integer[] {3, 5};
97      Integer[] scanColumns = new Integer[] {3};
98      Long[] scanTimestamps = new Long[] {3L, 4L};
99      int scanMaxVersions = 2;
100 
101     put(ht, FAMILY, putRows, putColumns, putTimestamps);
102 
103     TEST_UTIL.flush(TABLE);
104 
105     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
106         scanTimestamps, scanMaxVersions);
107 
108     KeyValue[] kvs;
109 
110     kvs = scanner.next().raw();
111     assertEquals(2, kvs.length);
112     checkOneCell(kvs[0], FAMILY, 3, 3, 4);
113     checkOneCell(kvs[1], FAMILY, 3, 3, 3);
114     kvs = scanner.next().raw();
115     assertEquals(2, kvs.length);
116     checkOneCell(kvs[0], FAMILY, 5, 3, 4);
117     checkOneCell(kvs[1], FAMILY, 5, 3, 3);
118 
119     ht.close();
120   }
121 
122   @Test
123   public void testReseeksWithMultipleColumnOneTimestamp() throws IOException {
124     LOG.info("testReseeksWithMultipleColumnOneTimestamp");
125     byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
126     "ColumnOneTimestamps");
127     byte [] FAMILY = Bytes.toBytes("event_log");
128     byte [][] FAMILIES = new byte[][] { FAMILY };
129 
130     // create table; set versions to max...
131     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
132 
133     Integer[] putRows = new Integer[] {1, 3, 5, 7};
134     Integer[] putColumns = new Integer[] { 1, 3, 5};
135     Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
136 
137     Integer[] scanRows = new Integer[] {3, 5};
138     Integer[] scanColumns = new Integer[] {3,4};
139     Long[] scanTimestamps = new Long[] {3L};
140     int scanMaxVersions = 2;
141 
142     put(ht, FAMILY, putRows, putColumns, putTimestamps);
143 
144     TEST_UTIL.flush(TABLE);
145 
146     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
147         scanTimestamps, scanMaxVersions);
148 
149     KeyValue[] kvs;
150 
151     kvs = scanner.next().raw();
152     assertEquals(1, kvs.length);
153     checkOneCell(kvs[0], FAMILY, 3, 3, 3);
154     kvs = scanner.next().raw();
155     assertEquals(1, kvs.length);
156     checkOneCell(kvs[0], FAMILY, 5, 3, 3);
157 
158     ht.close();
159   }
160 
161   @Test
162   public void testReseeksWithMultipleColumnMultipleTimestamp() throws
163   IOException {
164     LOG.info("testReseeksWithMultipleColumnMultipleTimestamp");
165 
166     byte [] TABLE = Bytes.toBytes("testReseeksWithMultiple" +
167     "ColumnMiltipleTimestamps");
168     byte [] FAMILY = Bytes.toBytes("event_log");
169     byte [][] FAMILIES = new byte[][] { FAMILY };
170 
171     // create table; set versions to max...
172     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
173 
174     Integer[] putRows = new Integer[] {1, 3, 5, 7};
175     Integer[] putColumns = new Integer[] { 1, 3, 5};
176     Long[] putTimestamps = new Long[] {1L, 2L, 3L, 4L, 5L};
177 
178     Integer[] scanRows = new Integer[] {5, 7};
179     Integer[] scanColumns = new Integer[] {3, 4, 5};
180     Long[] scanTimestamps = new Long[] {2l, 3L};
181     int scanMaxVersions = 2;
182 
183     put(ht, FAMILY, putRows, putColumns, putTimestamps);
184 
185     TEST_UTIL.flush(TABLE);
186 
187     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
188         scanTimestamps, scanMaxVersions);
189 
190     KeyValue[] kvs;
191 
192     kvs = scanner.next().raw();
193     assertEquals(4, kvs.length);
194     checkOneCell(kvs[0], FAMILY, 5, 3, 3);
195     checkOneCell(kvs[1], FAMILY, 5, 3, 2);
196     checkOneCell(kvs[2], FAMILY, 5, 5, 3);
197     checkOneCell(kvs[3], FAMILY, 5, 5, 2);
198     kvs = scanner.next().raw();
199     assertEquals(4, kvs.length);
200     checkOneCell(kvs[0], FAMILY, 7, 3, 3);
201     checkOneCell(kvs[1], FAMILY, 7, 3, 2);
202     checkOneCell(kvs[2], FAMILY, 7, 5, 3);
203     checkOneCell(kvs[3], FAMILY, 7, 5, 2);
204 
205     ht.close();
206   }
207 
208   @Test
209   public void testReseeksWithMultipleFiles() throws IOException {
210     LOG.info("testReseeksWithMultipleFiles");
211     byte [] TABLE = Bytes.toBytes("testReseeksWithMultipleFiles");
212     byte [] FAMILY = Bytes.toBytes("event_log");
213     byte [][] FAMILIES = new byte[][] { FAMILY };
214 
215     // create table; set versions to max...
216     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
217 
218     Integer[] putRows1 = new Integer[] {1, 2, 3};
219     Integer[] putColumns1 = new Integer[] { 2, 5, 6};
220     Long[] putTimestamps1 = new Long[] {1L, 2L, 5L};
221 
222     Integer[] putRows2 = new Integer[] {6, 7};
223     Integer[] putColumns2 = new Integer[] {3, 6};
224     Long[] putTimestamps2 = new Long[] {4L, 5L};
225 
226     Integer[] putRows3 = new Integer[] {2, 3, 5};
227     Integer[] putColumns3 = new Integer[] {1, 2, 3};
228     Long[] putTimestamps3 = new Long[] {4L,8L};
229 
230 
231     Integer[] scanRows = new Integer[] {3, 5, 7};
232     Integer[] scanColumns = new Integer[] {3, 4, 5};
233     Long[] scanTimestamps = new Long[] {2l, 4L};
234     int scanMaxVersions = 5;
235 
236     put(ht, FAMILY, putRows1, putColumns1, putTimestamps1);
237     TEST_UTIL.flush(TABLE);
238     put(ht, FAMILY, putRows2, putColumns2, putTimestamps2);
239     TEST_UTIL.flush(TABLE);
240     put(ht, FAMILY, putRows3, putColumns3, putTimestamps3);
241 
242     ResultScanner scanner = scan(ht, FAMILY, scanRows, scanColumns,
243         scanTimestamps, scanMaxVersions);
244 
245     KeyValue[] kvs;
246 
247     kvs = scanner.next().raw();
248     assertEquals(2, kvs.length);
249     checkOneCell(kvs[0], FAMILY, 3, 3, 4);
250     checkOneCell(kvs[1], FAMILY, 3, 5, 2);
251 
252     kvs = scanner.next().raw();
253     assertEquals(1, kvs.length);
254     checkOneCell(kvs[0], FAMILY, 5, 3, 4);
255 
256     kvs = scanner.next().raw();
257     assertEquals(1, kvs.length);
258     checkOneCell(kvs[0], FAMILY, 6, 3, 4);
259 
260     kvs = scanner.next().raw();
261     assertEquals(1, kvs.length);
262     checkOneCell(kvs[0], FAMILY, 7, 3, 4);
263 
264     ht.close();
265   }
266 
267   @Test
268   public void testWithVersionDeletes() throws Exception {
269 
270     // first test from memstore (without flushing).
271     testWithVersionDeletes(false);
272 
273     // run same test against HFiles (by forcing a flush).
274     testWithVersionDeletes(true);
275   }
276 
277   public void testWithVersionDeletes(boolean flushTables) throws IOException {
278     LOG.info("testWithVersionDeletes_"+
279         (flushTables ? "flush" : "noflush"));
280 
281     byte [] TABLE = Bytes.toBytes("testWithVersionDeletes_" +
282         (flushTables ? "flush" : "noflush"));
283 
284     byte [] FAMILY = Bytes.toBytes("event_log");
285     byte [][] FAMILIES = new byte[][] { FAMILY };
286 
287     // create table; set versions to max...
288     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
289 
290     // For row:0, col:0: insert versions 1 through 5.
291     putNVersions(ht, FAMILY, 0, 0, 1, 5);
292 
293     if (flushTables) {
294       TEST_UTIL.flush(TABLE);
295     }
296 
297     // delete version 4.
298     deleteOneVersion(ht, FAMILY, 0, 0, 4);
299 
300     // request a bunch of versions including the deleted version. We should
301     // only get back entries for the versions that exist.
302     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0,
303         Arrays.asList(2L, 3L, 4L, 5L));
304     assertEquals(3, kvs.length);
305     checkOneCell(kvs[0], FAMILY, 0, 0, 5);
306     checkOneCell(kvs[1], FAMILY, 0, 0, 3);
307     checkOneCell(kvs[2], FAMILY, 0, 0, 2);
308 
309     ht.close();
310   }
311 
312   @Test
313   public void testWithMultipleVersionDeletes() throws IOException {
314     LOG.info("testWithMultipleVersionDeletes");
315 
316     byte [] TABLE = Bytes.toBytes("testWithMultipleVersionDeletes");
317     byte [] FAMILY = Bytes.toBytes("event_log");
318     byte [][] FAMILIES = new byte[][] { FAMILY };
319 
320     // create table; set versions to max...
321     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
322 
323     // For row:0, col:0: insert versions 1 through 5.
324     putNVersions(ht, FAMILY, 0, 0, 1, 5);
325 
326     TEST_UTIL.flush(TABLE);
327 
328     // delete all versions before 4.
329     deleteAllVersionsBefore(ht, FAMILY, 0, 0, 4);
330 
331     // request a bunch of versions including the deleted version. We should
332     // only get back entries for the versions that exist.
333     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
334     assertEquals(0, kvs.length);
335 
336     ht.close();
337   }
338 
339   @Test
340   public void testWithColumnDeletes() throws IOException {
341     byte [] TABLE = Bytes.toBytes("testWithColumnDeletes");
342     byte [] FAMILY = Bytes.toBytes("event_log");
343     byte [][] FAMILIES = new byte[][] { FAMILY };
344 
345     // create table; set versions to max...
346     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
347 
348     // For row:0, col:0: insert versions 1 through 5.
349     putNVersions(ht, FAMILY, 0, 0, 1, 5);
350 
351     TEST_UTIL.flush(TABLE);
352 
353     // delete all versions before 4.
354     deleteColumn(ht, FAMILY, 0, 0);
355 
356     // request a bunch of versions including the deleted version. We should
357     // only get back entries for the versions that exist.
358     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
359     assertEquals(0, kvs.length);
360 
361     ht.close();
362   }
363 
364   @Test
365   public void testWithFamilyDeletes() throws IOException {
366     byte [] TABLE = Bytes.toBytes("testWithFamilyDeletes");
367     byte [] FAMILY = Bytes.toBytes("event_log");
368     byte [][] FAMILIES = new byte[][] { FAMILY };
369 
370     // create table; set versions to max...
371     HTable ht = TEST_UTIL.createTable(TABLE, FAMILIES, Integer.MAX_VALUE);
372 
373     // For row:0, col:0: insert versions 1 through 5.
374     putNVersions(ht, FAMILY, 0, 0, 1, 5);
375 
376     TEST_UTIL.flush(TABLE);
377 
378     // delete all versions before 4.
379     deleteFamily(ht, FAMILY, 0);
380 
381     // request a bunch of versions including the deleted version. We should
382     // only get back entries for the versions that exist.
383     KeyValue kvs[] = getNVersions(ht, FAMILY, 0, 0, Arrays.asList(2L, 3L));
384     assertEquals(0, kvs.length);
385 
386     ht.close();
387   }
388 
389   /**
390    * Assert that the passed in KeyValue has expected contents for the
391    * specified row, column & timestamp.
392    */
393   private void checkOneCell(KeyValue kv, byte[] cf,
394       int rowIdx, int colIdx, long ts) {
395 
396     String ctx = "rowIdx=" + rowIdx + "; colIdx=" + colIdx + "; ts=" + ts;
397 
398     assertEquals("Row mismatch which checking: " + ctx,
399         "row:"+ rowIdx, Bytes.toString(kv.getRow()));
400 
401     assertEquals("ColumnFamily mismatch while checking: " + ctx,
402         Bytes.toString(cf), Bytes.toString(kv.getFamily()));
403 
404     assertEquals("Column qualifier mismatch while checking: " + ctx,
405         "column:" + colIdx,
406         Bytes.toString(kv.getQualifier()));
407 
408     assertEquals("Timestamp mismatch while checking: " + ctx,
409         ts, kv.getTimestamp());
410 
411     assertEquals("Value mismatch while checking: " + ctx,
412         "value-version-" + ts, Bytes.toString(kv.getValue()));
413   }
414 
415   /**
416    * Uses the TimestampFilter on a Get to request a specified list of
417    * versions for the row/column specified by rowIdx & colIdx.
418    *
419    */
420   private  KeyValue[] getNVersions(HTable ht, byte[] cf, int rowIdx,
421       int colIdx, List<Long> versions)
422   throws IOException {
423     byte row[] = Bytes.toBytes("row:" + rowIdx);
424     byte column[] = Bytes.toBytes("column:" + colIdx);
425     Get get = new Get(row);
426     get.addColumn(cf, column);
427     get.setMaxVersions();
428     get.setTimeRange(Collections.min(versions), Collections.max(versions)+1);
429     Result result = ht.get(get);
430 
431     return result.raw();
432   }
433 
434   private  ResultScanner scan(HTable ht, byte[] cf,
435       Integer[] rowIndexes, Integer[] columnIndexes,
436       Long[] versions, int maxVersions)
437   throws IOException {
438     Arrays.asList(rowIndexes);
439     byte startRow[] = Bytes.toBytes("row:" +
440         Collections.min( Arrays.asList(rowIndexes)));
441     byte endRow[] = Bytes.toBytes("row:" +
442         Collections.max( Arrays.asList(rowIndexes))+1);
443     Scan scan = new Scan(startRow, endRow);
444     for (Integer colIdx: columnIndexes) {
445       byte column[] = Bytes.toBytes("column:" + colIdx);
446       scan.addColumn(cf, column);
447     }
448     scan.setMaxVersions(maxVersions);
449     scan.setTimeRange(Collections.min(Arrays.asList(versions)),
450         Collections.max(Arrays.asList(versions))+1);
451     ResultScanner scanner = ht.getScanner(scan);
452     return scanner;
453   }
454 
455   private void put(HTable ht, byte[] cf, Integer[] rowIndexes,
456       Integer[] columnIndexes, Long[] versions)
457   throws IOException {
458     for (int rowIdx: rowIndexes) {
459       byte row[] = Bytes.toBytes("row:" + rowIdx);
460       Put put = new Put(row);
461       put.setWriteToWAL(false);
462       for(int colIdx: columnIndexes) {
463         byte column[] = Bytes.toBytes("column:" + colIdx);
464         for (long version: versions) {
465           put.add(cf, column, version, Bytes.toBytes("value-version-" +
466               version));
467         }
468       }
469       ht.put(put);
470     }
471   }
472 
473   /**
474    * Insert in specific row/column versions with timestamps
475    * versionStart..versionEnd.
476    */
477   private void putNVersions(HTable ht, byte[] cf, int rowIdx, int colIdx,
478       long versionStart, long versionEnd)
479   throws IOException {
480     byte row[] = Bytes.toBytes("row:" + rowIdx);
481     byte column[] = Bytes.toBytes("column:" + colIdx);
482     Put put = new Put(row);
483     put.setWriteToWAL(false);
484 
485     for (long idx = versionStart; idx <= versionEnd; idx++) {
486       put.add(cf, column, idx, Bytes.toBytes("value-version-" + idx));
487     }
488 
489     ht.put(put);
490   }
491 
492   /**
493    * For row/column specified by rowIdx/colIdx, delete the cell
494    * corresponding to the specified version.
495    */
496   private void deleteOneVersion(HTable ht, byte[] cf, int rowIdx,
497       int colIdx, long version)
498   throws IOException {
499     byte row[] = Bytes.toBytes("row:" + rowIdx);
500     byte column[] = Bytes.toBytes("column:" + colIdx);
501     Delete del = new Delete(row);
502     del.deleteColumn(cf, column, version);
503     ht.delete(del);
504   }
505 
506   /**
507    * For row/column specified by rowIdx/colIdx, delete all cells
508    * preceeding the specified version.
509    */
510   private void deleteAllVersionsBefore(HTable ht, byte[] cf, int rowIdx,
511       int colIdx, long version)
512   throws IOException {
513     byte row[] = Bytes.toBytes("row:" + rowIdx);
514     byte column[] = Bytes.toBytes("column:" + colIdx);
515     Delete del = new Delete(row);
516     del.deleteColumns(cf, column, version);
517     ht.delete(del);
518   }
519 
520   private void deleteColumn(HTable ht, byte[] cf, int rowIdx, int colIdx) throws IOException {
521     byte row[] = Bytes.toBytes("row:" + rowIdx);
522     byte column[] = Bytes.toBytes("column:" + colIdx);
523     Delete del = new Delete(row);
524     del.deleteColumns(cf, column);
525     ht.delete(del);
526   }
527 
528   private void deleteFamily(HTable ht, byte[] cf, int rowIdx) throws IOException {
529     byte row[] = Bytes.toBytes("row:" + rowIdx);
530     Delete del = new Delete(row);
531     del.deleteFamily(cf);
532     ht.delete(del);
533   }
534 
535   @org.junit.Rule
536   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
537     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
538 }
539 
540