1   /**
2    * Copyright 2011 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  package org.apache.hadoop.hbase.regionserver;
21  
22  import java.util.ArrayList;
23  import java.util.List;
24  
25  import org.apache.hadoop.hbase.*;
26  import org.apache.hadoop.hbase.client.Delete;
27  import org.apache.hadoop.hbase.client.Get;
28  import org.apache.hadoop.hbase.client.Put;
29  import org.apache.hadoop.hbase.client.Result;
30  import org.apache.hadoop.hbase.filter.TimestampsFilter;
31  import org.apache.hadoop.hbase.util.Bytes;
32  import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
33  import org.junit.experimental.categories.Category;
34  
35  /**
36   * Test Minimum Versions feature (HBASE-4071).
37   */
38  @Category(SmallTests.class)
39  public class TestMinVersions extends HBaseTestCase {
40    private final byte[] T0 = Bytes.toBytes("0");
41    private final byte[] T1 = Bytes.toBytes("1");
42    private final byte[] T2 = Bytes.toBytes("2");
43    private final byte[] T3 = Bytes.toBytes("3");
44    private final byte[] T4 = Bytes.toBytes("4");
45    private final byte[] T5 = Bytes.toBytes("5");
46  
47    private final byte[] c0 = COLUMNS[0];
48  
49    /**
50     * Verify behavior of getClosestBefore(...)
51     */
52    public void testGetClosestBefore() throws Exception {
53      HTableDescriptor htd = createTableDescriptor(getName(), 1, 1000, 1, false);
54      HRegion region = createNewHRegion(htd, null, null);
55      try {
56  
57        // 2s in the past
58        long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
59  
60        Put p = new Put(T1, ts);
61        p.add(c0, c0, T1);
62        region.put(p);
63  
64        p = new Put(T1, ts+1);
65        p.add(c0, c0, T4);
66        region.put(p);
67  
68        p = new Put(T3, ts);
69        p.add(c0, c0, T3);
70        region.put(p);
71  
72        // now make sure that getClosestBefore(...) get can
73        // rows that would be expired without minVersion.
74        // also make sure it gets the latest version
75        Result r = region.getClosestRowBefore(T1, c0);
76        checkResult(r, c0, T4);
77  
78        r = region.getClosestRowBefore(T2, c0);
79        checkResult(r, c0, T4);
80  
81        // now flush/compact
82        region.flushcache();
83        region.compactStores(true);
84  
85        r = region.getClosestRowBefore(T1, c0);
86        checkResult(r, c0, T4);
87  
88        r = region.getClosestRowBefore(T2, c0);
89        checkResult(r, c0, T4);
90      } finally {
91        region.close();
92        region.getLog().closeAndDelete();
93      }
94    }
95  
96    /**
97     * Test mixed memstore and storefile scanning
98     * with minimum versions.
99     */
100   public void testStoreMemStore() throws Exception {
101     // keep 3 versions minimum
102     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false);
103     HRegion region = createNewHRegion(htd, null, null);
104     // 2s in the past
105     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
106 
107     try {
108       Put p = new Put(T1, ts-1);
109       p.add(c0, c0, T2);
110       region.put(p);
111 
112       p = new Put(T1, ts-3);
113       p.add(c0, c0, T0);
114       region.put(p);
115 
116       // now flush/compact
117       region.flushcache();
118       region.compactStores(true);
119 
120       p = new Put(T1, ts);
121       p.add(c0, c0, T3);
122       region.put(p);
123 
124       p = new Put(T1, ts-2);
125       p.add(c0, c0, T1);
126       region.put(p);
127 
128       p = new Put(T1, ts-3);
129       p.add(c0, c0, T0);
130       region.put(p);
131 
132       // newest version in the memstore
133       // the 2nd oldest in the store file
134       // and the 3rd, 4th oldest also in the memstore
135 
136       Get g = new Get(T1);
137       g.setMaxVersions();
138       Result r = region.get(g, null); // this'll use ScanWildcardColumnTracker
139       checkResult(r, c0, T3,T2,T1);
140 
141       g = new Get(T1);
142       g.setMaxVersions();
143       g.addColumn(c0, c0);
144       r = region.get(g, null);  // this'll use ExplicitColumnTracker
145       checkResult(r, c0, T3,T2,T1);
146     } finally {
147       region.close();
148       region.getLog().closeAndDelete();
149     }
150   }
151 
152   /**
153    * Make sure the Deletes behave as expected with minimum versions
154    */
155   public void testDelete() throws Exception {
156     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, false);
157     HRegion region = createNewHRegion(htd, null, null);
158 
159     // 2s in the past
160     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
161 
162     try {
163       Put p = new Put(T1, ts-2);
164       p.add(c0, c0, T1);
165       region.put(p);
166 
167       p = new Put(T1, ts-1);
168       p.add(c0, c0, T2);
169       region.put(p);
170 
171       p = new Put(T1, ts);
172       p.add(c0, c0, T3);
173       region.put(p);
174 
175       Delete d = new Delete(T1, ts-1, null);
176       region.delete(d, null, true);
177 
178       Get g = new Get(T1);
179       g.setMaxVersions();
180       Result r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
181       checkResult(r, c0, T3);
182 
183       g = new Get(T1);
184       g.setMaxVersions();
185       g.addColumn(c0, c0);
186       r = region.get(g, null);  // this'll use ExplicitColumnTracker
187       checkResult(r, c0, T3);
188 
189       // now flush/compact
190       region.flushcache();
191       region.compactStores(true);
192 
193       // try again
194       g = new Get(T1);
195       g.setMaxVersions();
196       r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
197       checkResult(r, c0, T3);
198 
199       g = new Get(T1);
200       g.setMaxVersions();
201       g.addColumn(c0, c0);
202       r = region.get(g, null);  // this'll use ExplicitColumnTracker
203       checkResult(r, c0, T3);
204     } finally {
205       region.close();
206       region.getLog().closeAndDelete();
207     }
208   }
209 
210   /**
211    * Make sure the memstor behaves correctly with minimum versions
212    */
213   public void testMemStore() throws Exception {
214     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
215     HRegion region = createNewHRegion(htd, null, null);
216 
217     // 2s in the past
218     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
219 
220     try {
221       // 2nd version
222       Put p = new Put(T1, ts-2);
223       p.add(c0, c0, T2);
224       region.put(p);
225 
226       // 3rd version
227       p = new Put(T1, ts-1);
228       p.add(c0, c0, T3);
229       region.put(p);
230 
231       // 4th version
232       p = new Put(T1, ts);
233       p.add(c0, c0, T4);
234       region.put(p);
235 
236       // now flush/compact
237       region.flushcache();
238       region.compactStores(true);
239 
240       // now put the first version (backdated)
241       p = new Put(T1, ts-3);
242       p.add(c0, c0, T1);
243       region.put(p);
244 
245       // now the latest change is in the memstore,
246       // but it is not the latest version
247 
248       Result r = region.get(new Get(T1), null);
249       checkResult(r, c0, T4);
250 
251       Get g = new Get(T1);
252       g.setMaxVersions();
253       r = region.get(g, null); // this'll use ScanWildcardColumnTracker
254       checkResult(r, c0, T4,T3);
255 
256       g = new Get(T1);
257       g.setMaxVersions();
258       g.addColumn(c0, c0);
259       r = region.get(g, null);  // this'll use ExplicitColumnTracker
260       checkResult(r, c0, T4,T3);
261 
262       p = new Put(T1, ts+1);
263       p.add(c0, c0, T5);
264       region.put(p);
265 
266       // now the latest version is in the memstore
267 
268       g = new Get(T1);
269       g.setMaxVersions();
270       r = region.get(g, null);  // this'll use ScanWildcardColumnTracker
271       checkResult(r, c0, T5,T4);
272 
273       g = new Get(T1);
274       g.setMaxVersions();
275       g.addColumn(c0, c0);
276       r = region.get(g, null);  // this'll use ExplicitColumnTracker
277       checkResult(r, c0, T5,T4);
278     } finally {
279       region.close();
280       region.getLog().closeAndDelete();
281     }
282   }
283 
284   /**
285    * Verify basic minimum versions functionality
286    */
287   public void testBaseCase() throws Exception {
288     // 1 version minimum, 1000 versions maximum, ttl = 1s
289     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
290     HRegion region = createNewHRegion(htd, null, null);
291     try {
292 
293       // 2s in the past
294       long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
295 
296        // 1st version
297       Put p = new Put(T1, ts-3);
298       p.add(c0, c0, T1);
299       region.put(p);
300 
301       // 2nd version
302       p = new Put(T1, ts-2);
303       p.add(c0, c0, T2);
304       region.put(p);
305 
306       // 3rd version
307       p = new Put(T1, ts-1);
308       p.add(c0, c0, T3);
309       region.put(p);
310 
311       // 4th version
312       p = new Put(T1, ts);
313       p.add(c0, c0, T4);
314       region.put(p);
315 
316       Result r = region.get(new Get(T1), null);
317       checkResult(r, c0, T4);
318 
319       Get g = new Get(T1);
320       g.setTimeRange(0L, ts+1);
321       r = region.get(g, null);
322       checkResult(r, c0, T4);
323 
324   // oldest version still exists
325       g.setTimeRange(0L, ts-2);
326       r = region.get(g, null);
327       checkResult(r, c0, T1);
328 
329       // gets see only available versions
330       // even before compactions
331       g = new Get(T1);
332       g.setMaxVersions();
333       r = region.get(g, null); // this'll use ScanWildcardColumnTracker
334       checkResult(r, c0, T4,T3);
335 
336       g = new Get(T1);
337       g.setMaxVersions();
338       g.addColumn(c0, c0);
339       r = region.get(g, null);  // this'll use ExplicitColumnTracker
340       checkResult(r, c0, T4,T3);
341 
342       // now flush
343       region.flushcache();
344 
345       // with HBASE-4241 a flush will eliminate the expired rows
346       g = new Get(T1);
347       g.setTimeRange(0L, ts-2);
348       r = region.get(g, null);
349       assertTrue(r.isEmpty());
350 
351       // major compaction
352       region.compactStores(true);
353 
354       // after compaction the 4th version is still available
355       g = new Get(T1);
356       g.setTimeRange(0L, ts+1);
357       r = region.get(g, null);
358       checkResult(r, c0, T4);
359 
360       // so is the 3rd
361       g.setTimeRange(0L, ts);
362       r = region.get(g, null);
363       checkResult(r, c0, T3);
364 
365       // but the 2nd and earlier versions are gone
366       g.setTimeRange(0L, ts-1);
367       r = region.get(g, null);
368       assertTrue(r.isEmpty());
369     } finally {
370       region.close();
371       region.getLog().closeAndDelete();
372     }
373   }
374 
375   /**
376    * Verify that basic filters still behave correctly with
377    * minimum versions enabled.
378    */
379   public void testFilters() throws Exception {
380     HTableDescriptor htd = createTableDescriptor(getName(), 2, 1000, 1, false);
381     HRegion region = createNewHRegion(htd, null, null);
382     final byte [] c1 = COLUMNS[1];
383 
384     // 2s in the past
385     long ts = EnvironmentEdgeManager.currentTimeMillis() - 2000;
386     try {
387 
388       Put p = new Put(T1, ts-3);
389       p.add(c0, c0, T0);
390       p.add(c1, c1, T0);
391       region.put(p);
392 
393       p = new Put(T1, ts-2);
394       p.add(c0, c0, T1);
395       p.add(c1, c1, T1);
396       region.put(p);
397 
398       p = new Put(T1, ts-1);
399       p.add(c0, c0, T2);
400       p.add(c1, c1, T2);
401       region.put(p);
402 
403       p = new Put(T1, ts);
404       p.add(c0, c0, T3);
405       p.add(c1, c1, T3);
406       region.put(p);
407 
408       List<Long> tss = new ArrayList<Long>();
409       tss.add(ts-1);
410       tss.add(ts-2);
411 
412       Get g = new Get(T1);
413       g.addColumn(c1,c1);
414       g.setFilter(new TimestampsFilter(tss));
415       g.setMaxVersions();
416       Result r = region.get(g, null);
417       checkResult(r, c1, T2,T1);
418 
419       g = new Get(T1);
420       g.addColumn(c0,c0);
421       g.setFilter(new TimestampsFilter(tss));
422       g.setMaxVersions();
423       r = region.get(g, null);
424       checkResult(r, c0, T2,T1);
425 
426       // now flush/compact
427       region.flushcache();
428       region.compactStores(true);
429 
430       g = new Get(T1);
431       g.addColumn(c1,c1);
432       g.setFilter(new TimestampsFilter(tss));
433       g.setMaxVersions();
434       r = region.get(g, null);
435       checkResult(r, c1, T2);
436 
437       g = new Get(T1);
438       g.addColumn(c0,c0);
439       g.setFilter(new TimestampsFilter(tss));
440       g.setMaxVersions();
441       r = region.get(g, null);
442       checkResult(r, c0, T2);
443     } finally {
444       region.close();
445       region.getLog().closeAndDelete();
446     }
447   }
448 
449   private void checkResult(Result r, byte[] col, byte[] ... vals) {
450     assertEquals(r.size(), vals.length);
451     List<KeyValue> kvs = r.getColumn(col, col);
452     assertEquals(kvs.size(), vals.length);
453     for (int i=0;i<vals.length;i++) {
454       assertEquals(kvs.get(i).getValue(), vals[i]);
455     }
456   }
457 
458   @org.junit.Rule
459   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
460     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
461 }
462